15190f052SMike Marciniszyn /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
25190f052SMike Marciniszyn /*
35190f052SMike Marciniszyn * Copyright(c) 2018 Intel Corporation.
45190f052SMike Marciniszyn *
55190f052SMike Marciniszyn */
65190f052SMike Marciniszyn #ifndef HFI1_TID_RDMA_H
75190f052SMike Marciniszyn #define HFI1_TID_RDMA_H
85190f052SMike Marciniszyn
9838b6fd2SKaike Wan #include <linux/circ_buf.h>
10838b6fd2SKaike Wan #include "common.h"
11838b6fd2SKaike Wan
12838b6fd2SKaike Wan /* Add a convenience helper */
13838b6fd2SKaike Wan #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
14838b6fd2SKaike Wan #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
15838b6fd2SKaike Wan #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
16838b6fd2SKaike Wan
17f1ab4efaSKaike Wan #define TID_RDMA_MIN_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */
18d22a207dSKaike Wan #define TID_RDMA_MAX_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */
19838b6fd2SKaike Wan #define TID_RDMA_MAX_PAGES (BIT(18) >> PAGE_SHIFT)
20*c2be3865SKaike Wan #define TID_RDMA_SEGMENT_SHIFT 18
21d22a207dSKaike Wan
22a0b34f75SKaike Wan /*
23a0b34f75SKaike Wan * Bit definitions for priv->s_flags.
24a0b34f75SKaike Wan * These bit flags overload the bit flags defined for the QP's s_flags.
25a0b34f75SKaike Wan * Due to the fact that these bit fields are used only for the QP priv
26a0b34f75SKaike Wan * s_flags, there are no collisions.
27a0b34f75SKaike Wan *
28a0b34f75SKaike Wan * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock
29c6c23117SKaike Wan * HFI1_R_TID_WAIT_INTERLCK - QP is waiting for responder interlock
30a0b34f75SKaike Wan */
3170dcb2e3SKaike Wan #define HFI1_S_TID_BUSY_SET BIT(0)
3270dcb2e3SKaike Wan /* BIT(1) reserved for RVT_S_BUSY. */
333c759e00SKaike Wan #define HFI1_R_TID_RSC_TIMER BIT(2)
3470dcb2e3SKaike Wan /* BIT(3) reserved for RVT_S_RESP_PENDING. */
3507b92370SKaike Wan /* BIT(4) reserved for RVT_S_ACK_PENDING. */
36a0b34f75SKaike Wan #define HFI1_S_TID_WAIT_INTERLCK BIT(5)
37c6c23117SKaike Wan #define HFI1_R_TID_WAIT_INTERLCK BIT(6)
3870dcb2e3SKaike Wan /* BIT(7) - BIT(15) reserved for RVT_S_WAIT_*. */
39c6c23117SKaike Wan /* BIT(16) reserved for RVT_S_SEND_ONE */
40829eaee5SKaike Wan #define HFI1_S_TID_RETRY_TIMER BIT(17)
41c6c23117SKaike Wan /* BIT(18) reserved for RVT_S_ECN. */
4207b92370SKaike Wan #define HFI1_R_TID_SW_PSN BIT(19)
43c6c23117SKaike Wan /* BIT(26) reserved for HFI1_S_WAIT_HALT */
44c6c23117SKaike Wan /* BIT(27) reserved for HFI1_S_WAIT_TID_RESP */
45c6c23117SKaike Wan /* BIT(28) reserved for HFI1_S_WAIT_TID_SPACE */
46a0b34f75SKaike Wan
47f5a4a95fSKaike Wan /*
48f5a4a95fSKaike Wan * Unlike regular IB RDMA VERBS, which do not require an entry
49f5a4a95fSKaike Wan * in the s_ack_queue, TID RDMA WRITE requests do because they
50f5a4a95fSKaike Wan * generate responses.
51f5a4a95fSKaike Wan * Therefore, the s_ack_queue needs to be extended by a certain
52f5a4a95fSKaike Wan * amount. The key point is that the queue needs to be extended
53f5a4a95fSKaike Wan * without letting the "user" know so they user doesn't end up
54f5a4a95fSKaike Wan * using these extra entries.
55f5a4a95fSKaike Wan */
56f5a4a95fSKaike Wan #define HFI1_TID_RDMA_WRITE_CNT 8
57f5a4a95fSKaike Wan
58d22a207dSKaike Wan struct tid_rdma_params {
59d22a207dSKaike Wan struct rcu_head rcu_head;
60d22a207dSKaike Wan u32 qp;
61d22a207dSKaike Wan u32 max_len;
62d22a207dSKaike Wan u16 jkey;
63d22a207dSKaike Wan u8 max_read;
64d22a207dSKaike Wan u8 max_write;
65d22a207dSKaike Wan u8 timeout;
66d22a207dSKaike Wan u8 urg;
67d22a207dSKaike Wan u8 version;
68d22a207dSKaike Wan };
69d22a207dSKaike Wan
70d22a207dSKaike Wan struct tid_rdma_qp_params {
7137356e78SKaike Wan struct work_struct trigger_work;
72d22a207dSKaike Wan struct tid_rdma_params local;
73d22a207dSKaike Wan struct tid_rdma_params __rcu *remote;
74d22a207dSKaike Wan };
75d22a207dSKaike Wan
7637356e78SKaike Wan /* Track state for each hardware flow */
7737356e78SKaike Wan struct tid_flow_state {
7837356e78SKaike Wan u32 generation;
7937356e78SKaike Wan u32 psn;
8037356e78SKaike Wan u8 index;
8137356e78SKaike Wan u8 last_index;
8237356e78SKaike Wan };
8337356e78SKaike Wan
84742a3826SKaike Wan enum tid_rdma_req_state {
85742a3826SKaike Wan TID_REQUEST_INACTIVE = 0,
86742a3826SKaike Wan TID_REQUEST_INIT,
87742a3826SKaike Wan TID_REQUEST_INIT_RESEND,
88742a3826SKaike Wan TID_REQUEST_ACTIVE,
89742a3826SKaike Wan TID_REQUEST_RESEND,
90742a3826SKaike Wan TID_REQUEST_RESEND_ACTIVE,
91742a3826SKaike Wan TID_REQUEST_QUEUED,
92742a3826SKaike Wan TID_REQUEST_SYNC,
93742a3826SKaike Wan TID_REQUEST_RNR_NAK,
94742a3826SKaike Wan TID_REQUEST_COMPLETE,
95742a3826SKaike Wan };
96742a3826SKaike Wan
97838b6fd2SKaike Wan struct tid_rdma_request {
98838b6fd2SKaike Wan struct rvt_qp *qp;
99838b6fd2SKaike Wan struct hfi1_ctxtdata *rcd;
100838b6fd2SKaike Wan union {
101838b6fd2SKaike Wan struct rvt_swqe *swqe;
102838b6fd2SKaike Wan struct rvt_ack_entry *ack;
103838b6fd2SKaike Wan } e;
104838b6fd2SKaike Wan
105838b6fd2SKaike Wan struct tid_rdma_flow *flows; /* array of tid flows */
10607b92370SKaike Wan struct rvt_sge_state ss; /* SGE state for TID RDMA requests */
107838b6fd2SKaike Wan u16 n_flows; /* size of the flow buffer window */
108838b6fd2SKaike Wan u16 setup_head; /* flow index we are setting up */
109838b6fd2SKaike Wan u16 clear_tail; /* flow index we are clearing */
110838b6fd2SKaike Wan u16 flow_idx; /* flow index most recently set up */
11107b92370SKaike Wan u16 acked_tail;
112838b6fd2SKaike Wan
113838b6fd2SKaike Wan u32 seg_len;
114d0d564a1SKaike Wan u32 total_len;
1159e93e967SKaike Wan u32 r_ack_psn; /* next expected ack PSN */
116d0d564a1SKaike Wan u32 r_flow_psn; /* IB PSN of next segment start */
11772a0ea99SKaike Wan u32 r_last_acked; /* IB PSN of last ACK'ed packet */
118742a3826SKaike Wan u32 s_next_psn; /* IB PSN of next segment start for read */
119838b6fd2SKaike Wan
120d0d564a1SKaike Wan u32 total_segs; /* segments required to complete a request */
121742a3826SKaike Wan u32 cur_seg; /* index of current segment */
122d0d564a1SKaike Wan u32 comp_seg; /* index of last completed segment */
123d0d564a1SKaike Wan u32 ack_seg; /* index of last ack'ed segment */
12407b92370SKaike Wan u32 alloc_seg; /* index of next segment to be allocated */
125838b6fd2SKaike Wan u32 isge; /* index of "current" sge */
126742a3826SKaike Wan u32 ack_pending; /* num acks pending for this request */
127742a3826SKaike Wan
128742a3826SKaike Wan enum tid_rdma_req_state state;
129838b6fd2SKaike Wan };
130838b6fd2SKaike Wan
131838b6fd2SKaike Wan /*
132838b6fd2SKaike Wan * When header suppression is used, PSNs associated with a "flow" are
133838b6fd2SKaike Wan * relevant (and not the PSNs maintained by verbs). Track per-flow
134838b6fd2SKaike Wan * PSNs here for a TID RDMA segment.
135838b6fd2SKaike Wan *
136838b6fd2SKaike Wan */
137838b6fd2SKaike Wan struct flow_state {
138838b6fd2SKaike Wan u32 flags;
139838b6fd2SKaike Wan u32 resp_ib_psn; /* The IB PSN of the response for this flow */
140838b6fd2SKaike Wan u32 generation; /* generation of flow */
141838b6fd2SKaike Wan u32 spsn; /* starting PSN in TID space */
142838b6fd2SKaike Wan u32 lpsn; /* last PSN in TID space */
143838b6fd2SKaike Wan u32 r_next_psn; /* next PSN to be received (in TID space) */
144742a3826SKaike Wan
145742a3826SKaike Wan /* For tid rdma read */
146742a3826SKaike Wan u32 ib_spsn; /* starting PSN in Verbs space */
147742a3826SKaike Wan u32 ib_lpsn; /* last PSn in Verbs space */
148838b6fd2SKaike Wan };
149838b6fd2SKaike Wan
150838b6fd2SKaike Wan struct tid_rdma_pageset {
151838b6fd2SKaike Wan dma_addr_t addr : 48; /* Only needed for the first page */
152838b6fd2SKaike Wan u8 idx: 8;
153838b6fd2SKaike Wan u8 count : 7;
154838b6fd2SKaike Wan u8 mapped: 1;
155838b6fd2SKaike Wan };
156838b6fd2SKaike Wan
157838b6fd2SKaike Wan /**
158838b6fd2SKaike Wan * kern_tid_node - used for managing TID's in TID groups
159838b6fd2SKaike Wan *
160838b6fd2SKaike Wan * @grp_idx: rcd relative index to tid_group
161838b6fd2SKaike Wan * @map: grp->map captured prior to programming this TID group in HW
162838b6fd2SKaike Wan * @cnt: Only @cnt of available group entries are actually programmed
163838b6fd2SKaike Wan */
164838b6fd2SKaike Wan struct kern_tid_node {
165838b6fd2SKaike Wan struct tid_group *grp;
166838b6fd2SKaike Wan u8 map;
167838b6fd2SKaike Wan u8 cnt;
168838b6fd2SKaike Wan };
169838b6fd2SKaike Wan
170838b6fd2SKaike Wan /* Overall info for a TID RDMA segment */
171838b6fd2SKaike Wan struct tid_rdma_flow {
172838b6fd2SKaike Wan /*
173838b6fd2SKaike Wan * While a TID RDMA segment is being transferred, it uses a QP number
174838b6fd2SKaike Wan * from the "KDETH section of QP numbers" (which is different from the
175838b6fd2SKaike Wan * QP number that originated the request). Bits 11-15 of these QP
176838b6fd2SKaike Wan * numbers identify the "TID flow" for the segment.
177838b6fd2SKaike Wan */
178838b6fd2SKaike Wan struct flow_state flow_state;
179838b6fd2SKaike Wan struct tid_rdma_request *req;
180d0d564a1SKaike Wan u32 tid_qpn;
181d0d564a1SKaike Wan u32 tid_offset;
182838b6fd2SKaike Wan u32 length;
183742a3826SKaike Wan u32 sent;
184838b6fd2SKaike Wan u8 tnode_cnt;
185838b6fd2SKaike Wan u8 tidcnt;
186742a3826SKaike Wan u8 tid_idx;
187838b6fd2SKaike Wan u8 idx;
188838b6fd2SKaike Wan u8 npagesets;
189838b6fd2SKaike Wan u8 npkts;
190742a3826SKaike Wan u8 pkt;
19172a0ea99SKaike Wan u8 resync_npkts;
192838b6fd2SKaike Wan struct kern_tid_node tnode[TID_RDMA_MAX_PAGES];
193838b6fd2SKaike Wan struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES];
194838b6fd2SKaike Wan u32 tid_entry[TID_RDMA_MAX_PAGES];
195838b6fd2SKaike Wan };
196838b6fd2SKaike Wan
19707b92370SKaike Wan enum tid_rnr_nak_state {
19807b92370SKaike Wan TID_RNR_NAK_INIT = 0,
19907b92370SKaike Wan TID_RNR_NAK_SEND,
20007b92370SKaike Wan TID_RNR_NAK_SENT,
20107b92370SKaike Wan };
20207b92370SKaike Wan
203d22a207dSKaike Wan bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data);
204d22a207dSKaike Wan bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data);
205d22a207dSKaike Wan bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data);
206d22a207dSKaike Wan void tid_rdma_conn_error(struct rvt_qp *qp);
207d22a207dSKaike Wan void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p);
208d22a207dSKaike Wan
209d22a207dSKaike Wan int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit);
210838b6fd2SKaike Wan int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req,
211838b6fd2SKaike Wan struct rvt_sge_state *ss, bool *last);
212838b6fd2SKaike Wan int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req);
213838b6fd2SKaike Wan void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req);
214838b6fd2SKaike Wan void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
215838b6fd2SKaike Wan
216838b6fd2SKaike Wan /**
217838b6fd2SKaike Wan * trdma_clean_swqe - clean flows for swqe if large send queue
218838b6fd2SKaike Wan * @qp: the qp
219838b6fd2SKaike Wan * @wqe: the send wqe
220838b6fd2SKaike Wan */
trdma_clean_swqe(struct rvt_qp * qp,struct rvt_swqe * wqe)221838b6fd2SKaike Wan static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
222838b6fd2SKaike Wan {
223838b6fd2SKaike Wan if (!wqe->priv)
224838b6fd2SKaike Wan return;
225838b6fd2SKaike Wan __trdma_clean_swqe(qp, wqe);
226838b6fd2SKaike Wan }
227d22a207dSKaike Wan
2289905bf06SKaike Wan void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp);
2299905bf06SKaike Wan
2305190f052SMike Marciniszyn int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp,
2315190f052SMike Marciniszyn struct ib_qp_init_attr *init_attr);
23248a615dcSKaike Wan void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp);
2335190f052SMike Marciniszyn
23437356e78SKaike Wan void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp);
23537356e78SKaike Wan
23637356e78SKaike Wan int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp);
23737356e78SKaike Wan void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp);
23837356e78SKaike Wan void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd);
23937356e78SKaike Wan
2402f16a696SKaike Wan struct cntr_entry;
2412f16a696SKaike Wan u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry,
2422f16a696SKaike Wan void *context, int vl, int mode, u64 data);
2432f16a696SKaike Wan
244742a3826SKaike Wan u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe,
245742a3826SKaike Wan struct ib_other_headers *ohdr,
246742a3826SKaike Wan u32 *bth1, u32 *bth2, u32 *len);
247742a3826SKaike Wan u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
248742a3826SKaike Wan struct ib_other_headers *ohdr, u32 *bth1,
249742a3826SKaike Wan u32 *bth2, u32 *len);
250d0d564a1SKaike Wan void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet);
2511db21b50SKaike Wan u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
2521db21b50SKaike Wan struct ib_other_headers *ohdr, u32 *bth0,
2531db21b50SKaike Wan u32 *bth1, u32 *bth2, u32 *len, bool *last);
2549905bf06SKaike Wan void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet);
2559905bf06SKaike Wan bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
2569905bf06SKaike Wan struct hfi1_pportdata *ppd,
2579905bf06SKaike Wan struct hfi1_packet *packet);
258b126078eSKaike Wan void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
259b126078eSKaike Wan u32 *bth2);
26024b11923SKaike Wan void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp);
261a0b34f75SKaike Wan bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe);
262742a3826SKaike Wan
263f1ab4efaSKaike Wan void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
hfi1_setup_tid_rdma_wqe(struct rvt_qp * qp,struct rvt_swqe * wqe)264f1ab4efaSKaike Wan static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp,
265f1ab4efaSKaike Wan struct rvt_swqe *wqe)
266f1ab4efaSKaike Wan {
267f1ab4efaSKaike Wan if (wqe->priv &&
268ad00889eSKaike Wan (wqe->wr.opcode == IB_WR_RDMA_READ ||
269ad00889eSKaike Wan wqe->wr.opcode == IB_WR_RDMA_WRITE) &&
270f1ab4efaSKaike Wan wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE)
271f1ab4efaSKaike Wan setup_tid_rdma_wqe(qp, wqe);
272f1ab4efaSKaike Wan }
273f1ab4efaSKaike Wan
274c098bbb0SKaike Wan u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
275c098bbb0SKaike Wan struct ib_other_headers *ohdr,
276c098bbb0SKaike Wan u32 *bth1, u32 *bth2, u32 *len);
27707b92370SKaike Wan
27807b92370SKaike Wan void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet);
27907b92370SKaike Wan
28038d46d36SKaike Wan u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
28138d46d36SKaike Wan struct ib_other_headers *ohdr, u32 *bth1,
28238d46d36SKaike Wan u32 bth2, u32 *len,
28338d46d36SKaike Wan struct rvt_sge_state **ss);
28438d46d36SKaike Wan
2853c759e00SKaike Wan void hfi1_del_tid_reap_timer(struct rvt_qp *qp);
2863c759e00SKaike Wan
28772a0ea99SKaike Wan void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet);
28872a0ea99SKaike Wan
289539e1908SKaike Wan bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe,
290539e1908SKaike Wan struct ib_other_headers *ohdr,
291539e1908SKaike Wan u32 *bth1, u32 *bth2, u32 *len);
292539e1908SKaike Wan
293d72fe7d5SKaike Wan void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet);
294d72fe7d5SKaike Wan
2950f75e325SKaike Wan u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e,
2960f75e325SKaike Wan struct ib_other_headers *ohdr, u16 iflow,
2970f75e325SKaike Wan u32 *bth1, u32 *bth2);
2980f75e325SKaike Wan
2999e93e967SKaike Wan void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet);
3009e93e967SKaike Wan
301829eaee5SKaike Wan void hfi1_add_tid_retry_timer(struct rvt_qp *qp);
302829eaee5SKaike Wan void hfi1_del_tid_retry_timer(struct rvt_qp *qp);
303829eaee5SKaike Wan
3046e391c6aSKaike Wan u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe,
3056e391c6aSKaike Wan struct ib_other_headers *ohdr, u32 *bth1,
3066e391c6aSKaike Wan u32 *bth2, u16 fidx);
3076e391c6aSKaike Wan
3087cf0ad67SKaike Wan void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet);
3097cf0ad67SKaike Wan
31070dcb2e3SKaike Wan struct hfi1_pkt_state;
31170dcb2e3SKaike Wan int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
31270dcb2e3SKaike Wan
313572f0c33SKaike Wan void _hfi1_do_tid_send(struct work_struct *work);
314572f0c33SKaike Wan
315572f0c33SKaike Wan bool hfi1_schedule_tid_send(struct rvt_qp *qp);
316572f0c33SKaike Wan
317c6c23117SKaike Wan bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e);
318c6c23117SKaike Wan
3195190f052SMike Marciniszyn #endif /* HFI1_TID_RDMA_H */
320