xref: /linux/drivers/infiniband/hw/irdma/user.h (revision 0ea5c948cb64bab5bc7a5516774eb8536f05aa0d)
1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2015 - 2020 Intel Corporation */
3 #ifndef IRDMA_USER_H
4 #define IRDMA_USER_H
5 
6 #define irdma_handle void *
7 #define irdma_adapter_handle irdma_handle
8 #define irdma_qp_handle irdma_handle
9 #define irdma_cq_handle irdma_handle
10 #define irdma_pd_id irdma_handle
11 #define irdma_stag_handle irdma_handle
12 #define irdma_stag_index u32
13 #define irdma_stag u32
14 #define irdma_stag_key u8
15 #define irdma_tagged_offset u64
16 #define irdma_access_privileges u32
17 #define irdma_physical_fragment u64
18 #define irdma_address_list u64 *
19 
20 #define	IRDMA_MAX_MR_SIZE       0x200000000000ULL
21 
22 #define IRDMA_ACCESS_FLAGS_LOCALREAD		0x01
23 #define IRDMA_ACCESS_FLAGS_LOCALWRITE		0x02
24 #define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY	0x04
25 #define IRDMA_ACCESS_FLAGS_REMOTEREAD		0x05
26 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY	0x08
27 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE		0x0a
28 #define IRDMA_ACCESS_FLAGS_BIND_WINDOW		0x10
29 #define IRDMA_ACCESS_FLAGS_ZERO_BASED		0x20
30 #define IRDMA_ACCESS_FLAGS_ALL			0x3f
31 
32 #define IRDMA_OP_TYPE_RDMA_WRITE		0x00
33 #define IRDMA_OP_TYPE_RDMA_READ			0x01
34 #define IRDMA_OP_TYPE_SEND			0x03
35 #define IRDMA_OP_TYPE_SEND_INV			0x04
36 #define IRDMA_OP_TYPE_SEND_SOL			0x05
37 #define IRDMA_OP_TYPE_SEND_SOL_INV		0x06
38 #define IRDMA_OP_TYPE_RDMA_WRITE_SOL		0x0d
39 #define IRDMA_OP_TYPE_BIND_MW			0x08
40 #define IRDMA_OP_TYPE_FAST_REG_NSMR		0x09
41 #define IRDMA_OP_TYPE_INV_STAG			0x0a
42 #define IRDMA_OP_TYPE_RDMA_READ_INV_STAG	0x0b
43 #define IRDMA_OP_TYPE_NOP			0x0c
44 #define IRDMA_OP_TYPE_REC	0x3e
45 #define IRDMA_OP_TYPE_REC_IMM	0x3f
46 
47 #define IRDMA_FLUSH_MAJOR_ERR	1
48 
49 enum irdma_device_caps_const {
50 	IRDMA_WQE_SIZE =			4,
51 	IRDMA_CQP_WQE_SIZE =			8,
52 	IRDMA_CQE_SIZE =			4,
53 	IRDMA_EXTENDED_CQE_SIZE =		8,
54 	IRDMA_AEQE_SIZE =			2,
55 	IRDMA_CEQE_SIZE =			1,
56 	IRDMA_CQP_CTX_SIZE =			8,
57 	IRDMA_SHADOW_AREA_SIZE =		8,
58 	IRDMA_QUERY_FPM_BUF_SIZE =		176,
59 	IRDMA_COMMIT_FPM_BUF_SIZE =		176,
60 	IRDMA_GATHER_STATS_BUF_SIZE =		1024,
61 	IRDMA_MIN_IW_QP_ID =			0,
62 	IRDMA_MAX_IW_QP_ID =			262143,
63 	IRDMA_MIN_CEQID =			0,
64 	IRDMA_MAX_CEQID =			1023,
65 	IRDMA_CEQ_MAX_COUNT =			IRDMA_MAX_CEQID + 1,
66 	IRDMA_MIN_CQID =			0,
67 	IRDMA_MAX_CQID =			524287,
68 	IRDMA_MIN_AEQ_ENTRIES =			1,
69 	IRDMA_MAX_AEQ_ENTRIES =			524287,
70 	IRDMA_MIN_CEQ_ENTRIES =			1,
71 	IRDMA_MAX_CEQ_ENTRIES =			262143,
72 	IRDMA_MIN_CQ_SIZE =			1,
73 	IRDMA_MAX_CQ_SIZE =			1048575,
74 	IRDMA_DB_ID_ZERO =			0,
75 	IRDMA_MAX_WQ_FRAGMENT_COUNT =		13,
76 	IRDMA_MAX_SGE_RD =			13,
77 	IRDMA_MAX_OUTBOUND_MSG_SIZE =		2147483647,
78 	IRDMA_MAX_INBOUND_MSG_SIZE =		2147483647,
79 	IRDMA_MAX_PUSH_PAGE_COUNT =		1024,
80 	IRDMA_MAX_PE_ENA_VF_COUNT =		32,
81 	IRDMA_MAX_VF_FPM_ID =			47,
82 	IRDMA_MAX_SQ_PAYLOAD_SIZE =		2145386496,
83 	IRDMA_MAX_INLINE_DATA_SIZE =		101,
84 	IRDMA_MAX_WQ_ENTRIES =			32768,
85 	IRDMA_Q2_BUF_SIZE =			256,
86 	IRDMA_QP_CTX_SIZE =			256,
87 	IRDMA_MAX_PDS =				262144,
88 	IRDMA_MIN_WQ_SIZE_GEN2 =                8,
89 };
90 
91 enum irdma_addressing_type {
92 	IRDMA_ADDR_TYPE_ZERO_BASED = 0,
93 	IRDMA_ADDR_TYPE_VA_BASED   = 1,
94 };
95 
96 enum irdma_flush_opcode {
97 	FLUSH_INVALID = 0,
98 	FLUSH_GENERAL_ERR,
99 	FLUSH_PROT_ERR,
100 	FLUSH_REM_ACCESS_ERR,
101 	FLUSH_LOC_QP_OP_ERR,
102 	FLUSH_REM_OP_ERR,
103 	FLUSH_LOC_LEN_ERR,
104 	FLUSH_FATAL_ERR,
105 	FLUSH_RETRY_EXC_ERR,
106 	FLUSH_MW_BIND_ERR,
107 	FLUSH_REM_INV_REQ_ERR,
108 };
109 
110 enum irdma_cmpl_status {
111 	IRDMA_COMPL_STATUS_SUCCESS = 0,
112 	IRDMA_COMPL_STATUS_FLUSHED,
113 	IRDMA_COMPL_STATUS_INVALID_WQE,
114 	IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
115 	IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
116 	IRDMA_COMPL_STATUS_INVALID_STAG,
117 	IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
118 	IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
119 	IRDMA_COMPL_STATUS_INVALID_PD_ID,
120 	IRDMA_COMPL_STATUS_WRAP_ERROR,
121 	IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
122 	IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
123 	IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
124 	IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
125 	IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
126 	IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
127 	IRDMA_COMPL_STATUS_INVALID_FBO,
128 	IRDMA_COMPL_STATUS_INVALID_LEN,
129 	IRDMA_COMPL_STATUS_INVALID_ACCESS,
130 	IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
131 	IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
132 	IRDMA_COMPL_STATUS_INVALID_REGION,
133 	IRDMA_COMPL_STATUS_INVALID_WINDOW,
134 	IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
135 	IRDMA_COMPL_STATUS_UNKNOWN,
136 };
137 
138 enum irdma_cmpl_notify {
139 	IRDMA_CQ_COMPL_EVENT     = 0,
140 	IRDMA_CQ_COMPL_SOLICITED = 1,
141 };
142 
143 enum irdma_qp_caps {
144 	IRDMA_WRITE_WITH_IMM = 1,
145 	IRDMA_SEND_WITH_IMM  = 2,
146 	IRDMA_ROCE	     = 4,
147 	IRDMA_PUSH_MODE      = 8,
148 };
149 
150 struct irdma_qp_uk;
151 struct irdma_cq_uk;
152 struct irdma_qp_uk_init_info;
153 struct irdma_cq_uk_init_info;
154 
155 struct irdma_ring {
156 	u32 head;
157 	u32 tail;
158 	u32 size;
159 };
160 
161 struct irdma_cqe {
162 	__le64 buf[IRDMA_CQE_SIZE];
163 };
164 
165 struct irdma_extended_cqe {
166 	__le64 buf[IRDMA_EXTENDED_CQE_SIZE];
167 };
168 
169 struct irdma_post_send {
170 	struct ib_sge *sg_list;
171 	u32 num_sges;
172 	u32 qkey;
173 	u32 dest_qp;
174 	u32 ah_id;
175 };
176 
177 struct irdma_post_rq_info {
178 	u64 wr_id;
179 	struct ib_sge *sg_list;
180 	u32 num_sges;
181 };
182 
183 struct irdma_rdma_write {
184 	struct ib_sge *lo_sg_list;
185 	u32 num_lo_sges;
186 	struct ib_sge rem_addr;
187 };
188 
189 struct irdma_rdma_read {
190 	struct ib_sge *lo_sg_list;
191 	u32 num_lo_sges;
192 	struct ib_sge rem_addr;
193 };
194 
195 struct irdma_bind_window {
196 	irdma_stag mr_stag;
197 	u64 bind_len;
198 	void *va;
199 	enum irdma_addressing_type addressing_type;
200 	bool ena_reads:1;
201 	bool ena_writes:1;
202 	irdma_stag mw_stag;
203 	bool mem_window_type_1:1;
204 };
205 
206 struct irdma_inv_local_stag {
207 	irdma_stag target_stag;
208 };
209 
210 struct irdma_post_sq_info {
211 	u64 wr_id;
212 	u8 op_type;
213 	u8 l4len;
214 	bool signaled:1;
215 	bool read_fence:1;
216 	bool local_fence:1;
217 	bool inline_data:1;
218 	bool imm_data_valid:1;
219 	bool report_rtt:1;
220 	bool udp_hdr:1;
221 	bool defer_flag:1;
222 	u32 imm_data;
223 	u32 stag_to_inv;
224 	union {
225 		struct irdma_post_send send;
226 		struct irdma_rdma_write rdma_write;
227 		struct irdma_rdma_read rdma_read;
228 		struct irdma_bind_window bind_window;
229 		struct irdma_inv_local_stag inv_local_stag;
230 	} op;
231 };
232 
233 struct irdma_cq_poll_info {
234 	u64 wr_id;
235 	irdma_qp_handle qp_handle;
236 	u32 bytes_xfered;
237 	u32 tcp_seq_num_rtt;
238 	u32 qp_id;
239 	u32 ud_src_qpn;
240 	u32 imm_data;
241 	irdma_stag inv_stag; /* or L_R_Key */
242 	enum irdma_cmpl_status comp_status;
243 	u16 major_err;
244 	u16 minor_err;
245 	u16 ud_vlan;
246 	u8 ud_smac[6];
247 	u8 op_type;
248 	u8 q_type;
249 	bool stag_invalid_set:1; /* or L_R_Key set */
250 	bool error:1;
251 	bool solicited_event:1;
252 	bool ipv4:1;
253 	bool ud_vlan_valid:1;
254 	bool ud_smac_valid:1;
255 	bool imm_valid:1;
256 };
257 
258 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
259 			       struct irdma_post_sq_info *info, bool post_sq);
260 int irdma_uk_inline_send(struct irdma_qp_uk *qp,
261 			 struct irdma_post_sq_info *info, bool post_sq);
262 int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,
263 		      bool post_sq);
264 int irdma_uk_post_receive(struct irdma_qp_uk *qp,
265 			  struct irdma_post_rq_info *info);
266 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
267 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
268 		       bool inv_stag, bool post_sq);
269 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
270 			bool post_sq);
271 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
272 		  bool post_sq);
273 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
274 				   struct irdma_post_sq_info *info,
275 				   bool post_sq);
276 
277 struct irdma_wqe_uk_ops {
278 	void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list,
279 				    u32 num_sges, u8 polarity);
280 	u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
281 	void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
282 				u8 valid);
283 	void (*iw_set_mw_bind_wqe)(__le64 *wqe,
284 				   struct irdma_bind_window *op_info);
285 };
286 
287 int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
288 			  struct irdma_cq_poll_info *info);
289 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
290 				      enum irdma_cmpl_notify cq_notify);
291 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
292 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
293 void irdma_uk_cq_init(struct irdma_cq_uk *cq,
294 		      struct irdma_cq_uk_init_info *info);
295 int irdma_uk_qp_init(struct irdma_qp_uk *qp,
296 		     struct irdma_qp_uk_init_info *info);
297 void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
298 			    u8 *rq_shift);
299 int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
300 				 u32 *sq_depth, u8 *sq_shift);
301 int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
302 				 u32 *rq_depth, u8 *rq_shift);
303 struct irdma_sq_uk_wr_trk_info {
304 	u64 wrid;
305 	u32 wr_len;
306 	u16 quanta;
307 	u8 reserved[2];
308 };
309 
310 struct irdma_qp_quanta {
311 	__le64 elem[IRDMA_WQE_SIZE];
312 };
313 
314 struct irdma_qp_uk {
315 	struct irdma_qp_quanta *sq_base;
316 	struct irdma_qp_quanta *rq_base;
317 	struct irdma_uk_attrs *uk_attrs;
318 	u32 __iomem *wqe_alloc_db;
319 	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
320 	u64 *rq_wrid_array;
321 	__le64 *shadow_area;
322 	struct irdma_ring sq_ring;
323 	struct irdma_ring rq_ring;
324 	struct irdma_ring initial_ring;
325 	u32 qp_id;
326 	u32 qp_caps;
327 	u32 sq_size;
328 	u32 rq_size;
329 	u32 max_sq_frag_cnt;
330 	u32 max_rq_frag_cnt;
331 	u32 max_inline_data;
332 	struct irdma_wqe_uk_ops wqe_ops;
333 	u16 conn_wqes;
334 	u8 qp_type;
335 	u8 swqe_polarity;
336 	u8 swqe_polarity_deferred;
337 	u8 rwqe_polarity;
338 	u8 rq_wqe_size;
339 	u8 rq_wqe_size_multiplier;
340 	bool deferred_flag:1;
341 	bool first_sq_wq:1;
342 	bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */
343 	bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */
344 	bool destroy_pending:1; /* Indicates the QP is being destroyed */
345 	void *back_qp;
346 	u8 dbg_rq_flushed;
347 	u8 sq_flush_seen;
348 	u8 rq_flush_seen;
349 };
350 
351 struct irdma_cq_uk {
352 	struct irdma_cqe *cq_base;
353 	u32 __iomem *cqe_alloc_db;
354 	u32 __iomem *cq_ack_db;
355 	__le64 *shadow_area;
356 	u32 cq_id;
357 	u32 cq_size;
358 	struct irdma_ring cq_ring;
359 	u8 polarity;
360 	bool avoid_mem_cflct:1;
361 };
362 
363 struct irdma_qp_uk_init_info {
364 	struct irdma_qp_quanta *sq;
365 	struct irdma_qp_quanta *rq;
366 	struct irdma_uk_attrs *uk_attrs;
367 	u32 __iomem *wqe_alloc_db;
368 	__le64 *shadow_area;
369 	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
370 	u64 *rq_wrid_array;
371 	u32 qp_id;
372 	u32 qp_caps;
373 	u32 sq_size;
374 	u32 rq_size;
375 	u32 max_sq_frag_cnt;
376 	u32 max_rq_frag_cnt;
377 	u32 max_inline_data;
378 	u32 sq_depth;
379 	u32 rq_depth;
380 	u8 first_sq_wq;
381 	u8 type;
382 	u8 sq_shift;
383 	u8 rq_shift;
384 	int abi_ver;
385 	bool legacy_mode;
386 };
387 
388 struct irdma_cq_uk_init_info {
389 	u32 __iomem *cqe_alloc_db;
390 	u32 __iomem *cq_ack_db;
391 	struct irdma_cqe *cq_base;
392 	__le64 *shadow_area;
393 	u32 cq_size;
394 	u32 cq_id;
395 	bool avoid_mem_cflct;
396 };
397 
398 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
399 				   u16 quanta, u32 total_size,
400 				   struct irdma_post_sq_info *info);
401 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
402 void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
403 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
404 int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
405 int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
406 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
407 			 u32 inline_data, u8 *shift);
408 int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
409 		      u32 *wqdepth);
410 int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
411 		      u32 *wqdepth);
412 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
413 #endif /* IRDMA_USER_H */
414