1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #ifndef IRDMA_VERBS_H
4 #define IRDMA_VERBS_H
5
6 #define IRDMA_MAX_SAVED_PHY_PGADDR 4
7 #define IRDMA_FLUSH_DELAY_MS 20
8
9 #define IRDMA_PKEY_TBL_SZ 1
10 #define IRDMA_DEFAULT_PKEY 0xFFFF
11 #define IRDMA_SHADOW_PGCNT 1
12
13 struct irdma_ucontext {
14 struct ib_ucontext ibucontext;
15 struct irdma_device *iwdev;
16 struct rdma_user_mmap_entry *db_mmap_entry;
17 struct list_head cq_reg_mem_list;
18 spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
19 struct list_head qp_reg_mem_list;
20 spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */
21 struct list_head srq_reg_mem_list;
22 spinlock_t srq_reg_mem_list_lock; /* protect SRQ memory list */
23 int abi_ver;
24 u8 legacy_mode : 1;
25 u8 use_raw_attrs : 1;
26 };
27
28 struct irdma_pd {
29 struct ib_pd ibpd;
30 struct irdma_sc_pd sc_pd;
31 };
32
33 union irdma_sockaddr {
34 struct sockaddr_in saddr_in;
35 struct sockaddr_in6 saddr_in6;
36 };
37
38 struct irdma_av {
39 u8 macaddr[16];
40 struct rdma_ah_attr attrs;
41 union irdma_sockaddr sgid_addr;
42 union irdma_sockaddr dgid_addr;
43 u8 net_type;
44 };
45
46 struct irdma_ah {
47 struct ib_ah ibah;
48 struct irdma_sc_ah sc_ah;
49 struct irdma_pd *pd;
50 struct irdma_av av;
51 u8 sgid_index;
52 union ib_gid dgid;
53 struct hlist_node list;
54 refcount_t refcnt;
55 struct irdma_ah *parent_ah; /* AH from cached list */
56 };
57
58 struct irdma_hmc_pble {
59 union {
60 u32 idx;
61 dma_addr_t addr;
62 };
63 };
64
65 struct irdma_cq_mr {
66 struct irdma_hmc_pble cq_pbl;
67 dma_addr_t shadow;
68 bool split;
69 };
70
71 struct irdma_srq_mr {
72 struct irdma_hmc_pble srq_pbl;
73 dma_addr_t shadow;
74 };
75
76 struct irdma_qp_mr {
77 struct irdma_hmc_pble sq_pbl;
78 struct irdma_hmc_pble rq_pbl;
79 dma_addr_t shadow;
80 dma_addr_t rq_pa;
81 struct page *sq_page;
82 };
83
84 struct irdma_cq_buf {
85 struct irdma_dma_mem kmem_buf;
86 struct irdma_cq_uk cq_uk;
87 struct irdma_hw *hw;
88 struct list_head list;
89 struct work_struct work;
90 };
91
92 struct irdma_pbl {
93 struct list_head list;
94 union {
95 struct irdma_qp_mr qp_mr;
96 struct irdma_cq_mr cq_mr;
97 struct irdma_srq_mr srq_mr;
98 };
99
100 bool pbl_allocated:1;
101 bool on_list:1;
102 u64 user_base;
103 struct irdma_pble_alloc pble_alloc;
104 struct irdma_mr *iwmr;
105 };
106
107 struct irdma_mr {
108 union {
109 struct ib_mr ibmr;
110 struct ib_mw ibmw;
111 };
112 struct ib_umem *region;
113 int access;
114 u8 is_hwreg;
115 u16 type;
116 u32 page_cnt;
117 u64 page_size;
118 u32 npages;
119 u32 stag;
120 u64 len;
121 u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];
122 struct irdma_pbl iwpbl;
123 };
124
125 struct irdma_srq {
126 struct ib_srq ibsrq;
127 struct irdma_sc_srq sc_srq __aligned(64);
128 struct irdma_dma_mem kmem;
129 u64 *srq_wrid_mem;
130 refcount_t refcnt;
131 spinlock_t lock; /* for poll srq */
132 struct irdma_pbl *iwpbl;
133 struct irdma_sge *sg_list;
134 u16 srq_head;
135 u32 srq_num;
136 u32 max_wr;
137 bool user_mode:1;
138 };
139
140 struct irdma_cq {
141 struct ib_cq ibcq;
142 struct irdma_sc_cq sc_cq;
143 u32 cq_num;
144 bool user_mode;
145 atomic_t armed;
146 enum irdma_cmpl_notify last_notify;
147 struct irdma_dma_mem kmem;
148 struct irdma_dma_mem kmem_shadow;
149 struct completion free_cq;
150 refcount_t refcnt;
151 spinlock_t lock; /* for poll cq */
152 struct list_head resize_list;
153 struct irdma_cq_poll_info cur_cqe;
154 struct list_head cmpl_generated;
155 };
156
157 struct irdma_cmpl_gen {
158 struct list_head list;
159 struct irdma_cq_poll_info cpi;
160 };
161
162 struct disconn_work {
163 struct work_struct work;
164 struct irdma_qp *iwqp;
165 };
166
167 struct iw_cm_id;
168
169 struct irdma_qp_kmode {
170 struct irdma_dma_mem dma_mem;
171 struct irdma_sq_uk_wr_trk_info *sq_wrid_mem;
172 u64 *rq_wrid_mem;
173 };
174
175 struct irdma_qp {
176 struct ib_qp ibqp;
177 struct irdma_sc_qp sc_qp;
178 struct irdma_device *iwdev;
179 struct irdma_cq *iwscq;
180 struct irdma_cq *iwrcq;
181 struct irdma_pd *iwpd;
182 struct rdma_user_mmap_entry *push_wqe_mmap_entry;
183 struct rdma_user_mmap_entry *push_db_mmap_entry;
184 struct irdma_qp_host_ctx_info ctx_info;
185 union {
186 struct irdma_iwarp_offload_info iwarp_info;
187 struct irdma_roce_offload_info roce_info;
188 };
189
190 union {
191 struct irdma_tcp_offload_info tcp_info;
192 struct irdma_udp_offload_info udp_info;
193 };
194
195 struct irdma_ah roce_ah;
196 struct list_head teardown_entry;
197 refcount_t refcnt;
198 struct iw_cm_id *cm_id;
199 struct irdma_cm_node *cm_node;
200 struct delayed_work dwork_flush;
201 struct ib_mr *lsmm_mr;
202 atomic_t hw_mod_qp_pend;
203 enum ib_qp_state ibqp_state;
204 u32 qp_mem_size;
205 u32 last_aeq;
206 int max_send_wr;
207 int max_recv_wr;
208 atomic_t close_timer_started;
209 spinlock_t lock; /* serialize posting WRs to SQ/RQ */
210 struct irdma_qp_context *iwqp_context;
211 void *pbl_vbase;
212 dma_addr_t pbl_pbase;
213 struct page *page;
214 u8 active_conn : 1;
215 u8 user_mode : 1;
216 u8 hte_added : 1;
217 u8 flush_issued : 1;
218 u8 sig_all : 1;
219 u8 pau_mode : 1;
220 u8 suspend_pending : 1;
221 u8 rsvd : 1;
222 u8 iwarp_state;
223 u16 term_sq_flush_code;
224 u16 term_rq_flush_code;
225 u8 hw_iwarp_state;
226 u8 hw_tcp_state;
227 struct irdma_qp_kmode kqp;
228 struct irdma_dma_mem host_ctx;
229 struct timer_list terminate_timer;
230 struct irdma_pbl *iwpbl;
231 struct irdma_dma_mem q2_ctx_mem;
232 struct irdma_dma_mem ietf_mem;
233 struct completion free_qp;
234 wait_queue_head_t waitq;
235 wait_queue_head_t mod_qp_waitq;
236 u8 rts_ae_rcvd;
237 };
238
239 enum irdma_mmap_flag {
240 IRDMA_MMAP_IO_NC,
241 IRDMA_MMAP_IO_WC,
242 };
243
244 struct irdma_user_mmap_entry {
245 struct rdma_user_mmap_entry rdma_entry;
246 u64 bar_offset;
247 u8 mmap_flag;
248 };
249
irdma_fw_major_ver(struct irdma_sc_dev * dev)250 static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev)
251 {
252 return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
253 }
254
irdma_fw_minor_ver(struct irdma_sc_dev * dev)255 static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev)
256 {
257 return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
258 }
259
set_ib_wc_op_sq(struct irdma_cq_poll_info * cq_poll_info,struct ib_wc * entry)260 static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
261 struct ib_wc *entry)
262 {
263 switch (cq_poll_info->op_type) {
264 case IRDMA_OP_TYPE_RDMA_WRITE:
265 case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
266 entry->opcode = IB_WC_RDMA_WRITE;
267 break;
268 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
269 case IRDMA_OP_TYPE_RDMA_READ:
270 entry->opcode = IB_WC_RDMA_READ;
271 break;
272 case IRDMA_OP_TYPE_SEND_SOL:
273 case IRDMA_OP_TYPE_SEND_SOL_INV:
274 case IRDMA_OP_TYPE_SEND_INV:
275 case IRDMA_OP_TYPE_SEND:
276 entry->opcode = IB_WC_SEND;
277 break;
278 case IRDMA_OP_TYPE_FAST_REG_NSMR:
279 entry->opcode = IB_WC_REG_MR;
280 break;
281 case IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP:
282 entry->opcode = IB_WC_COMP_SWAP;
283 break;
284 case IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD:
285 entry->opcode = IB_WC_FETCH_ADD;
286 break;
287 case IRDMA_OP_TYPE_INV_STAG:
288 entry->opcode = IB_WC_LOCAL_INV;
289 break;
290 default:
291 entry->status = IB_WC_GENERAL_ERR;
292 }
293 }
294
set_ib_wc_op_rq_gen_3(struct irdma_cq_poll_info * info,struct ib_wc * entry)295 static inline void set_ib_wc_op_rq_gen_3(struct irdma_cq_poll_info *info,
296 struct ib_wc *entry)
297 {
298 switch (info->op_type) {
299 case IRDMA_OP_TYPE_RDMA_WRITE:
300 case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
301 entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
302 break;
303 default:
304 entry->opcode = IB_WC_RECV;
305 }
306 }
307
set_ib_wc_op_rq(struct irdma_cq_poll_info * cq_poll_info,struct ib_wc * entry,bool send_imm_support)308 static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
309 struct ib_wc *entry, bool send_imm_support)
310 {
311 /**
312 * iWARP does not support sendImm, so the presence of Imm data
313 * must be WriteImm.
314 */
315 if (!send_imm_support) {
316 entry->opcode = cq_poll_info->imm_valid ?
317 IB_WC_RECV_RDMA_WITH_IMM :
318 IB_WC_RECV;
319 return;
320 }
321
322 switch (cq_poll_info->op_type) {
323 case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
324 case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
325 entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
326 break;
327 default:
328 entry->opcode = IB_WC_RECV;
329 }
330 }
331
332 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4);
333 int irdma_ib_register_device(struct irdma_device *iwdev);
334 void irdma_ib_unregister_device(struct irdma_device *iwdev);
335 void irdma_ib_dealloc_device(struct ib_device *ibdev);
336 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);
337 void irdma_generate_flush_completions(struct irdma_qp *iwqp);
338 void irdma_remove_cmpls_list(struct irdma_cq *iwcq);
339 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info);
340 #endif /* IRDMA_VERBS_H */
341