1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "osdep.h"
4 #include "defs.h"
5 #include "user.h"
6 #include "irdma.h"
7
8 /**
9 * irdma_set_fragment - set fragment in wqe
10 * @wqe: wqe for setting fragment
11 * @offset: offset value
12 * @sge: sge length and stag
13 * @valid: The wqe valid
14 */
irdma_set_fragment(__le64 * wqe,u32 offset,struct ib_sge * sge,u8 valid)15 static void irdma_set_fragment(__le64 *wqe, u32 offset, struct ib_sge *sge,
16 u8 valid)
17 {
18 if (sge) {
19 set_64bit_val(wqe, offset,
20 FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
21 set_64bit_val(wqe, offset + 8,
22 FIELD_PREP(IRDMAQPSQ_VALID, valid) |
23 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
24 FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
25 } else {
26 set_64bit_val(wqe, offset, 0);
27 set_64bit_val(wqe, offset + 8,
28 FIELD_PREP(IRDMAQPSQ_VALID, valid));
29 }
30 }
31
32 /**
33 * irdma_set_fragment_gen_1 - set fragment in wqe
34 * @wqe: wqe for setting fragment
35 * @offset: offset value
36 * @sge: sge length and stag
37 * @valid: wqe valid flag
38 */
irdma_set_fragment_gen_1(__le64 * wqe,u32 offset,struct ib_sge * sge,u8 valid)39 static void irdma_set_fragment_gen_1(__le64 *wqe, u32 offset,
40 struct ib_sge *sge, u8 valid)
41 {
42 if (sge) {
43 set_64bit_val(wqe, offset,
44 FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
45 set_64bit_val(wqe, offset + 8,
46 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
47 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
48 } else {
49 set_64bit_val(wqe, offset, 0);
50 set_64bit_val(wqe, offset + 8, 0);
51 }
52 }
53
54 /**
55 * irdma_nop_1 - insert a NOP wqe
56 * @qp: hw qp ptr
57 */
irdma_nop_1(struct irdma_qp_uk * qp)58 static int irdma_nop_1(struct irdma_qp_uk *qp)
59 {
60 u64 hdr;
61 __le64 *wqe;
62 u32 wqe_idx;
63 bool signaled = false;
64
65 if (!qp->sq_ring.head)
66 return -EINVAL;
67
68 wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
69 wqe = qp->sq_base[wqe_idx].elem;
70
71 qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;
72
73 set_64bit_val(wqe, 0, 0);
74 set_64bit_val(wqe, 8, 0);
75 set_64bit_val(wqe, 16, 0);
76
77 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
78 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
79 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
80
81 /* make sure WQE is written before valid bit is set */
82 dma_wmb();
83
84 set_64bit_val(wqe, 24, hdr);
85
86 return 0;
87 }
88
89 /**
90 * irdma_clr_wqes - clear next 128 sq entries
91 * @qp: hw qp ptr
92 * @qp_wqe_idx: wqe_idx
93 */
irdma_clr_wqes(struct irdma_qp_uk * qp,u32 qp_wqe_idx)94 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx)
95 {
96 struct irdma_qp_quanta *sq;
97 u32 wqe_idx;
98
99 if (!(qp_wqe_idx & 0x7F)) {
100 wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size;
101 sq = qp->sq_base + wqe_idx;
102 if (wqe_idx)
103 memset(sq, qp->swqe_polarity ? 0 : 0xFF,
104 128 * sizeof(*sq));
105 else
106 memset(sq, qp->swqe_polarity ? 0xFF : 0,
107 128 * sizeof(*sq));
108 }
109 }
110
111 /**
112 * irdma_uk_qp_post_wr - ring doorbell
113 * @qp: hw qp ptr
114 */
irdma_uk_qp_post_wr(struct irdma_qp_uk * qp)115 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp)
116 {
117 writel(qp->qp_id, qp->wqe_alloc_db);
118 }
119
120 /**
121 * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go
122 * @qp: hw qp ptr
123 * @wqe_idx: return wqe index
124 * @quanta: size of WR in quanta
125 * @total_size: size of WR in bytes
126 * @info: info on WR
127 */
irdma_qp_get_next_send_wqe(struct irdma_qp_uk * qp,u32 * wqe_idx,u16 quanta,u32 total_size,struct irdma_post_sq_info * info)128 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
129 u16 quanta, u32 total_size,
130 struct irdma_post_sq_info *info)
131 {
132 __le64 *wqe;
133 __le64 *wqe_0 = NULL;
134 u16 avail_quanta;
135 u16 i;
136
137 avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
138 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
139 qp->uk_attrs->max_hw_sq_chunk);
140 if (quanta <= avail_quanta) {
141 /* WR fits in current chunk */
142 if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
143 return NULL;
144 } else {
145 /* Need to pad with NOP */
146 if (quanta + avail_quanta >
147 IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
148 return NULL;
149
150 for (i = 0; i < avail_quanta; i++) {
151 irdma_nop_1(qp);
152 IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring);
153 }
154 }
155
156 *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
157 if (!*wqe_idx)
158 qp->swqe_polarity = !qp->swqe_polarity;
159
160 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
161
162 wqe = qp->sq_base[*wqe_idx].elem;
163 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
164 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
165 wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
166 wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity ? 0 : 1));
167 }
168 qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
169 qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
170 qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
171 qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled;
172
173 return wqe;
174 }
175
irdma_srq_get_next_recv_wqe(struct irdma_srq_uk * srq,u32 * wqe_idx)176 __le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx)
177 {
178 int ret_code;
179 __le64 *wqe;
180
181 if (IRDMA_RING_FULL_ERR(srq->srq_ring))
182 return NULL;
183
184 IRDMA_ATOMIC_RING_MOVE_HEAD(srq->srq_ring, *wqe_idx, ret_code);
185 if (ret_code)
186 return NULL;
187
188 if (!*wqe_idx)
189 srq->srwqe_polarity = !srq->srwqe_polarity;
190 /* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */
191 wqe = srq->srq_base[*wqe_idx * (srq->wqe_size_multiplier)].elem;
192
193 return wqe;
194 }
195
196 /**
197 * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe
198 * @qp: hw qp ptr
199 * @wqe_idx: return wqe index
200 */
irdma_qp_get_next_recv_wqe(struct irdma_qp_uk * qp,u32 * wqe_idx)201 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)
202 {
203 __le64 *wqe;
204 int ret_code;
205
206 if (IRDMA_RING_FULL_ERR(qp->rq_ring))
207 return NULL;
208
209 IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
210 if (ret_code)
211 return NULL;
212
213 if (!*wqe_idx)
214 qp->rwqe_polarity = !qp->rwqe_polarity;
215 /* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */
216 wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem;
217
218 return wqe;
219 }
220
221 /**
222 * irdma_uk_rdma_write - rdma write operation
223 * @qp: hw qp ptr
224 * @info: post sq information
225 * @post_sq: flag to post sq
226 */
irdma_uk_rdma_write(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)227 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
228 bool post_sq)
229 {
230 u64 hdr;
231 __le64 *wqe;
232 struct irdma_rdma_write *op_info;
233 u32 i, wqe_idx;
234 u32 total_size = 0, byte_off;
235 int ret_code;
236 u32 frag_cnt, addl_frag_cnt;
237 bool read_fence = false;
238 u16 quanta;
239
240 op_info = &info->op.rdma_write;
241 if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
242 return -EINVAL;
243
244 for (i = 0; i < op_info->num_lo_sges; i++)
245 total_size += op_info->lo_sg_list[i].length;
246
247 read_fence |= info->read_fence;
248
249 if (info->imm_data_valid)
250 frag_cnt = op_info->num_lo_sges + 1;
251 else
252 frag_cnt = op_info->num_lo_sges;
253 addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
254 ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta);
255 if (ret_code)
256 return ret_code;
257
258 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
259 info);
260 if (!wqe)
261 return -ENOMEM;
262
263 irdma_clr_wqes(qp, wqe_idx);
264
265 set_64bit_val(wqe, 16,
266 FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
267
268 if (info->imm_data_valid) {
269 set_64bit_val(wqe, 0,
270 FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
271 i = 0;
272 } else {
273 qp->wqe_ops.iw_set_fragment(wqe, 0,
274 op_info->lo_sg_list,
275 qp->swqe_polarity);
276 i = 1;
277 }
278
279 for (byte_off = 32; i < op_info->num_lo_sges; i++) {
280 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
281 &op_info->lo_sg_list[i],
282 qp->swqe_polarity);
283 byte_off += 16;
284 }
285
286 /* if not an odd number set valid bit in next fragment */
287 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
288 frag_cnt) {
289 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
290 qp->swqe_polarity);
291 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
292 ++addl_frag_cnt;
293 }
294
295 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
296 FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
297 FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
298 FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
299 FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
300 FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
301 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
302 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
303 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
304
305 dma_wmb(); /* make sure WQE is populated before valid bit is set */
306
307 set_64bit_val(wqe, 24, hdr);
308
309 if (post_sq)
310 irdma_uk_qp_post_wr(qp);
311
312 return 0;
313 }
314
315 /**
316 * irdma_uk_atomic_fetch_add - atomic fetch and add operation
317 * @qp: hw qp ptr
318 * @info: post sq information
319 * @post_sq: flag to post sq
320 */
irdma_uk_atomic_fetch_add(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)321 int irdma_uk_atomic_fetch_add(struct irdma_qp_uk *qp,
322 struct irdma_post_sq_info *info, bool post_sq)
323 {
324 struct irdma_atomic_fetch_add *op_info;
325 u32 total_size = 0;
326 u16 quanta = 2;
327 u32 wqe_idx;
328 __le64 *wqe;
329 u64 hdr;
330
331 op_info = &info->op.atomic_fetch_add;
332 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
333 info);
334 if (!wqe)
335 return -ENOMEM;
336
337 set_64bit_val(wqe, 0, op_info->tagged_offset);
338 set_64bit_val(wqe, 8,
339 FIELD_PREP(IRDMAQPSQ_STAG, op_info->stag));
340 set_64bit_val(wqe, 16, op_info->remote_tagged_offset);
341
342 hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, 1) |
343 FIELD_PREP(IRDMAQPSQ_REMOTE_STAG, op_info->remote_stag) |
344 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_ATOMIC_FETCH_ADD) |
345 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
346 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
347 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
348 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
349
350 set_64bit_val(wqe, 32, op_info->fetch_add_data_bytes);
351 set_64bit_val(wqe, 40, 0);
352 set_64bit_val(wqe, 48, 0);
353 set_64bit_val(wqe, 56,
354 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity));
355
356 dma_wmb(); /* make sure WQE is populated before valid bit is set */
357
358 set_64bit_val(wqe, 24, hdr);
359
360 if (post_sq)
361 irdma_uk_qp_post_wr(qp);
362
363 return 0;
364 }
365
366 /**
367 * irdma_uk_atomic_compare_swap - atomic compare and swap operation
368 * @qp: hw qp ptr
369 * @info: post sq information
370 * @post_sq: flag to post sq
371 */
irdma_uk_atomic_compare_swap(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)372 int irdma_uk_atomic_compare_swap(struct irdma_qp_uk *qp,
373 struct irdma_post_sq_info *info, bool post_sq)
374 {
375 struct irdma_atomic_compare_swap *op_info;
376 u32 total_size = 0;
377 u16 quanta = 2;
378 u32 wqe_idx;
379 __le64 *wqe;
380 u64 hdr;
381
382 op_info = &info->op.atomic_compare_swap;
383 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
384 info);
385 if (!wqe)
386 return -ENOMEM;
387
388 set_64bit_val(wqe, 0, op_info->tagged_offset);
389 set_64bit_val(wqe, 8,
390 FIELD_PREP(IRDMAQPSQ_STAG, op_info->stag));
391 set_64bit_val(wqe, 16, op_info->remote_tagged_offset);
392
393 hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, 1) |
394 FIELD_PREP(IRDMAQPSQ_REMOTE_STAG, op_info->remote_stag) |
395 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_ATOMIC_COMPARE_SWAP_ADD) |
396 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
397 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
398 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
399 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
400
401 set_64bit_val(wqe, 32, op_info->swap_data_bytes);
402 set_64bit_val(wqe, 40, op_info->compare_data_bytes);
403 set_64bit_val(wqe, 48, 0);
404 set_64bit_val(wqe, 56,
405 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity));
406
407 dma_wmb(); /* make sure WQE is populated before valid bit is set */
408
409 set_64bit_val(wqe, 24, hdr);
410
411 if (post_sq)
412 irdma_uk_qp_post_wr(qp);
413
414 return 0;
415 }
416
417 /**
418 * irdma_uk_srq_post_receive - post a receive wqe to a shared rq
419 * @srq: shared rq ptr
420 * @info: post rq information
421 */
irdma_uk_srq_post_receive(struct irdma_srq_uk * srq,struct irdma_post_rq_info * info)422 int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,
423 struct irdma_post_rq_info *info)
424 {
425 u32 wqe_idx, i, byte_off;
426 u32 addl_frag_cnt;
427 __le64 *wqe;
428 u64 hdr;
429
430 if (srq->max_srq_frag_cnt < info->num_sges)
431 return -EINVAL;
432
433 wqe = irdma_srq_get_next_recv_wqe(srq, &wqe_idx);
434 if (!wqe)
435 return -ENOMEM;
436
437 addl_frag_cnt = info->num_sges > 1 ? info->num_sges - 1 : 0;
438 srq->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list,
439 srq->srwqe_polarity);
440
441 for (i = 1, byte_off = 32; i < info->num_sges; i++) {
442 srq->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
443 srq->srwqe_polarity);
444 byte_off += 16;
445 }
446
447 /* if not an odd number set valid bit in next fragment */
448 if (srq->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
449 info->num_sges) {
450 srq->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
451 srq->srwqe_polarity);
452 if (srq->uk_attrs->hw_rev == IRDMA_GEN_2)
453 ++addl_frag_cnt;
454 }
455
456 set_64bit_val(wqe, 16, (u64)info->wr_id);
457 hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
458 FIELD_PREP(IRDMAQPSQ_VALID, srq->srwqe_polarity);
459
460 dma_wmb(); /* make sure WQE is populated before valid bit is set */
461
462 set_64bit_val(wqe, 24, hdr);
463
464 set_64bit_val(srq->shadow_area, 0, (wqe_idx + 1) % srq->srq_ring.size);
465
466 return 0;
467 }
468
469 /**
470 * irdma_uk_rdma_read - rdma read command
471 * @qp: hw qp ptr
472 * @info: post sq information
473 * @inv_stag: flag for inv_stag
474 * @post_sq: flag to post sq
475 */
irdma_uk_rdma_read(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool inv_stag,bool post_sq)476 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
477 bool inv_stag, bool post_sq)
478 {
479 struct irdma_rdma_read *op_info;
480 int ret_code;
481 u32 i, byte_off, total_size = 0;
482 bool local_fence = false;
483 u32 addl_frag_cnt;
484 __le64 *wqe;
485 u32 wqe_idx;
486 u16 quanta;
487 u64 hdr;
488
489 op_info = &info->op.rdma_read;
490 if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
491 return -EINVAL;
492
493 for (i = 0; i < op_info->num_lo_sges; i++)
494 total_size += op_info->lo_sg_list[i].length;
495
496 ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
497 if (ret_code)
498 return ret_code;
499
500 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
501 info);
502 if (!wqe)
503 return -ENOMEM;
504
505 irdma_clr_wqes(qp, wqe_idx);
506
507 addl_frag_cnt = op_info->num_lo_sges > 1 ?
508 (op_info->num_lo_sges - 1) : 0;
509 local_fence |= info->local_fence;
510
511 qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list,
512 qp->swqe_polarity);
513 for (i = 1, byte_off = 32; i < op_info->num_lo_sges; ++i) {
514 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
515 &op_info->lo_sg_list[i],
516 qp->swqe_polarity);
517 byte_off += 16;
518 }
519
520 /* if not an odd number set valid bit in next fragment */
521 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 &&
522 !(op_info->num_lo_sges & 0x01) && op_info->num_lo_sges) {
523 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
524 qp->swqe_polarity);
525 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
526 ++addl_frag_cnt;
527 }
528 set_64bit_val(wqe, 16,
529 FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
530 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
531 FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
532 FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
533 FIELD_PREP(IRDMAQPSQ_OPCODE,
534 (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |
535 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
536 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
537 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
538 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
539
540 dma_wmb(); /* make sure WQE is populated before valid bit is set */
541
542 set_64bit_val(wqe, 24, hdr);
543
544 if (post_sq)
545 irdma_uk_qp_post_wr(qp);
546
547 return 0;
548 }
549
550 /**
551 * irdma_uk_send - rdma send command
552 * @qp: hw qp ptr
553 * @info: post sq information
554 * @post_sq: flag to post sq
555 */
irdma_uk_send(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)556 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
557 bool post_sq)
558 {
559 __le64 *wqe;
560 struct irdma_post_send *op_info;
561 u64 hdr;
562 u32 i, wqe_idx, total_size = 0, byte_off;
563 int ret_code;
564 u32 frag_cnt, addl_frag_cnt;
565 bool read_fence = false;
566 u16 quanta;
567
568 op_info = &info->op.send;
569 if (qp->max_sq_frag_cnt < op_info->num_sges)
570 return -EINVAL;
571
572 for (i = 0; i < op_info->num_sges; i++)
573 total_size += op_info->sg_list[i].length;
574
575 if (info->imm_data_valid)
576 frag_cnt = op_info->num_sges + 1;
577 else
578 frag_cnt = op_info->num_sges;
579 ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta);
580 if (ret_code)
581 return ret_code;
582
583 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
584 info);
585 if (!wqe)
586 return -ENOMEM;
587
588 irdma_clr_wqes(qp, wqe_idx);
589
590 read_fence |= info->read_fence;
591 addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
592 if (info->imm_data_valid) {
593 set_64bit_val(wqe, 0,
594 FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
595 i = 0;
596 } else {
597 qp->wqe_ops.iw_set_fragment(wqe, 0,
598 frag_cnt ? op_info->sg_list : NULL,
599 qp->swqe_polarity);
600 i = 1;
601 }
602
603 for (byte_off = 32; i < op_info->num_sges; i++) {
604 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i],
605 qp->swqe_polarity);
606 byte_off += 16;
607 }
608
609 /* if not an odd number set valid bit in next fragment */
610 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
611 frag_cnt) {
612 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
613 qp->swqe_polarity);
614 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
615 ++addl_frag_cnt;
616 }
617
618 set_64bit_val(wqe, 16,
619 FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
620 FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
621 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
622 FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
623 FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
624 (info->imm_data_valid ? 1 : 0)) |
625 FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
626 FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
627 FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
628 FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
629 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
630 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
631 FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
632 FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
633 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
634
635 dma_wmb(); /* make sure WQE is populated before valid bit is set */
636
637 set_64bit_val(wqe, 24, hdr);
638
639 if (post_sq)
640 irdma_uk_qp_post_wr(qp);
641
642 return 0;
643 }
644
645 /**
646 * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe
647 * @wqe: wqe for setting fragment
648 * @op_info: info for setting bind wqe values
649 */
irdma_set_mw_bind_wqe_gen_1(__le64 * wqe,struct irdma_bind_window * op_info)650 static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe,
651 struct irdma_bind_window *op_info)
652 {
653 set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
654 set_64bit_val(wqe, 8,
655 FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
656 FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
657 set_64bit_val(wqe, 16, op_info->bind_len);
658 }
659
660 /**
661 * irdma_copy_inline_data_gen_1 - Copy inline data to wqe
662 * @wqe: pointer to wqe
663 * @sge_list: table of pointers to inline data
664 * @num_sges: Total inline data length
665 * @polarity: compatibility parameter
666 */
irdma_copy_inline_data_gen_1(u8 * wqe,struct ib_sge * sge_list,u32 num_sges,u8 polarity)667 static void irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,
668 u32 num_sges, u8 polarity)
669 {
670 u32 quanta_bytes_remaining = 16;
671 int i;
672
673 for (i = 0; i < num_sges; i++) {
674 u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
675 u32 sge_len = sge_list[i].length;
676
677 while (sge_len) {
678 u32 bytes_copied;
679
680 bytes_copied = min(sge_len, quanta_bytes_remaining);
681 memcpy(wqe, cur_sge, bytes_copied);
682 wqe += bytes_copied;
683 cur_sge += bytes_copied;
684 quanta_bytes_remaining -= bytes_copied;
685 sge_len -= bytes_copied;
686
687 if (!quanta_bytes_remaining) {
688 /* Remaining inline bytes reside after hdr */
689 wqe += 16;
690 quanta_bytes_remaining = 32;
691 }
692 }
693 }
694 }
695
696 /**
697 * irdma_inline_data_size_to_quanta_gen_1 - based on inline data, quanta
698 * @data_size: data size for inline
699 *
700 * Gets the quanta based on inline and immediate data.
701 */
irdma_inline_data_size_to_quanta_gen_1(u32 data_size)702 static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size)
703 {
704 return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2;
705 }
706
707 /**
708 * irdma_set_mw_bind_wqe - set mw bind in wqe
709 * @wqe: wqe for setting mw bind
710 * @op_info: info for setting wqe values
711 */
irdma_set_mw_bind_wqe(__le64 * wqe,struct irdma_bind_window * op_info)712 static void irdma_set_mw_bind_wqe(__le64 *wqe,
713 struct irdma_bind_window *op_info)
714 {
715 set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
716 set_64bit_val(wqe, 8,
717 FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) |
718 FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag));
719 set_64bit_val(wqe, 16, op_info->bind_len);
720 }
721
722 /**
723 * irdma_copy_inline_data - Copy inline data to wqe
724 * @wqe: pointer to wqe
725 * @sge_list: table of pointers to inline data
726 * @num_sges: number of SGE's
727 * @polarity: polarity of wqe valid bit
728 */
irdma_copy_inline_data(u8 * wqe,struct ib_sge * sge_list,u32 num_sges,u8 polarity)729 static void irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,
730 u32 num_sges, u8 polarity)
731 {
732 u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
733 u32 quanta_bytes_remaining = 8;
734 bool first_quanta = true;
735 int i;
736
737 wqe += 8;
738
739 for (i = 0; i < num_sges; i++) {
740 u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
741 u32 sge_len = sge_list[i].length;
742
743 while (sge_len) {
744 u32 bytes_copied;
745
746 bytes_copied = min(sge_len, quanta_bytes_remaining);
747 memcpy(wqe, cur_sge, bytes_copied);
748 wqe += bytes_copied;
749 cur_sge += bytes_copied;
750 quanta_bytes_remaining -= bytes_copied;
751 sge_len -= bytes_copied;
752
753 if (!quanta_bytes_remaining) {
754 quanta_bytes_remaining = 31;
755
756 /* Remaining inline bytes reside after hdr */
757 if (first_quanta) {
758 first_quanta = false;
759 wqe += 16;
760 } else {
761 *wqe = inline_valid;
762 wqe++;
763 }
764 }
765 }
766 }
767 if (!first_quanta && quanta_bytes_remaining < 31)
768 *(wqe + quanta_bytes_remaining) = inline_valid;
769 }
770
771 /**
772 * irdma_inline_data_size_to_quanta - based on inline data, quanta
773 * @data_size: data size for inline
774 *
775 * Gets the quanta based on inline and immediate data.
776 */
irdma_inline_data_size_to_quanta(u32 data_size)777 static u16 irdma_inline_data_size_to_quanta(u32 data_size)
778 {
779 if (data_size <= 8)
780 return IRDMA_QP_WQE_MIN_QUANTA;
781 else if (data_size <= 39)
782 return 2;
783 else if (data_size <= 70)
784 return 3;
785 else if (data_size <= 101)
786 return 4;
787 else if (data_size <= 132)
788 return 5;
789 else if (data_size <= 163)
790 return 6;
791 else if (data_size <= 194)
792 return 7;
793 else
794 return 8;
795 }
796
797 /**
798 * irdma_uk_inline_rdma_write - inline rdma write operation
799 * @qp: hw qp ptr
800 * @info: post sq information
801 * @post_sq: flag to post sq
802 */
irdma_uk_inline_rdma_write(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)803 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
804 struct irdma_post_sq_info *info, bool post_sq)
805 {
806 __le64 *wqe;
807 struct irdma_rdma_write *op_info;
808 u64 hdr = 0;
809 u32 wqe_idx;
810 bool read_fence = false;
811 u32 i, total_size = 0;
812 u16 quanta;
813
814 op_info = &info->op.rdma_write;
815
816 if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
817 return -EINVAL;
818
819 for (i = 0; i < op_info->num_lo_sges; i++)
820 total_size += op_info->lo_sg_list[i].length;
821
822 if (unlikely(total_size > qp->max_inline_data))
823 return -EINVAL;
824
825 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
826 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
827 info);
828 if (!wqe)
829 return -ENOMEM;
830
831 irdma_clr_wqes(qp, wqe_idx);
832
833 read_fence |= info->read_fence;
834 set_64bit_val(wqe, 16,
835 FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
836
837 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
838 FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
839 FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
840 FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
841 FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
842 FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) |
843 FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
844 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
845 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
846 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
847
848 if (info->imm_data_valid)
849 set_64bit_val(wqe, 0,
850 FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
851
852 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
853 op_info->num_lo_sges,
854 qp->swqe_polarity);
855 dma_wmb(); /* make sure WQE is populated before valid bit is set */
856
857 set_64bit_val(wqe, 24, hdr);
858
859 if (post_sq)
860 irdma_uk_qp_post_wr(qp);
861
862 return 0;
863 }
864
865 /**
866 * irdma_uk_inline_send - inline send operation
867 * @qp: hw qp ptr
868 * @info: post sq information
869 * @post_sq: flag to post sq
870 */
irdma_uk_inline_send(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)871 int irdma_uk_inline_send(struct irdma_qp_uk *qp,
872 struct irdma_post_sq_info *info, bool post_sq)
873 {
874 __le64 *wqe;
875 struct irdma_post_send *op_info;
876 u64 hdr;
877 u32 wqe_idx;
878 bool read_fence = false;
879 u32 i, total_size = 0;
880 u16 quanta;
881
882 op_info = &info->op.send;
883
884 if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
885 return -EINVAL;
886
887 for (i = 0; i < op_info->num_sges; i++)
888 total_size += op_info->sg_list[i].length;
889
890 if (unlikely(total_size > qp->max_inline_data))
891 return -EINVAL;
892
893 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
894 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
895 info);
896 if (!wqe)
897 return -ENOMEM;
898
899 irdma_clr_wqes(qp, wqe_idx);
900
901 set_64bit_val(wqe, 16,
902 FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
903 FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
904
905 read_fence |= info->read_fence;
906 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
907 FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
908 FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
909 FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
910 FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
911 (info->imm_data_valid ? 1 : 0)) |
912 FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
913 FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
914 FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
915 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
916 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
917 FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
918 FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
919 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
920
921 if (info->imm_data_valid)
922 set_64bit_val(wqe, 0,
923 FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
924 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
925 op_info->num_sges, qp->swqe_polarity);
926
927 dma_wmb(); /* make sure WQE is populated before valid bit is set */
928
929 set_64bit_val(wqe, 24, hdr);
930
931 if (post_sq)
932 irdma_uk_qp_post_wr(qp);
933
934 return 0;
935 }
936
937 /**
938 * irdma_uk_stag_local_invalidate - stag invalidate operation
939 * @qp: hw qp ptr
940 * @info: post sq information
941 * @post_sq: flag to post sq
942 */
irdma_uk_stag_local_invalidate(struct irdma_qp_uk * qp,struct irdma_post_sq_info * info,bool post_sq)943 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
944 struct irdma_post_sq_info *info,
945 bool post_sq)
946 {
947 __le64 *wqe;
948 struct irdma_inv_local_stag *op_info;
949 u64 hdr;
950 u32 wqe_idx;
951 bool local_fence = false;
952 struct ib_sge sge = {};
953
954 op_info = &info->op.inv_local_stag;
955 local_fence = info->local_fence;
956
957 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
958 0, info);
959 if (!wqe)
960 return -ENOMEM;
961
962 irdma_clr_wqes(qp, wqe_idx);
963
964 sge.lkey = op_info->target_stag;
965 qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0);
966
967 set_64bit_val(wqe, 16, 0);
968
969 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) |
970 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
971 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
972 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
973 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
974
975 dma_wmb(); /* make sure WQE is populated before valid bit is set */
976
977 set_64bit_val(wqe, 24, hdr);
978
979 if (post_sq)
980 irdma_uk_qp_post_wr(qp);
981
982 return 0;
983 }
984
985 /**
986 * irdma_uk_post_receive - post receive wqe
987 * @qp: hw qp ptr
988 * @info: post rq information
989 */
irdma_uk_post_receive(struct irdma_qp_uk * qp,struct irdma_post_rq_info * info)990 int irdma_uk_post_receive(struct irdma_qp_uk *qp,
991 struct irdma_post_rq_info *info)
992 {
993 u32 wqe_idx, i, byte_off;
994 u32 addl_frag_cnt;
995 __le64 *wqe;
996 u64 hdr;
997
998 if (qp->max_rq_frag_cnt < info->num_sges)
999 return -EINVAL;
1000
1001 wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
1002 if (!wqe)
1003 return -ENOMEM;
1004
1005 qp->rq_wrid_array[wqe_idx] = info->wr_id;
1006 addl_frag_cnt = info->num_sges > 1 ? (info->num_sges - 1) : 0;
1007 qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list,
1008 qp->rwqe_polarity);
1009
1010 for (i = 1, byte_off = 32; i < info->num_sges; i++) {
1011 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
1012 qp->rwqe_polarity);
1013 byte_off += 16;
1014 }
1015
1016 /* if not an odd number set valid bit in next fragment */
1017 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
1018 info->num_sges) {
1019 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
1020 qp->rwqe_polarity);
1021 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
1022 ++addl_frag_cnt;
1023 }
1024
1025 set_64bit_val(wqe, 16, 0);
1026 hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
1027 FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
1028
1029 dma_wmb(); /* make sure WQE is populated before valid bit is set */
1030
1031 set_64bit_val(wqe, 24, hdr);
1032
1033 return 0;
1034 }
1035
1036 /**
1037 * irdma_uk_cq_resize - reset the cq buffer info
1038 * @cq: cq to resize
1039 * @cq_base: new cq buffer addr
1040 * @cq_size: number of cqes
1041 */
irdma_uk_cq_resize(struct irdma_cq_uk * cq,void * cq_base,int cq_size)1042 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int cq_size)
1043 {
1044 cq->cq_base = cq_base;
1045 cq->cq_size = cq_size;
1046 IRDMA_RING_INIT(cq->cq_ring, cq->cq_size);
1047 cq->polarity = 1;
1048 }
1049
1050 /**
1051 * irdma_uk_cq_set_resized_cnt - record the count of the resized buffers
1052 * @cq: cq to resize
1053 * @cq_cnt: the count of the resized cq buffers
1054 */
irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk * cq,u16 cq_cnt)1055 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *cq, u16 cq_cnt)
1056 {
1057 u64 temp_val;
1058 u16 sw_cq_sel;
1059 u8 arm_next_se;
1060 u8 arm_next;
1061 u8 arm_seq_num;
1062
1063 get_64bit_val(cq->shadow_area, 32, &temp_val);
1064
1065 sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
1066 sw_cq_sel += cq_cnt;
1067
1068 arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
1069 arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
1070 arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val);
1071
1072 temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
1073 FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
1074 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
1075 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
1076
1077 set_64bit_val(cq->shadow_area, 32, temp_val);
1078 }
1079
1080 /**
1081 * irdma_uk_cq_request_notification - cq notification request (door bell)
1082 * @cq: hw cq
1083 * @cq_notify: notification type
1084 */
irdma_uk_cq_request_notification(struct irdma_cq_uk * cq,enum irdma_cmpl_notify cq_notify)1085 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
1086 enum irdma_cmpl_notify cq_notify)
1087 {
1088 u64 temp_val;
1089 u16 sw_cq_sel;
1090 u8 arm_next_se = 0;
1091 u8 arm_next = 0;
1092 u8 arm_seq_num;
1093
1094 get_64bit_val(cq->shadow_area, 32, &temp_val);
1095 arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
1096 arm_seq_num++;
1097 sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
1098 arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
1099 arm_next_se |= 1;
1100 if (cq_notify == IRDMA_CQ_COMPL_EVENT)
1101 arm_next = 1;
1102 temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
1103 FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
1104 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
1105 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
1106
1107 set_64bit_val(cq->shadow_area, 32, temp_val);
1108
1109 writel(cq->cq_id, cq->cqe_alloc_db);
1110 }
1111
1112 /**
1113 * irdma_uk_cq_empty - Check if CQ is empty
1114 * @cq: hw cq
1115 */
irdma_uk_cq_empty(struct irdma_cq_uk * cq)1116 bool irdma_uk_cq_empty(struct irdma_cq_uk *cq)
1117 {
1118 __le64 *cqe;
1119 u8 polarity;
1120 u64 qword3;
1121
1122 if (cq->avoid_mem_cflct)
1123 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq);
1124 else
1125 cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);
1126
1127 get_64bit_val(cqe, 24, &qword3);
1128 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
1129
1130 return polarity != cq->polarity;
1131 }
1132
1133 /**
1134 * irdma_uk_cq_poll_cmpl - get cq completion info
1135 * @cq: hw cq
1136 * @info: cq poll information returned
1137 */
irdma_uk_cq_poll_cmpl(struct irdma_cq_uk * cq,struct irdma_cq_poll_info * info)1138 int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
1139 struct irdma_cq_poll_info *info)
1140 {
1141 u64 comp_ctx, qword0, qword2, qword3;
1142 __le64 *cqe;
1143 struct irdma_qp_uk *qp;
1144 struct irdma_srq_uk *srq;
1145 struct qp_err_code qp_err;
1146 u8 is_srq;
1147 struct irdma_ring *pring = NULL;
1148 u32 wqe_idx;
1149 int ret_code;
1150 bool move_cq_head = true;
1151 u8 polarity;
1152 bool ext_valid;
1153 __le64 *ext_cqe;
1154
1155 if (cq->avoid_mem_cflct)
1156 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq);
1157 else
1158 cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);
1159
1160 get_64bit_val(cqe, 24, &qword3);
1161 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
1162 if (polarity != cq->polarity)
1163 return -ENOENT;
1164
1165 /* Ensure CQE contents are read after valid bit is checked */
1166 dma_rmb();
1167
1168 ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
1169 if (ext_valid) {
1170 u64 qword6, qword7;
1171 u32 peek_head;
1172
1173 if (cq->avoid_mem_cflct) {
1174 ext_cqe = (__le64 *)((u8 *)cqe + 32);
1175 get_64bit_val(ext_cqe, 24, &qword7);
1176 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
1177 } else {
1178 peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size;
1179 ext_cqe = cq->cq_base[peek_head].buf;
1180 get_64bit_val(ext_cqe, 24, &qword7);
1181 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
1182 if (!peek_head)
1183 polarity ^= 1;
1184 }
1185 if (polarity != cq->polarity)
1186 return -ENOENT;
1187
1188 /* Ensure ext CQE contents are read after ext valid bit is checked */
1189 dma_rmb();
1190
1191 info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7);
1192 if (info->imm_valid) {
1193 u64 qword4;
1194
1195 get_64bit_val(ext_cqe, 0, &qword4);
1196 info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4);
1197 }
1198 info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
1199 info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
1200 if (info->ud_smac_valid || info->ud_vlan_valid) {
1201 get_64bit_val(ext_cqe, 16, &qword6);
1202 if (info->ud_vlan_valid)
1203 info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
1204 if (info->ud_smac_valid) {
1205 info->ud_smac[5] = qword6 & 0xFF;
1206 info->ud_smac[4] = (qword6 >> 8) & 0xFF;
1207 info->ud_smac[3] = (qword6 >> 16) & 0xFF;
1208 info->ud_smac[2] = (qword6 >> 24) & 0xFF;
1209 info->ud_smac[1] = (qword6 >> 32) & 0xFF;
1210 info->ud_smac[0] = (qword6 >> 40) & 0xFF;
1211 }
1212 }
1213 } else {
1214 info->imm_valid = false;
1215 info->ud_smac_valid = false;
1216 info->ud_vlan_valid = false;
1217 }
1218
1219 info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1220 is_srq = (u8)FIELD_GET(IRDMA_CQ_SRQ, qword3);
1221 info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
1222 info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
1223 get_64bit_val(cqe, 8, &comp_ctx);
1224 if (is_srq)
1225 get_64bit_val(cqe, 40, (u64 *)&qp);
1226 else
1227 qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
1228 if (info->error) {
1229 info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);
1230 info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);
1231 switch (info->major_err) {
1232 case IRDMA_SRQFLUSH_RSVD_MAJOR_ERR:
1233 qp_err = irdma_ae_to_qp_err_code(info->minor_err);
1234 info->minor_err = qp_err.flush_code;
1235 fallthrough;
1236 case IRDMA_FLUSH_MAJOR_ERR:
1237 /* Set the min error to standard flush error code for remaining cqes */
1238 if (info->minor_err != FLUSH_GENERAL_ERR) {
1239 qword3 &= ~IRDMA_CQ_MINERR;
1240 qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR);
1241 set_64bit_val(cqe, 24, qword3);
1242 }
1243 info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
1244 break;
1245 default:
1246 #define IRDMA_CIE_SIGNATURE 0xE
1247 #define IRDMA_CQMAJERR_HIGH_NIBBLE GENMASK(15, 12)
1248 if (info->q_type == IRDMA_CQE_QTYPE_SQ &&
1249 qp->qp_type == IRDMA_QP_TYPE_ROCE_UD &&
1250 FIELD_GET(IRDMA_CQMAJERR_HIGH_NIBBLE, info->major_err)
1251 == IRDMA_CIE_SIGNATURE) {
1252 info->error = 0;
1253 info->major_err = 0;
1254 info->minor_err = 0;
1255 info->comp_status = IRDMA_COMPL_STATUS_SUCCESS;
1256 } else {
1257 info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN;
1258 }
1259 break;
1260 }
1261 } else {
1262 info->comp_status = IRDMA_COMPL_STATUS_SUCCESS;
1263 }
1264
1265 get_64bit_val(cqe, 0, &qword0);
1266 get_64bit_val(cqe, 16, &qword2);
1267
1268 info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
1269 info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2);
1270
1271 get_64bit_val(cqe, 8, &comp_ctx);
1272
1273 info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);
1274 qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
1275 if (!qp || qp->destroy_pending) {
1276 ret_code = -EFAULT;
1277 goto exit;
1278 }
1279 wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
1280 info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
1281 info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
1282
1283 if (info->q_type == IRDMA_CQE_QTYPE_RQ && is_srq) {
1284 unsigned long flags;
1285
1286 srq = qp->srq_uk;
1287
1288 get_64bit_val(cqe, 8, &info->wr_id);
1289 info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
1290
1291 if (qword3 & IRDMACQ_STAG) {
1292 info->stag_invalid_set = true;
1293 info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG,
1294 qword2);
1295 } else {
1296 info->stag_invalid_set = false;
1297 }
1298 spin_lock_irqsave(srq->lock, flags);
1299 IRDMA_RING_MOVE_TAIL(srq->srq_ring);
1300 spin_unlock_irqrestore(srq->lock, flags);
1301 pring = &srq->srq_ring;
1302
1303 } else if (info->q_type == IRDMA_CQE_QTYPE_RQ && !is_srq) {
1304 u32 array_idx;
1305
1306 array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
1307
1308 if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED ||
1309 info->comp_status == IRDMA_COMPL_STATUS_UNKNOWN) {
1310 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) {
1311 ret_code = -ENOENT;
1312 goto exit;
1313 }
1314
1315 info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
1316 array_idx = qp->rq_ring.tail;
1317 } else {
1318 info->wr_id = qp->rq_wrid_array[array_idx];
1319 }
1320
1321 info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
1322
1323 if (qword3 & IRDMACQ_STAG) {
1324 info->stag_invalid_set = true;
1325 info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
1326 } else {
1327 info->stag_invalid_set = false;
1328 }
1329 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
1330 if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) {
1331 qp->rq_flush_seen = true;
1332 if (!IRDMA_RING_MORE_WORK(qp->rq_ring))
1333 qp->rq_flush_complete = true;
1334 else
1335 move_cq_head = false;
1336 }
1337 pring = &qp->rq_ring;
1338 } else { /* q_type is IRDMA_CQE_QTYPE_SQ */
1339 if (qp->first_sq_wq) {
1340 if (wqe_idx + 1 >= qp->conn_wqes)
1341 qp->first_sq_wq = false;
1342
1343 if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) {
1344 IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
1345 IRDMA_RING_MOVE_TAIL(cq->cq_ring);
1346 set_64bit_val(cq->shadow_area, 0,
1347 IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
1348 memset(info, 0,
1349 sizeof(struct irdma_cq_poll_info));
1350 return irdma_uk_cq_poll_cmpl(cq, info);
1351 }
1352 }
1353 if (info->comp_status != IRDMA_COMPL_STATUS_FLUSHED) {
1354 info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
1355 if (!info->comp_status)
1356 info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
1357 if (!qp->sq_wrtrk_array[wqe_idx].signaled) {
1358 ret_code = -EFAULT;
1359 goto exit;
1360 }
1361 info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
1362 IRDMA_RING_SET_TAIL(qp->sq_ring,
1363 wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
1364 } else {
1365 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
1366 ret_code = -ENOENT;
1367 goto exit;
1368 }
1369
1370 do {
1371 __le64 *sw_wqe;
1372 u64 wqe_qword;
1373 u32 tail;
1374
1375 tail = qp->sq_ring.tail;
1376 sw_wqe = qp->sq_base[tail].elem;
1377 get_64bit_val(sw_wqe, 24,
1378 &wqe_qword);
1379 info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
1380 wqe_qword);
1381 IRDMA_RING_SET_TAIL(qp->sq_ring,
1382 tail + qp->sq_wrtrk_array[tail].quanta);
1383 if (info->op_type != IRDMAQP_OP_NOP) {
1384 info->wr_id = qp->sq_wrtrk_array[tail].wrid;
1385 info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
1386 break;
1387 }
1388 } while (1);
1389 if (info->op_type == IRDMA_OP_TYPE_BIND_MW &&
1390 info->minor_err == FLUSH_PROT_ERR)
1391 info->minor_err = FLUSH_MW_BIND_ERR;
1392 qp->sq_flush_seen = true;
1393 if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
1394 qp->sq_flush_complete = true;
1395 }
1396 pring = &qp->sq_ring;
1397 }
1398
1399 ret_code = 0;
1400
1401 exit:
1402 if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) {
1403 if (pring && IRDMA_RING_MORE_WORK(*pring))
1404 /* Park CQ head during a flush to generate additional CQEs
1405 * from SW for all unprocessed WQEs. For GEN3 and beyond
1406 * FW will generate/flush these CQEs so move to the next CQE
1407 */
1408 move_cq_head = qp->uk_attrs->hw_rev > IRDMA_GEN_2;
1409 }
1410
1411 if (move_cq_head) {
1412 IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
1413 if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
1414 cq->polarity ^= 1;
1415
1416 if (ext_valid && !cq->avoid_mem_cflct) {
1417 IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
1418 if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
1419 cq->polarity ^= 1;
1420 }
1421
1422 IRDMA_RING_MOVE_TAIL(cq->cq_ring);
1423 if (!cq->avoid_mem_cflct && ext_valid)
1424 IRDMA_RING_MOVE_TAIL(cq->cq_ring);
1425 if (IRDMA_RING_CURRENT_HEAD(cq->cq_ring) & 0x3F || irdma_uk_cq_empty(cq))
1426 set_64bit_val(cq->shadow_area, 0,
1427 IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
1428 } else {
1429 qword3 &= ~IRDMA_CQ_WQEIDX;
1430 qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail);
1431 set_64bit_val(cqe, 24, qword3);
1432 }
1433
1434 return ret_code;
1435 }
1436
1437 /**
1438 * irdma_round_up_wq - return round up qp wq depth
1439 * @wqdepth: wq depth in quanta to round up
1440 */
irdma_round_up_wq(u64 wqdepth)1441 static u64 irdma_round_up_wq(u64 wqdepth)
1442 {
1443 int scount = 1;
1444
1445 for (wqdepth--; scount <= 16; scount *= 2)
1446 wqdepth |= wqdepth >> scount;
1447
1448 return ++wqdepth;
1449 }
1450
1451 /**
1452 * irdma_get_wqe_shift - get shift count for maximum wqe size
1453 * @uk_attrs: qp HW attributes
1454 * @sge: Maximum Scatter Gather Elements wqe
1455 * @inline_data: Maximum inline data size
1456 * @shift: Returns the shift needed based on sge
1457 *
1458 * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.
1459 * For 1 SGE or inline data <= 8, shift = 0 (wqe size of 32
1460 * bytes). For 2 or 3 SGEs or inline data <= 39, shift = 1 (wqe
1461 * size of 64 bytes).
1462 * For 4-7 SGE's and inline <= 101 Shift of 2 otherwise (wqe
1463 * size of 256 bytes).
1464 */
irdma_get_wqe_shift(struct irdma_uk_attrs * uk_attrs,u32 sge,u32 inline_data,u8 * shift)1465 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
1466 u32 inline_data, u8 *shift)
1467 {
1468 *shift = 0;
1469 if (uk_attrs->hw_rev >= IRDMA_GEN_2) {
1470 if (sge > 1 || inline_data > 8) {
1471 if (sge < 4 && inline_data <= 39)
1472 *shift = 1;
1473 else if (sge < 8 && inline_data <= 101)
1474 *shift = 2;
1475 else
1476 *shift = 3;
1477 }
1478 } else if (sge > 1 || inline_data > 16) {
1479 *shift = (sge < 4 && inline_data <= 48) ? 1 : 2;
1480 }
1481 }
1482
1483 /*
1484 * irdma_get_sqdepth - get SQ depth (quanta)
1485 * @uk_attrs: qp HW attributes
1486 * @sq_size: SQ size
1487 * @shift: shift which determines size of WQE
1488 * @sqdepth: depth of SQ
1489 *
1490 */
irdma_get_sqdepth(struct irdma_uk_attrs * uk_attrs,u32 sq_size,u8 shift,u32 * sqdepth)1491 int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
1492 u32 *sqdepth)
1493 {
1494 u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift;
1495 u64 hw_quanta =
1496 irdma_round_up_wq(((u64)sq_size << shift) + IRDMA_SQ_RSVD);
1497
1498 if (hw_quanta < min_hw_quanta)
1499 hw_quanta = min_hw_quanta;
1500 else if (hw_quanta > uk_attrs->max_hw_wq_quanta)
1501 return -EINVAL;
1502
1503 *sqdepth = hw_quanta;
1504 return 0;
1505 }
1506
1507 /*
1508 * irdma_get_rqdepth - get RQ depth (quanta)
1509 * @uk_attrs: qp HW attributes
1510 * @rq_size: RQ size
1511 * @shift: shift which determines size of WQE
1512 * @rqdepth: depth of RQ
1513 */
irdma_get_rqdepth(struct irdma_uk_attrs * uk_attrs,u32 rq_size,u8 shift,u32 * rqdepth)1514 int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
1515 u32 *rqdepth)
1516 {
1517 u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift;
1518 u64 hw_quanta =
1519 irdma_round_up_wq(((u64)rq_size << shift) + IRDMA_RQ_RSVD);
1520
1521 if (hw_quanta < min_hw_quanta)
1522 hw_quanta = min_hw_quanta;
1523 else if (hw_quanta > uk_attrs->max_hw_rq_quanta)
1524 return -EINVAL;
1525
1526 *rqdepth = hw_quanta;
1527 return 0;
1528 }
1529
1530 /*
1531 * irdma_get_srqdepth - get SRQ depth (quanta)
1532 * @uk_attrs: qp HW attributes
1533 * @srq_size: SRQ size
1534 * @shift: shift which determines size of WQE
1535 * @srqdepth: depth of SRQ
1536 */
irdma_get_srqdepth(struct irdma_uk_attrs * uk_attrs,u32 srq_size,u8 shift,u32 * srqdepth)1537 int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,
1538 u32 *srqdepth)
1539 {
1540 u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift;
1541 u64 hw_quanta =
1542 irdma_round_up_wq(((u64)srq_size << shift) + IRDMA_RQ_RSVD);
1543
1544 if (hw_quanta < min_hw_quanta)
1545 hw_quanta = min_hw_quanta;
1546 else if (hw_quanta > uk_attrs->max_hw_srq_quanta)
1547 return -EINVAL;
1548
1549 *srqdepth = hw_quanta;
1550 return 0;
1551 }
1552
1553 static const struct irdma_wqe_uk_ops iw_wqe_uk_ops = {
1554 .iw_copy_inline_data = irdma_copy_inline_data,
1555 .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta,
1556 .iw_set_fragment = irdma_set_fragment,
1557 .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe,
1558 };
1559
1560 static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = {
1561 .iw_copy_inline_data = irdma_copy_inline_data_gen_1,
1562 .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1,
1563 .iw_set_fragment = irdma_set_fragment_gen_1,
1564 .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1,
1565 };
1566
1567 /**
1568 * irdma_setup_connection_wqes - setup WQEs necessary to complete
1569 * connection.
1570 * @qp: hw qp (user and kernel)
1571 * @info: qp initialization info
1572 */
irdma_setup_connection_wqes(struct irdma_qp_uk * qp,struct irdma_qp_uk_init_info * info)1573 static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
1574 struct irdma_qp_uk_init_info *info)
1575 {
1576 u16 move_cnt = 1;
1577
1578 if (!info->legacy_mode &&
1579 (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE))
1580 move_cnt = 3;
1581
1582 qp->conn_wqes = move_cnt;
1583 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
1584 IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
1585 }
1586
1587 /**
1588 * irdma_uk_srq_init - initialize shared qp
1589 * @srq: hw srq (user and kernel)
1590 * @info: srq initialization info
1591 *
1592 * Initializes the vars used in both user and kernel mode.
1593 * The size of the wqe depends on number of max fragments
1594 * allowed. Then size of wqe * the number of wqes should be the
1595 * amount of memory allocated for srq.
1596 */
irdma_uk_srq_init(struct irdma_srq_uk * srq,struct irdma_srq_uk_init_info * info)1597 int irdma_uk_srq_init(struct irdma_srq_uk *srq,
1598 struct irdma_srq_uk_init_info *info)
1599 {
1600 u8 rqshift;
1601
1602 srq->uk_attrs = info->uk_attrs;
1603 if (info->max_srq_frag_cnt > srq->uk_attrs->max_hw_wq_frags)
1604 return -EINVAL;
1605
1606 irdma_get_wqe_shift(srq->uk_attrs, info->max_srq_frag_cnt, 0, &rqshift);
1607 srq->srq_caps = info->srq_caps;
1608 srq->srq_base = info->srq;
1609 srq->shadow_area = info->shadow_area;
1610 srq->srq_id = info->srq_id;
1611 srq->srwqe_polarity = 0;
1612 srq->srq_size = info->srq_size;
1613 srq->wqe_size = rqshift;
1614 srq->max_srq_frag_cnt = min(srq->uk_attrs->max_hw_wq_frags,
1615 ((u32)2 << rqshift) - 1);
1616 IRDMA_RING_INIT(srq->srq_ring, srq->srq_size);
1617 srq->wqe_size_multiplier = 1 << rqshift;
1618 srq->wqe_ops = iw_wqe_uk_ops;
1619
1620 return 0;
1621 }
1622
1623 /**
1624 * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
1625 * @ukinfo: qp initialization info
1626 * @sq_shift: Returns shift of SQ
1627 * @rq_shift: Returns shift of RQ
1628 */
irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info * ukinfo,u8 * sq_shift,u8 * rq_shift)1629 void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
1630 u8 *rq_shift)
1631 {
1632 bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2;
1633
1634 irdma_get_wqe_shift(ukinfo->uk_attrs,
1635 imm_support ? ukinfo->max_sq_frag_cnt + 1 :
1636 ukinfo->max_sq_frag_cnt,
1637 ukinfo->max_inline_data, sq_shift);
1638
1639 irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
1640 rq_shift);
1641
1642 if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
1643 if (ukinfo->abi_ver > 4)
1644 *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
1645 }
1646 }
1647
1648 /**
1649 * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
1650 * @ukinfo: qp initialization info
1651 * @sq_depth: Returns depth of SQ
1652 * @sq_shift: Returns shift of SQ
1653 */
irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info * ukinfo,u32 * sq_depth,u8 * sq_shift)1654 int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
1655 u32 *sq_depth, u8 *sq_shift)
1656 {
1657 bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2;
1658 int status;
1659
1660 irdma_get_wqe_shift(ukinfo->uk_attrs,
1661 imm_support ? ukinfo->max_sq_frag_cnt + 1 :
1662 ukinfo->max_sq_frag_cnt,
1663 ukinfo->max_inline_data, sq_shift);
1664 status = irdma_get_sqdepth(ukinfo->uk_attrs, ukinfo->sq_size,
1665 *sq_shift, sq_depth);
1666
1667 return status;
1668 }
1669
1670 /**
1671 * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size.
1672 * @ukinfo: qp initialization info
1673 * @rq_depth: Returns depth of RQ
1674 * @rq_shift: Returns shift of RQ
1675 */
irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info * ukinfo,u32 * rq_depth,u8 * rq_shift)1676 int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
1677 u32 *rq_depth, u8 *rq_shift)
1678 {
1679 int status;
1680
1681 irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
1682 rq_shift);
1683
1684 if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
1685 if (ukinfo->abi_ver > 4)
1686 *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
1687 }
1688
1689 status = irdma_get_rqdepth(ukinfo->uk_attrs, ukinfo->rq_size,
1690 *rq_shift, rq_depth);
1691
1692 return status;
1693 }
1694
1695 /**
1696 * irdma_uk_qp_init - initialize shared qp
1697 * @qp: hw qp (user and kernel)
1698 * @info: qp initialization info
1699 *
1700 * initializes the vars used in both user and kernel mode.
1701 * size of the wqe depends on numbers of max. fragements
1702 * allowed. Then size of wqe * the number of wqes should be the
1703 * amount of memory allocated for sq and rq.
1704 */
irdma_uk_qp_init(struct irdma_qp_uk * qp,struct irdma_qp_uk_init_info * info)1705 int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
1706 {
1707 int ret_code = 0;
1708 u32 sq_ring_size;
1709
1710 qp->uk_attrs = info->uk_attrs;
1711 if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
1712 info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
1713 return -EINVAL;
1714
1715 qp->qp_caps = info->qp_caps;
1716 qp->sq_base = info->sq;
1717 qp->rq_base = info->rq;
1718 qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;
1719 qp->shadow_area = info->shadow_area;
1720 qp->sq_wrtrk_array = info->sq_wrtrk_array;
1721
1722 qp->rq_wrid_array = info->rq_wrid_array;
1723 qp->wqe_alloc_db = info->wqe_alloc_db;
1724 qp->qp_id = info->qp_id;
1725 qp->sq_size = info->sq_size;
1726 qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
1727 sq_ring_size = qp->sq_size << info->sq_shift;
1728 IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
1729 if (info->first_sq_wq) {
1730 irdma_setup_connection_wqes(qp, info);
1731 qp->swqe_polarity = 1;
1732 qp->first_sq_wq = true;
1733 } else {
1734 qp->swqe_polarity = 0;
1735 }
1736 qp->swqe_polarity_deferred = 1;
1737 qp->rwqe_polarity = 0;
1738 qp->rq_size = info->rq_size;
1739 qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
1740 qp->max_inline_data = info->max_inline_data;
1741 qp->rq_wqe_size = info->rq_shift;
1742 IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
1743 qp->rq_wqe_size_multiplier = 1 << info->rq_shift;
1744 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
1745 qp->wqe_ops = iw_wqe_uk_ops_gen_1;
1746 else
1747 qp->wqe_ops = iw_wqe_uk_ops;
1748 qp->srq_uk = info->srq_uk;
1749 return ret_code;
1750 }
1751
1752 /**
1753 * irdma_uk_cq_init - initialize shared cq (user and kernel)
1754 * @cq: hw cq
1755 * @info: hw cq initialization info
1756 */
irdma_uk_cq_init(struct irdma_cq_uk * cq,struct irdma_cq_uk_init_info * info)1757 void irdma_uk_cq_init(struct irdma_cq_uk *cq,
1758 struct irdma_cq_uk_init_info *info)
1759 {
1760 cq->cq_base = info->cq_base;
1761 cq->cq_id = info->cq_id;
1762 cq->cq_size = info->cq_size;
1763 cq->cqe_alloc_db = info->cqe_alloc_db;
1764 cq->cq_ack_db = info->cq_ack_db;
1765 cq->shadow_area = info->shadow_area;
1766 cq->avoid_mem_cflct = info->avoid_mem_cflct;
1767 IRDMA_RING_INIT(cq->cq_ring, cq->cq_size);
1768 cq->polarity = 1;
1769 }
1770
1771 /**
1772 * irdma_uk_clean_cq - clean cq entries
1773 * @q: completion context
1774 * @cq: cq to clean
1775 */
irdma_uk_clean_cq(void * q,struct irdma_cq_uk * cq)1776 void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)
1777 {
1778 __le64 *cqe;
1779 u64 qword3, comp_ctx;
1780 u32 cq_head;
1781 u8 polarity, temp;
1782
1783 cq_head = cq->cq_ring.head;
1784 temp = cq->polarity;
1785 do {
1786 if (cq->avoid_mem_cflct)
1787 cqe = ((struct irdma_extended_cqe *)(cq->cq_base))[cq_head].buf;
1788 else
1789 cqe = cq->cq_base[cq_head].buf;
1790 get_64bit_val(cqe, 24, &qword3);
1791 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
1792
1793 if (polarity != temp)
1794 break;
1795
1796 /* Ensure CQE contents are read after valid bit is checked */
1797 dma_rmb();
1798
1799 get_64bit_val(cqe, 8, &comp_ctx);
1800 if ((void *)(unsigned long)comp_ctx == q)
1801 set_64bit_val(cqe, 8, 0);
1802
1803 cq_head = (cq_head + 1) % cq->cq_ring.size;
1804 if (!cq_head)
1805 temp ^= 1;
1806 } while (true);
1807 }
1808
1809 /**
1810 * irdma_nop - post a nop
1811 * @qp: hw qp ptr
1812 * @wr_id: work request id
1813 * @signaled: signaled for completion
1814 * @post_sq: ring doorbell
1815 */
irdma_nop(struct irdma_qp_uk * qp,u64 wr_id,bool signaled,bool post_sq)1816 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
1817 {
1818 __le64 *wqe;
1819 u64 hdr;
1820 u32 wqe_idx;
1821 struct irdma_post_sq_info info = {};
1822
1823 info.wr_id = wr_id;
1824 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
1825 0, &info);
1826 if (!wqe)
1827 return -ENOMEM;
1828
1829 irdma_clr_wqes(qp, wqe_idx);
1830
1831 set_64bit_val(wqe, 0, 0);
1832 set_64bit_val(wqe, 8, 0);
1833 set_64bit_val(wqe, 16, 0);
1834
1835 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
1836 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
1837 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
1838
1839 dma_wmb(); /* make sure WQE is populated before valid bit is set */
1840
1841 set_64bit_val(wqe, 24, hdr);
1842 if (post_sq)
1843 irdma_uk_qp_post_wr(qp);
1844
1845 return 0;
1846 }
1847
1848 /**
1849 * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ
1850 * @frag_cnt: number of fragments
1851 * @quanta: quanta for frag_cnt
1852 */
irdma_fragcnt_to_quanta_sq(u32 frag_cnt,u16 * quanta)1853 int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)
1854 {
1855 switch (frag_cnt) {
1856 case 0:
1857 case 1:
1858 *quanta = IRDMA_QP_WQE_MIN_QUANTA;
1859 break;
1860 case 2:
1861 case 3:
1862 *quanta = 2;
1863 break;
1864 case 4:
1865 case 5:
1866 *quanta = 3;
1867 break;
1868 case 6:
1869 case 7:
1870 *quanta = 4;
1871 break;
1872 case 8:
1873 case 9:
1874 *quanta = 5;
1875 break;
1876 case 10:
1877 case 11:
1878 *quanta = 6;
1879 break;
1880 case 12:
1881 case 13:
1882 *quanta = 7;
1883 break;
1884 case 14:
1885 case 15: /* when immediate data is present */
1886 *quanta = 8;
1887 break;
1888 default:
1889 return -EINVAL;
1890 }
1891
1892 return 0;
1893 }
1894
1895 /**
1896 * irdma_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ
1897 * @frag_cnt: number of fragments
1898 * @wqe_size: size in bytes given frag_cnt
1899 */
irdma_fragcnt_to_wqesize_rq(u32 frag_cnt,u16 * wqe_size)1900 int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size)
1901 {
1902 switch (frag_cnt) {
1903 case 0:
1904 case 1:
1905 *wqe_size = 32;
1906 break;
1907 case 2:
1908 case 3:
1909 *wqe_size = 64;
1910 break;
1911 case 4:
1912 case 5:
1913 case 6:
1914 case 7:
1915 *wqe_size = 128;
1916 break;
1917 case 8:
1918 case 9:
1919 case 10:
1920 case 11:
1921 case 12:
1922 case 13:
1923 case 14:
1924 *wqe_size = 256;
1925 break;
1926 default:
1927 return -EINVAL;
1928 }
1929
1930 return 0;
1931 }
1932