1 /*
2 * Copyright (c) 2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: Doorbell handling functions.
29 */
30
31 #include <malloc.h>
32 #include <unistd.h>
33
34 #include "abi.h"
35 #include "main.h"
36
37 #define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000
38 #define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00
39
40 #define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000
41 #define BNXT_RE_MAX_FIFO_DEPTH_P7 0x7fff
42
43 #define BNXT_RE_DB_FIFO_ROOM_SHIFT 15
44 #define BNXT_RE_DB_THRESHOLD 20
45
46 #define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \
47 (_is_chip_thor2((ctx)) ? \
48 BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\
49 BNXT_RE_DB_FIFO_ROOM_MASK_P5)
50 #define BNXT_RE_MAX_FIFO_DEPTH(ctx) \
51 (_is_chip_thor2((ctx)) ? \
52 BNXT_RE_MAX_FIFO_DEPTH_P7 :\
53 BNXT_RE_MAX_FIFO_DEPTH_P5)
54
xorshift32(struct xorshift32_state * state)55 static uint32_t xorshift32(struct xorshift32_state *state)
56 {
57 /* Algorithm "xor" from p. 4 of Marsaglia, "Xorshift RNGs" */
58 uint32_t x = state->seed;
59
60 x ^= x << 13;
61 x ^= x >> 17;
62 x ^= x << 5;
63 return state->seed = x;
64 }
65
rnd(struct xorshift32_state * state,uint16_t range)66 static uint16_t rnd(struct xorshift32_state *state, uint16_t range)
67 {
68 /* range must be a power of 2 - 1 */
69 return (xorshift32(state) & range);
70 }
71
calculate_fifo_occupancy(struct bnxt_re_context * cntx)72 static int calculate_fifo_occupancy(struct bnxt_re_context *cntx)
73 {
74 uint32_t *dbr_map = cntx->bar_map + 0x1a8;
75 uint32_t read_val, fifo_occup;
76
77 read_val = *dbr_map;
78 fifo_occup = BNXT_RE_MAX_FIFO_DEPTH(cntx->cctx) -
79 ((read_val & BNXT_RE_DB_FIFO_ROOM_MASK(cntx->cctx)) >>
80 BNXT_RE_DB_FIFO_ROOM_SHIFT);
81
82 return fifo_occup;
83 }
84
find_min(uint32_t x,uint32_t y)85 static inline uint32_t find_min(uint32_t x, uint32_t y)
86 {
87 return (y > x ? x : y);
88 }
89
bnxt_re_do_pacing(struct bnxt_re_context * cntx,struct xorshift32_state * state)90 int bnxt_re_do_pacing(struct bnxt_re_context *cntx, struct xorshift32_state *state)
91 {
92 /* First 4 bytes of shared page (pacing_info) contains the DBR
93 * pacing information. Second 4 bytes (pacing_th) contains
94 * the pacing threshold value to determine whether to
95 * add delay or not
96 */
97 struct bnxt_re_pacing_data *pacing_data =
98 (struct bnxt_re_pacing_data *)cntx->dbr_page;
99 uint32_t wait_time = 1;
100 uint32_t fifo_occup;
101
102 if (!pacing_data)
103 return 0;
104 /* If the device in error recovery state, return error to
105 * not to ring new doorbells in this state.
106 */
107 if (pacing_data->dev_err_state)
108 return -EFAULT;
109
110 if (rnd(state, BNXT_RE_MAX_DO_PACING) < pacing_data->do_pacing) {
111 while ((fifo_occup = calculate_fifo_occupancy(cntx))
112 > pacing_data->pacing_th) {
113 struct bnxt_re_cq *cq;
114 uint32_t usec_wait;
115
116 if (pacing_data->alarm_th && fifo_occup > pacing_data->alarm_th) {
117 cq = container_of(cntx->dbr_cq, struct bnxt_re_cq, ibvcq);
118 bnxt_re_poll_kernel_cq(cq);
119 }
120 usec_wait = rnd(state, wait_time - 1);
121 if (usec_wait)
122 bnxt_re_sub_sec_busy_wait(usec_wait * 1000);
123 /* wait time capped at 128 us */
124 wait_time = find_min(wait_time * 2, 128);
125 }
126 }
127 return 0;
128 }
129
bnxt_re_ring_db(struct bnxt_re_dpi * dpi,__u64 key,uint64_t * db_key,uint8_t * lock)130 static inline void bnxt_re_ring_db(struct bnxt_re_dpi *dpi, __u64 key,
131 uint64_t *db_key, uint8_t *lock)
132 {
133 while (1) {
134 if (__sync_bool_compare_and_swap(lock, 0, 1)) {
135 *db_key = key;
136 bnxt_re_wm_barrier();
137 iowrite64(dpi->dbpage, key);
138 bnxt_re_wm_barrier();
139 *lock = 0;
140 break;
141 }
142 }
143 }
144
bnxt_re_init_push_hdr(struct bnxt_re_db_hdr * hdr,uint32_t indx,uint32_t qid,uint32_t typ,uint32_t pidx)145 static inline void bnxt_re_init_push_hdr(struct bnxt_re_db_hdr *hdr,
146 uint32_t indx, uint32_t qid,
147 uint32_t typ, uint32_t pidx)
148 {
149 __u64 key_lo, key_hi;
150
151 key_lo = (((pidx & BNXT_RE_DB_PILO_MASK) << BNXT_RE_DB_PILO_SHIFT) |
152 (indx & BNXT_RE_DB_INDX_MASK));
153 key_hi = ((((pidx & BNXT_RE_DB_PIHI_MASK) << BNXT_RE_DB_PIHI_SHIFT) |
154 (qid & BNXT_RE_DB_QID_MASK)) |
155 ((typ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) |
156 (0x1UL << BNXT_RE_DB_VALID_SHIFT));
157 hdr->typ_qid_indx = htole64((key_lo | (key_hi << 32)));
158 }
159
bnxt_re_init_db_hdr(struct bnxt_re_db_hdr * hdr,uint32_t indx,uint32_t toggle,uint32_t qid,uint32_t typ)160 static inline void bnxt_re_init_db_hdr(struct bnxt_re_db_hdr *hdr,
161 uint32_t indx, uint32_t toggle,
162 uint32_t qid, uint32_t typ)
163 {
164 __u64 key_lo, key_hi;
165
166 key_lo = htole32(indx | toggle);
167 key_hi = ((qid & BNXT_RE_DB_QID_MASK) |
168 ((typ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) |
169 (0x1UL << BNXT_RE_DB_VALID_SHIFT));
170 hdr->typ_qid_indx = htole64((key_lo | (key_hi << 32)));
171 }
172
__bnxt_re_ring_pend_db(__u64 * ucdb,__u64 key,struct bnxt_re_qp * qp)173 static inline void __bnxt_re_ring_pend_db(__u64 *ucdb, __u64 key,
174 struct bnxt_re_qp *qp)
175 {
176 struct bnxt_re_db_hdr hdr;
177
178 bnxt_re_init_db_hdr(&hdr,
179 (*qp->jsqq->hwque->dbtail |
180 ((qp->jsqq->hwque->flags &
181 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
182 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
183 qp->qpid,
184 BNXT_RE_QUE_TYPE_SQ);
185
186 while (1) {
187 if (__sync_bool_compare_and_swap(&qp->sq_dbr_lock, 0, 1)) {
188 qp->sq_shadow_db_key = hdr.typ_qid_indx;
189 bnxt_re_wm_barrier();
190 iowrite64(ucdb, key);
191 bnxt_re_wm_barrier();
192 qp->sq_dbr_lock = 0;
193 break;
194 }
195 }
196 }
197
bnxt_re_ring_rq_db(struct bnxt_re_qp * qp)198 void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp)
199 {
200 struct bnxt_re_db_hdr hdr;
201
202 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
203 return;
204 bnxt_re_init_db_hdr(&hdr,
205 (*qp->jrqq->hwque->dbtail |
206 ((qp->jrqq->hwque->flags &
207 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
208 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
209 qp->qpid,
210 BNXT_RE_QUE_TYPE_RQ);
211 bnxt_re_ring_db(qp->udpi, hdr.typ_qid_indx, &qp->rq_shadow_db_key,
212 &qp->rq_dbr_lock);
213 }
214
bnxt_re_ring_sq_db(struct bnxt_re_qp * qp)215 void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp)
216 {
217 struct bnxt_re_db_hdr hdr;
218
219 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
220 return;
221 bnxt_re_init_db_hdr(&hdr,
222 (*qp->jsqq->hwque->dbtail |
223 ((qp->jsqq->hwque->flags &
224 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
225 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
226 qp->qpid,
227 BNXT_RE_QUE_TYPE_SQ);
228 bnxt_re_ring_db(qp->udpi, hdr.typ_qid_indx, &qp->sq_shadow_db_key,
229 &qp->sq_dbr_lock);
230 }
231
bnxt_re_ring_srq_db(struct bnxt_re_srq * srq)232 void bnxt_re_ring_srq_db(struct bnxt_re_srq *srq)
233 {
234 struct bnxt_re_db_hdr hdr;
235
236 if (bnxt_re_do_pacing(srq->uctx, &srq->rand))
237 return;
238 bnxt_re_init_db_hdr(&hdr,
239 (srq->srqq->tail |
240 ((srq->srqq->flags &
241 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
242 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
243 srq->srqid, BNXT_RE_QUE_TYPE_SRQ);
244 bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key,
245 &srq->dbr_lock);
246 }
247
bnxt_re_ring_srq_arm(struct bnxt_re_srq * srq)248 void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq)
249 {
250 struct bnxt_re_db_hdr hdr;
251 uint32_t toggle = 0;
252
253 if (srq->srq_page)
254 toggle = *(uint32_t *)srq->srq_page;
255
256 if (bnxt_re_do_pacing(srq->uctx, &srq->rand))
257 return;
258 bnxt_re_init_db_hdr(&hdr, srq->cap.srq_limit,
259 toggle << BNXT_RE_DB_TOGGLE_SHIFT, srq->srqid,
260 BNXT_RE_QUE_TYPE_SRQ_ARM);
261 bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key,
262 &srq->dbr_lock);
263 }
264
bnxt_re_ring_cq_db(struct bnxt_re_cq * cq)265 void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq)
266 {
267 struct bnxt_re_db_hdr hdr;
268
269 if (bnxt_re_do_pacing(cq->cntx, &cq->rand))
270 return;
271 bnxt_re_init_db_hdr(&hdr,
272 (cq->cqq->head |
273 ((cq->cqq->flags &
274 BNXT_RE_FLAG_EPOCH_HEAD_MASK) <<
275 BNXT_RE_DB_EPOCH_HEAD_SHIFT)), 0,
276 cq->cqid,
277 BNXT_RE_QUE_TYPE_CQ);
278 bnxt_re_ring_db(cq->udpi, hdr.typ_qid_indx, &cq->shadow_db_key,
279 &cq->dbr_lock);
280 }
281
bnxt_re_ring_cq_arm_db(struct bnxt_re_cq * cq,uint8_t aflag)282 void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag)
283 {
284 uint32_t *cq_page = cq->cq_page;
285 struct bnxt_re_db_hdr hdr;
286 uint32_t toggle = 0;
287
288 if (cq_page)
289 toggle = *cq_page;
290
291 if (bnxt_re_do_pacing(cq->cntx, &cq->rand))
292 return;
293 bnxt_re_init_db_hdr(&hdr,
294 (cq->cqq->head |
295 ((cq->cqq->flags &
296 BNXT_RE_FLAG_EPOCH_HEAD_MASK) <<
297 BNXT_RE_DB_EPOCH_HEAD_SHIFT)),
298 toggle << BNXT_RE_DB_TOGGLE_SHIFT,
299 cq->cqid, aflag);
300 bnxt_re_ring_db(cq->udpi, hdr.typ_qid_indx, &cq->shadow_db_key,
301 &cq->dbr_lock);
302 }
303
bnxt_re_ring_pstart_db(struct bnxt_re_qp * qp,struct bnxt_re_push_buffer * pbuf)304 void bnxt_re_ring_pstart_db(struct bnxt_re_qp *qp,
305 struct bnxt_re_push_buffer *pbuf)
306 {
307 __u64 key;
308
309 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
310 return;
311 key = ((((pbuf->wcdpi & BNXT_RE_DB_PIHI_MASK) <<
312 BNXT_RE_DB_PIHI_SHIFT) | (pbuf->qpid & BNXT_RE_DB_QID_MASK)) |
313 ((BNXT_RE_PUSH_TYPE_START & BNXT_RE_DB_TYP_MASK) <<
314 BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT));
315 key <<= 32;
316 key |= ((((__u32)pbuf->wcdpi & BNXT_RE_DB_PILO_MASK) <<
317 BNXT_RE_DB_PILO_SHIFT) | (pbuf->st_idx &
318 BNXT_RE_DB_INDX_MASK));
319 bnxt_re_wm_barrier();
320 iowrite64(pbuf->ucdb, key);
321 }
322
bnxt_re_ring_pend_db(struct bnxt_re_qp * qp,struct bnxt_re_push_buffer * pbuf)323 void bnxt_re_ring_pend_db(struct bnxt_re_qp *qp,
324 struct bnxt_re_push_buffer *pbuf)
325 {
326 __u64 key;
327
328 if (bnxt_re_do_pacing(qp->cntx, &qp->rand))
329 return;
330 key = ((((pbuf->wcdpi & BNXT_RE_DB_PIHI_MASK) <<
331 BNXT_RE_DB_PIHI_SHIFT) | (pbuf->qpid & BNXT_RE_DB_QID_MASK)) |
332 ((BNXT_RE_PUSH_TYPE_END & BNXT_RE_DB_TYP_MASK) <<
333 BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT));
334 key <<= 32;
335 key |= ((((__u32)pbuf->wcdpi & BNXT_RE_DB_PILO_MASK) <<
336 BNXT_RE_DB_PILO_SHIFT) | (pbuf->tail &
337 BNXT_RE_DB_INDX_MASK));
338 __bnxt_re_ring_pend_db(pbuf->ucdb, key, qp);
339 }
340
bnxt_re_fill_ppp(struct bnxt_re_push_buffer * pbuf,struct bnxt_re_qp * qp,uint8_t len,uint32_t idx)341 void bnxt_re_fill_ppp(struct bnxt_re_push_buffer *pbuf,
342 struct bnxt_re_qp *qp, uint8_t len, uint32_t idx)
343 {
344 struct bnxt_re_db_ppp_hdr phdr = {};
345 __u64 *dst, *src;
346 __u8 plen;
347 int indx;
348
349 src = (__u64 *)&phdr;
350 plen = len + sizeof(phdr) + bnxt_re_get_sqe_hdr_sz();
351
352 bnxt_re_init_db_hdr(&phdr.db_hdr,
353 (*qp->jsqq->hwque->dbtail |
354 ((qp->jsqq->hwque->flags &
355 BNXT_RE_FLAG_EPOCH_TAIL_MASK) <<
356 BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0,
357 qp->qpid,
358 BNXT_RE_QUE_TYPE_SQ);
359
360 phdr.rsv_psz_pidx = ((pbuf->st_idx & BNXT_RE_DB_INDX_MASK) |
361 (((plen % 8 ? (plen / 8) + 1 :
362 plen / 8) & BNXT_RE_PUSH_SIZE_MASK) <<
363 BNXT_RE_PUSH_SIZE_SHIFT));
364
365 bnxt_re_wm_barrier();
366 for (indx = 0; indx < 2; indx++) {
367 dst = (__u64 *)(pbuf->pbuf + indx);
368 iowrite64(dst, *src);
369 src++;
370 }
371 bnxt_re_copy_data_to_pb(pbuf, 1, idx);
372 mmio_flush_writes();
373 }
374
bnxt_re_fill_push_wcb(struct bnxt_re_qp * qp,struct bnxt_re_push_buffer * pbuf,uint32_t idx)375 void bnxt_re_fill_push_wcb(struct bnxt_re_qp *qp,
376 struct bnxt_re_push_buffer *pbuf, uint32_t idx)
377 {
378 bnxt_re_ring_pstart_db(qp, pbuf);
379 mmio_wc_start();
380 bnxt_re_copy_data_to_pb(pbuf, 0, idx);
381 /* Flush WQE write before push end db. */
382 mmio_flush_writes();
383 bnxt_re_ring_pend_db(qp, pbuf);
384 }
385
bnxt_re_init_pbuf_list(struct bnxt_re_context * ucntx)386 int bnxt_re_init_pbuf_list(struct bnxt_re_context *ucntx)
387 {
388 struct bnxt_re_push_buffer *pbuf;
389 int indx, wqesz;
390 int size, offt;
391 __u64 wcpage;
392 __u64 dbpage;
393 void *base;
394
395 size = (sizeof(*ucntx->pbrec) +
396 16 * (sizeof(*ucntx->pbrec->pbuf) +
397 sizeof(struct bnxt_re_push_wqe)));
398 ucntx->pbrec = calloc(1, size);
399 if (!ucntx->pbrec)
400 goto out;
401
402 offt = sizeof(*ucntx->pbrec);
403 base = ucntx->pbrec;
404 ucntx->pbrec->pbuf = (base + offt);
405 ucntx->pbrec->pbmap = ~0x00;
406 ucntx->pbrec->pbmap &= ~0x7fff; /* 15 bits */
407 ucntx->pbrec->udpi = &ucntx->udpi;
408
409 wqesz = sizeof(struct bnxt_re_push_wqe);
410 wcpage = (__u64)ucntx->udpi.wcdbpg;
411 dbpage = (__u64)ucntx->udpi.dbpage;
412 offt = sizeof(*ucntx->pbrec->pbuf) * 16;
413 base = (char *)ucntx->pbrec->pbuf + offt;
414 for (indx = 0; indx < 16; indx++) {
415 pbuf = &ucntx->pbrec->pbuf[indx];
416 pbuf->wqe = base + indx * wqesz;
417 pbuf->pbuf = (__u64 *)(wcpage + indx * wqesz);
418 pbuf->ucdb = (__u64 *)(dbpage + (indx + 1) * sizeof(__u64));
419 pbuf->wcdpi = ucntx->udpi.wcdpi;
420 }
421
422 return 0;
423 out:
424 return -ENOMEM;
425 }
426
bnxt_re_get_pbuf(uint8_t * push_st_en,uint8_t ppp_idx,struct bnxt_re_context * cntx)427 struct bnxt_re_push_buffer *bnxt_re_get_pbuf(uint8_t *push_st_en,
428 uint8_t ppp_idx,
429 struct bnxt_re_context *cntx)
430 {
431 struct bnxt_re_push_buffer *pbuf = NULL;
432 uint8_t buf_state = 0;
433 __u32 old;
434 int bit;
435
436 if (_is_chip_thor2(cntx->cctx)) {
437 buf_state = !!(*push_st_en & BNXT_RE_PPP_STATE_MASK);
438 pbuf = &cntx->pbrec->pbuf[(ppp_idx * 2) + buf_state];
439 /* Flip */
440 *push_st_en ^= 1UL << BNXT_RE_PPP_ST_SHIFT;
441 } else {
442 old = cntx->pbrec->pbmap;
443 while ((bit = __builtin_ffs(~cntx->pbrec->pbmap)) != 0) {
444 if (__sync_bool_compare_and_swap
445 (&cntx->pbrec->pbmap,
446 old,
447 (old | 0x01 << (bit - 1))))
448 break;
449 old = cntx->pbrec->pbmap;
450 }
451
452 if (bit) {
453 pbuf = &cntx->pbrec->pbuf[bit];
454 pbuf->nbit = bit;
455 }
456 }
457
458 return pbuf;
459 }
460
bnxt_re_put_pbuf(struct bnxt_re_context * cntx,struct bnxt_re_push_buffer * pbuf)461 void bnxt_re_put_pbuf(struct bnxt_re_context *cntx,
462 struct bnxt_re_push_buffer *pbuf)
463 {
464 struct bnxt_re_push_rec *pbrec;
465 __u32 old;
466 int bit;
467
468 if (_is_chip_thor2(cntx->cctx))
469 return;
470
471 pbrec = cntx->pbrec;
472
473 if (pbuf->nbit) {
474 bit = pbuf->nbit;
475 pbuf->nbit = 0;
476 old = pbrec->pbmap;
477 while (!__sync_bool_compare_and_swap(&pbrec->pbmap, old,
478 (old & (~(0x01 <<
479 (bit - 1))))))
480 old = pbrec->pbmap;
481 }
482 }
483
bnxt_re_destroy_pbuf_list(struct bnxt_re_context * cntx)484 void bnxt_re_destroy_pbuf_list(struct bnxt_re_context *cntx)
485 {
486 free(cntx->pbrec);
487 }
488
bnxt_re_replay_db(struct bnxt_re_context * cntx,struct xorshift32_state * state,struct bnxt_re_dpi * dpi,uint64_t * shadow_key,uint8_t * dbr_lock)489 void bnxt_re_replay_db(struct bnxt_re_context *cntx,
490 struct xorshift32_state *state, struct bnxt_re_dpi *dpi,
491 uint64_t *shadow_key, uint8_t *dbr_lock)
492 {
493 if (bnxt_re_do_pacing(cntx, state))
494 return;
495 cntx->replay_cnt++;
496 if (cntx->replay_cnt % BNXT_RE_DB_REPLAY_YIELD_CNT == 0)
497 pthread_yield();
498 if (__sync_bool_compare_and_swap(dbr_lock, 0, 1)) {
499 bnxt_re_wm_barrier();
500 if (*shadow_key == BNXT_RE_DB_KEY_INVALID) {
501 *dbr_lock = 0;
502 return;
503 }
504 iowrite64(dpi->dbpage, *shadow_key);
505 bnxt_re_wm_barrier();
506 *dbr_lock = 0;
507 }
508 }
509
bnxt_re_db_recovery(struct bnxt_re_context * cntx)510 void bnxt_re_db_recovery(struct bnxt_re_context *cntx)
511 {
512 struct bnxt_re_list_node *cur, *tmp;
513 struct bnxt_re_qp *qp;
514 struct bnxt_re_cq *cq;
515 struct bnxt_re_srq *srq;
516
517 pthread_spin_lock(&cntx->qp_dbr_res.lock);
518 list_for_each_node_safe(cur, tmp, &cntx->qp_dbr_res.head) {
519 qp = list_node(cur, struct bnxt_re_qp, dbnode);
520 bnxt_re_replay_db(cntx, &qp->rand, qp->udpi,
521 &qp->sq_shadow_db_key, &qp->sq_dbr_lock);
522 bnxt_re_replay_db(cntx, &qp->rand, qp->udpi,
523 &qp->rq_shadow_db_key, &qp->rq_dbr_lock);
524 }
525 pthread_spin_unlock(&cntx->qp_dbr_res.lock);
526 pthread_spin_lock(&cntx->cq_dbr_res.lock);
527 list_for_each_node_safe(cur, tmp, &cntx->cq_dbr_res.head) {
528 cq = list_node(cur, struct bnxt_re_cq, dbnode);
529 bnxt_re_replay_db(cntx, &cq->rand, cq->udpi,
530 &cq->shadow_db_key, &cq->dbr_lock);
531 }
532 pthread_spin_unlock(&cntx->cq_dbr_res.lock);
533 pthread_spin_lock(&cntx->srq_dbr_res.lock);
534 list_for_each_node_safe(cur, tmp, &cntx->srq_dbr_res.head) {
535 srq = list_node(cur, struct bnxt_re_srq, dbnode);
536 bnxt_re_replay_db(cntx, &srq->rand, srq->udpi,
537 &srq->shadow_db_key, &srq->dbr_lock);
538 }
539 pthread_spin_unlock(&cntx->srq_dbr_res.lock);
540 }
541
bnxt_re_dbr_thread(void * arg)542 void *bnxt_re_dbr_thread(void *arg)
543 {
544 uint32_t *epoch, *epoch_ack, usr_epoch;
545 struct bnxt_re_context *cntx = arg;
546 struct ibv_cq *ev_cq;
547 void *ev_ctx;
548 int ret;
549
550 while (1) {
551 ret = ibv_get_cq_event(cntx->dbr_ev_chan, &ev_cq, &ev_ctx);
552 if (ret) {
553 fprintf(stderr, "Failed to get cq_event\n");
554 pthread_exit(NULL);
555 }
556 epoch = cntx->db_recovery_page;
557 epoch_ack = epoch + 1;
558 if (!epoch || !epoch_ack) {
559 fprintf(stderr, "DB reovery page is NULL\n");
560 pthread_exit(NULL);
561 }
562 if (*epoch == *epoch_ack) {
563 ibv_ack_cq_events(ev_cq, 1);
564 continue;
565 }
566 usr_epoch = *epoch;
567 bnxt_re_db_recovery(cntx);
568 *epoch_ack = usr_epoch;
569 ibv_ack_cq_events(ev_cq, 1);
570 }
571 }
572