Lines Matching refs:iq
42 mlx5e_iq_poll(struct mlx5e_iq *iq, int budget) in mlx5e_iq_poll() argument
52 iqcc = iq->cc; in mlx5e_iq_poll()
56 cqe = mlx5e_get_cqe(&iq->cq); in mlx5e_iq_poll()
60 mlx5_cqwq_pop(&iq->cq.wq); in mlx5e_iq_poll()
62 ci = iqcc & iq->wq.sz_m1; in mlx5e_iq_poll()
64 if (likely(iq->data[ci].dma_sync != 0)) { in mlx5e_iq_poll()
66 bus_dmamap_sync(iq->dma_tag, iq->data[ci].dma_map, iq->data[ci].dma_sync); in mlx5e_iq_poll()
67 bus_dmamap_unload(iq->dma_tag, iq->data[ci].dma_map); in mlx5e_iq_poll()
69 iq->data[ci].dma_sync = 0; in mlx5e_iq_poll()
72 if (likely(iq->data[ci].callback != NULL)) { in mlx5e_iq_poll()
73 iq->data[ci].callback(iq->data[ci].arg); in mlx5e_iq_poll()
74 iq->data[ci].callback = NULL; in mlx5e_iq_poll()
77 if (unlikely(iq->data[ci].p_refcount != NULL)) { in mlx5e_iq_poll()
78 atomic_add_int(iq->data[ci].p_refcount, -1); in mlx5e_iq_poll()
79 iq->data[ci].p_refcount = NULL; in mlx5e_iq_poll()
81 iqcc += iq->data[ci].num_wqebbs; in mlx5e_iq_poll()
84 mlx5_cqwq_update_db_record(&iq->cq.wq); in mlx5e_iq_poll()
89 iq->cc = iqcc; in mlx5e_iq_poll()
95 struct mlx5e_iq *iq = container_of(mcq, struct mlx5e_iq, cq.mcq); in mlx5e_iq_completion() local
97 mtx_lock(&iq->comp_lock); in mlx5e_iq_completion()
98 mlx5e_iq_poll(iq, MLX5E_BUDGET_MAX); in mlx5e_iq_completion()
99 mlx5e_cq_arm(&iq->cq, MLX5_GET_DOORBELL_LOCK(&iq->priv->doorbell_lock)); in mlx5e_iq_completion()
100 mtx_unlock(&iq->comp_lock); in mlx5e_iq_completion()
104 mlx5e_iq_send_nop(struct mlx5e_iq *iq, u32 ds_cnt) in mlx5e_iq_send_nop() argument
106 u16 pi = iq->pc & iq->wq.sz_m1; in mlx5e_iq_send_nop()
107 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi); in mlx5e_iq_send_nop()
109 mtx_assert(&iq->lock, MA_OWNED); in mlx5e_iq_send_nop()
113 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) | MLX5_OPCODE_NOP); in mlx5e_iq_send_nop()
114 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt); in mlx5e_iq_send_nop()
118 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32)); in mlx5e_iq_send_nop()
120 iq->data[pi].callback = NULL; in mlx5e_iq_send_nop()
121 iq->data[pi].arg = NULL; in mlx5e_iq_send_nop()
122 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); in mlx5e_iq_send_nop()
123 iq->data[pi].dma_sync = 0; in mlx5e_iq_send_nop()
124 iq->pc += iq->data[pi].num_wqebbs; in mlx5e_iq_send_nop()
128 mlx5e_iq_free_db(struct mlx5e_iq *iq) in mlx5e_iq_free_db() argument
130 int wq_sz = mlx5_wq_cyc_get_size(&iq->wq); in mlx5e_iq_free_db()
134 if (likely(iq->data[x].dma_sync != 0)) { in mlx5e_iq_free_db()
135 bus_dmamap_unload(iq->dma_tag, iq->data[x].dma_map); in mlx5e_iq_free_db()
136 iq->data[x].dma_sync = 0; in mlx5e_iq_free_db()
138 if (likely(iq->data[x].callback != NULL)) { in mlx5e_iq_free_db()
139 iq->data[x].callback(iq->data[x].arg); in mlx5e_iq_free_db()
140 iq->data[x].callback = NULL; in mlx5e_iq_free_db()
142 if (unlikely(iq->data[x].p_refcount != NULL)) { in mlx5e_iq_free_db()
143 atomic_add_int(iq->data[x].p_refcount, -1); in mlx5e_iq_free_db()
144 iq->data[x].p_refcount = NULL; in mlx5e_iq_free_db()
146 bus_dmamap_destroy(iq->dma_tag, iq->data[x].dma_map); in mlx5e_iq_free_db()
148 free(iq->data, M_MLX5EN); in mlx5e_iq_free_db()
152 mlx5e_iq_alloc_db(struct mlx5e_iq *iq) in mlx5e_iq_alloc_db() argument
154 int wq_sz = mlx5_wq_cyc_get_size(&iq->wq); in mlx5e_iq_alloc_db()
158 iq->data = malloc_domainset(wq_sz * sizeof(iq->data[0]), M_MLX5EN, in mlx5e_iq_alloc_db()
159 mlx5_dev_domainset(iq->priv->mdev), M_WAITOK | M_ZERO); in mlx5e_iq_alloc_db()
163 err = -bus_dmamap_create(iq->dma_tag, 0, &iq->data[x].dma_map); in mlx5e_iq_alloc_db()
166 bus_dmamap_destroy(iq->dma_tag, iq->data[x].dma_map); in mlx5e_iq_alloc_db()
167 free(iq->data, M_MLX5EN); in mlx5e_iq_alloc_db()
177 struct mlx5e_iq *iq) in mlx5e_iq_create() argument
198 &iq->dma_tag))) in mlx5e_iq_create()
201 iq->mkey_be = cpu_to_be32(priv->mr.key); in mlx5e_iq_create()
202 iq->priv = priv; in mlx5e_iq_create()
205 &iq->wq, &iq->wq_ctrl); in mlx5e_iq_create()
209 iq->wq.db = &iq->wq.db[MLX5_SND_DBR]; in mlx5e_iq_create()
211 err = mlx5e_iq_alloc_db(iq); in mlx5e_iq_create()
218 mlx5_wq_destroy(&iq->wq_ctrl); in mlx5e_iq_create()
221 bus_dma_tag_destroy(iq->dma_tag); in mlx5e_iq_create()
227 mlx5e_iq_destroy(struct mlx5e_iq *iq) in mlx5e_iq_destroy() argument
229 mlx5e_iq_free_db(iq); in mlx5e_iq_destroy()
230 mlx5_wq_destroy(&iq->wq_ctrl); in mlx5e_iq_destroy()
231 bus_dma_tag_destroy(iq->dma_tag); in mlx5e_iq_destroy()
235 mlx5e_iq_enable(struct mlx5e_iq *iq, struct mlx5e_sq_param *param, in mlx5e_iq_enable() argument
246 sizeof(u64) * iq->wq_ctrl.buf.npages; in mlx5e_iq_enable()
251 iq->uar_map = bfreg->map; in mlx5e_iq_enable()
253 ts_format = mlx5_get_sq_default_ts(iq->priv->mdev); in mlx5e_iq_enable()
260 MLX5_SET(sqc, sqc, cqn, iq->cq.mcq.cqn); in mlx5e_iq_enable()
268 if (MLX5_CAP_QOS(iq->priv->mdev, qos_remap_pp)) { in mlx5e_iq_enable()
270 if (MLX5_CAP_ETH(iq->priv->mdev, reg_umr_sq)) in mlx5e_iq_enable()
273 mlx5_en_err(iq->priv->ifp, in mlx5e_iq_enable()
279 MLX5_SET(wq, wq, log_wq_pg_sz, iq->wq_ctrl.buf.page_shift - in mlx5e_iq_enable()
281 MLX5_SET64(wq, wq, dbr_addr, iq->wq_ctrl.db.dma); in mlx5e_iq_enable()
283 mlx5_fill_page_array(&iq->wq_ctrl.buf, in mlx5e_iq_enable()
286 err = mlx5_core_create_sq(iq->priv->mdev, in, inlen, &iq->sqn); in mlx5e_iq_enable()
294 mlx5e_iq_modify(struct mlx5e_iq *iq, int curr_state, int next_state) in mlx5e_iq_modify() argument
308 MLX5_SET(modify_sq_in, in, sqn, iq->sqn); in mlx5e_iq_modify()
312 err = mlx5_core_modify_sq(iq->priv->mdev, in, inlen); in mlx5e_iq_modify()
320 mlx5e_iq_disable(struct mlx5e_iq *iq) in mlx5e_iq_disable() argument
322 mlx5_core_destroy_sq(iq->priv->mdev, iq->sqn); in mlx5e_iq_disable()
329 struct mlx5e_iq *iq) in mlx5e_iq_open() argument
333 err = mlx5e_open_cq(c->priv, cq_param, &iq->cq, in mlx5e_iq_open()
338 err = mlx5e_iq_create(c, sq_param, iq); in mlx5e_iq_open()
342 err = mlx5e_iq_enable(iq, sq_param, &c->bfreg, c->priv->tisn[0]); in mlx5e_iq_open()
346 err = mlx5e_iq_modify(iq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY); in mlx5e_iq_open()
350 WRITE_ONCE(iq->running, 1); in mlx5e_iq_open()
355 mlx5e_iq_disable(iq); in mlx5e_iq_open()
357 mlx5e_iq_destroy(iq); in mlx5e_iq_open()
359 mlx5e_close_cq(&iq->cq); in mlx5e_iq_open()
365 mlx5e_iq_drain(struct mlx5e_iq *iq) in mlx5e_iq_drain() argument
367 struct mlx5_core_dev *mdev = iq->priv->mdev; in mlx5e_iq_drain()
378 if (READ_ONCE(iq->running) == 0) in mlx5e_iq_drain()
382 WRITE_ONCE(iq->running, 0); in mlx5e_iq_drain()
385 mtx_lock(&iq->lock); in mlx5e_iq_drain()
386 while (iq->cc != iq->pc && in mlx5e_iq_drain()
387 (iq->priv->media_status_last & IFM_ACTIVE) != 0 && in mlx5e_iq_drain()
390 mtx_unlock(&iq->lock); in mlx5e_iq_drain()
392 iq->cq.mcq.comp(&iq->cq.mcq, NULL); in mlx5e_iq_drain()
393 mtx_lock(&iq->lock); in mlx5e_iq_drain()
395 mtx_unlock(&iq->lock); in mlx5e_iq_drain()
398 (void) mlx5e_iq_modify(iq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR); in mlx5e_iq_drain()
401 mtx_lock(&iq->lock); in mlx5e_iq_drain()
402 while (iq->cc != iq->pc && in mlx5e_iq_drain()
405 mtx_unlock(&iq->lock); in mlx5e_iq_drain()
407 iq->cq.mcq.comp(&iq->cq.mcq, NULL); in mlx5e_iq_drain()
408 mtx_lock(&iq->lock); in mlx5e_iq_drain()
410 mtx_unlock(&iq->lock); in mlx5e_iq_drain()
414 mlx5e_iq_close(struct mlx5e_iq *iq) in mlx5e_iq_close() argument
416 mlx5e_iq_drain(iq); in mlx5e_iq_close()
417 mlx5e_iq_disable(iq); in mlx5e_iq_close()
418 mlx5e_iq_destroy(iq); in mlx5e_iq_close()
419 mlx5e_close_cq(&iq->cq); in mlx5e_iq_close()
423 mlx5e_iq_static_init(struct mlx5e_iq *iq) in mlx5e_iq_static_init() argument
425 mtx_init(&iq->lock, "mlx5iq", in mlx5e_iq_static_init()
427 mtx_init(&iq->comp_lock, "mlx5iq_comp", in mlx5e_iq_static_init()
432 mlx5e_iq_static_destroy(struct mlx5e_iq *iq) in mlx5e_iq_static_destroy() argument
434 mtx_destroy(&iq->lock); in mlx5e_iq_static_destroy()
435 mtx_destroy(&iq->comp_lock); in mlx5e_iq_static_destroy()
439 mlx5e_iq_notify_hw(struct mlx5e_iq *iq) in mlx5e_iq_notify_hw() argument
441 mtx_assert(&iq->lock, MA_OWNED); in mlx5e_iq_notify_hw()
444 if (unlikely(iq->db_inhibit != 0 || iq->doorbell.d64 == 0)) in mlx5e_iq_notify_hw()
450 *iq->wq.db = cpu_to_be32(iq->pc); in mlx5e_iq_notify_hw()
458 mlx5_write64(iq->doorbell.d32, iq->uar_map, in mlx5e_iq_notify_hw()
459 MLX5_GET_DOORBELL_LOCK(&iq->priv->doorbell_lock)); in mlx5e_iq_notify_hw()
461 iq->doorbell.d64 = 0; in mlx5e_iq_notify_hw()
465 mlx5e_iq_has_room_for(struct mlx5e_iq *iq, u16 n) in mlx5e_iq_has_room_for() argument
467 u16 cc = iq->cc; in mlx5e_iq_has_room_for()
468 u16 pc = iq->pc; in mlx5e_iq_has_room_for()
470 return ((iq->wq.sz_m1 & (cc - pc)) >= n || cc == pc); in mlx5e_iq_has_room_for()
474 mlx5e_iq_get_producer_index(struct mlx5e_iq *iq) in mlx5e_iq_get_producer_index() argument
478 mtx_assert(&iq->lock, MA_OWNED); in mlx5e_iq_get_producer_index()
480 if (unlikely(iq->running == 0)) in mlx5e_iq_get_producer_index()
482 if (unlikely(!mlx5e_iq_has_room_for(iq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) in mlx5e_iq_get_producer_index()
486 pi = ((~iq->pc) & iq->wq.sz_m1); in mlx5e_iq_get_producer_index()
489 mlx5e_iq_send_nop(iq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS); in mlx5e_iq_get_producer_index()
490 pi = ((~iq->pc) & iq->wq.sz_m1); in mlx5e_iq_get_producer_index()
494 return (iq->pc & iq->wq.sz_m1); in mlx5e_iq_get_producer_index()
513 mlx5e_iq_load_memory_single(struct mlx5e_iq *iq, u16 pi, void *buffer, size_t size, in mlx5e_iq_load_memory_single() argument
518 error = bus_dmamap_load(iq->dma_tag, iq->data[pi].dma_map, buffer, size, in mlx5e_iq_load_memory_single()
525 iq->data[pi].dma_sync = BUS_DMASYNC_POSTREAD; in mlx5e_iq_load_memory_single()
528 iq->data[pi].dma_sync = BUS_DMASYNC_POSTWRITE; in mlx5e_iq_load_memory_single()
535 bus_dmamap_sync(iq->dma_tag, iq->data[pi].dma_map, dma_sync); in mlx5e_iq_load_memory_single()