99 	mlx5e_rl_build_sq_param(rl, &cparam->sq);  in mlx5e_rl_build_channel_param()
104 mlx5e_rl_create_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,  in mlx5e_rl_create_sq()  argument
125 	    &sq->dma_tag)))  in mlx5e_rl_create_sq()
128 	sq->mkey_be = cpu_to_be32(priv->mr.key);  in mlx5e_rl_create_sq()
129 	sq->ifp = priv->ifp;  in mlx5e_rl_create_sq()
130 	sq->priv = priv;  in mlx5e_rl_create_sq()
132 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,  in mlx5e_rl_create_sq()
133 	    &sq->wq_ctrl);  in mlx5e_rl_create_sq()
137 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];  in mlx5e_rl_create_sq()
139 	err = mlx5e_alloc_sq_db(sq);  in mlx5e_rl_create_sq()
143 	mlx5e_update_sq_inline(sq);  in mlx5e_rl_create_sq()
148 	mlx5_wq_destroy(&sq->wq_ctrl);  in mlx5e_rl_create_sq()
150 	bus_dma_tag_destroy(sq->dma_tag);  in mlx5e_rl_create_sq()
156 mlx5e_rl_destroy_sq(struct mlx5e_sq *sq)  in mlx5e_rl_destroy_sq()  argument
159 	mlx5e_free_sq_db(sq);  in mlx5e_rl_destroy_sq()
160 	mlx5_wq_destroy(&sq->wq_ctrl);  in mlx5e_rl_destroy_sq()
161 	bus_dma_tag_destroy(sq->dma_tag);  in mlx5e_rl_destroy_sq()
165 mlx5e_rl_query_sq(struct mlx5e_sq *sq)  in mlx5e_rl_query_sq()  argument
176         err = mlx5_core_query_sq(sq->priv->mdev, sq->sqn, out);  in mlx5e_rl_query_sq()
180         sq->queue_handle = MLX5_GET(query_sq_out, out, sq_context.queue_handle);  in mlx5e_rl_query_sq()
188 mlx5e_rl_open_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,  in mlx5e_rl_open_sq()  argument
193 	err = mlx5e_rl_create_sq(priv, sq, param, ix);  in mlx5e_rl_open_sq()
197 	err = mlx5e_enable_sq(sq, param, &priv->channel[ix].bfreg, priv->rl.tisn);  in mlx5e_rl_open_sq()
201 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);  in mlx5e_rl_open_sq()
206 		err = mlx5e_rl_query_sq(sq);  in mlx5e_rl_open_sq()
209 			    "SQ remap - sqn=%u, err=(%d)\n", sq->sqn, err);  in mlx5e_rl_open_sq()
210 			sq->queue_handle = MLX5_INVALID_QUEUE_HANDLE;  in mlx5e_rl_open_sq()
213 		sq->queue_handle = MLX5_INVALID_QUEUE_HANDLE;  in mlx5e_rl_open_sq()
215 	WRITE_ONCE(sq->running, 1);  in mlx5e_rl_open_sq()
220 	mlx5e_disable_sq(sq);  in mlx5e_rl_open_sq()
222 	mlx5e_rl_destroy_sq(sq);  in mlx5e_rl_open_sq()
228 mlx5e_rl_chan_mtx_init(struct mlx5e_priv *priv, struct mlx5e_sq *sq)  in mlx5e_rl_chan_mtx_init()  argument
230 	mtx_init(&sq->lock, "mlx5tx-rl", NULL, MTX_DEF);  in mlx5e_rl_chan_mtx_init()
231 	mtx_init(&sq->comp_lock, "mlx5comp-rl", NULL, MTX_DEF);  in mlx5e_rl_chan_mtx_init()
233 	callout_init_mtx(&sq->cev_callout, &sq->lock, 0);  in mlx5e_rl_chan_mtx_init()
235 	sq->cev_factor = priv->rl.param.tx_completion_fact;  in mlx5e_rl_chan_mtx_init()
238 	if (sq->cev_factor == 0)  in mlx5e_rl_chan_mtx_init()
239 		sq->cev_factor = 1;  in mlx5e_rl_chan_mtx_init()
248 	struct mlx5e_sq *sq;  in mlx5e_rl_open_channel()  local
251 	sq = malloc(sizeof(*sq), M_MLX5EN, M_WAITOK | M_ZERO);  in mlx5e_rl_open_channel()
254 	mlx5e_rl_chan_mtx_init(priv, sq);  in mlx5e_rl_open_channel()
257 	err = mlx5e_open_cq(priv, &cparam->cq, &sq->cq,  in mlx5e_rl_open_channel()
262 	err = mlx5e_rl_open_sq(priv, sq, &cparam->sq, eq_ix);  in mlx5e_rl_open_channel()
267 	*ppsq = sq;  in mlx5e_rl_open_channel()
270 	sq->cq.mcq.comp(&sq->cq.mcq, NULL);  in mlx5e_rl_open_channel()
275 	mlx5e_close_cq(&sq->cq);  in mlx5e_rl_open_channel()
279 	mtx_destroy(&sq->lock);  in mlx5e_rl_open_channel()
280 	mtx_destroy(&sq->comp_lock);  in mlx5e_rl_open_channel()
281 	free(sq, M_MLX5EN);  in mlx5e_rl_open_channel()
289 	struct mlx5e_sq *sq = *ppsq;  in mlx5e_rl_close_channel()  local
292 	if (sq == NULL)  in mlx5e_rl_close_channel()
298 	mlx5e_drain_sq(sq);  in mlx5e_rl_close_channel()
299 	mlx5e_disable_sq(sq);  in mlx5e_rl_close_channel()
300 	mlx5e_rl_destroy_sq(sq);  in mlx5e_rl_close_channel()
303 	mlx5e_close_cq(&sq->cq);  in mlx5e_rl_close_channel()
306 	mtx_destroy(&sq->lock);  in mlx5e_rl_close_channel()
307 	mtx_destroy(&sq->comp_lock);  in mlx5e_rl_close_channel()
309 	free(sq, M_MLX5EN);  in mlx5e_rl_close_channel()
349 mlx5e_rl_modify_sq(struct mlx5e_sq *sq, uint16_t rl_index)  in mlx5e_rl_modify_sq()  argument
351 	struct mlx5e_priv *priv = sq->priv;  in mlx5e_rl_modify_sq()
366 	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);  in mlx5e_rl_modify_sq()
468 mlx5e_rl_remap_sq(struct mlx5e_sq *sq, uint16_t index,  in mlx5e_rl_remap_sq()  argument
477 	iq_channel = &sq->priv->channel[sq->sqn % sq->priv->params.num_channels];  in mlx5e_rl_remap_sq()
479 	sq_handle = sq->queue_handle;  in mlx5e_rl_remap_sq()
480 	scq_handle = mlx5_rl_get_scq_handle(sq->priv->mdev, index);  in mlx5e_rl_remap_sq()
502 	struct mlx5e_sq *sq;  in mlx5e_rlw_channel_set_rate_locked()  local
573 	sq = channel->sq;  in mlx5e_rlw_channel_set_rate_locked()
574 	if (sq != NULL && READ_ONCE(sq->running) != 0) {  in mlx5e_rlw_channel_set_rate_locked()
575 		if (!use_sq_remap || mlx5e_rl_remap_sq(sq, index, channel)) {  in mlx5e_rlw_channel_set_rate_locked()
580 			error = mlx5e_rl_modify_sq(sq, index);  in mlx5e_rlw_channel_set_rate_locked()
631 		    &priv->rl.chan_param, &channel->sq);  in mlx5e_rl_worker()
662 				if (channel->sq == NULL) {  in mlx5e_rl_worker()
665 					    &priv->rl.chan_param, &channel->sq);  in mlx5e_rl_worker()
675 					mlx5e_resume_sq(channel->sq);  in mlx5e_rl_worker()
696 				if (channel->sq != NULL) {  in mlx5e_rl_worker()
703 					mlx5e_drain_sq(channel->sq);  in mlx5e_rl_worker()
728 		if (channel->sq != NULL) {  in mlx5e_rl_worker()
730 			mlx5e_rl_close_channel(&channel->sq);  in mlx5e_rl_worker()
1157 		params->rate_limit.queue_level = mlx5e_sq_queue_level(channel->sq);  in mlx5e_rl_query()
1162 		params->rate_limit.queue_level = mlx5e_sq_queue_level(channel->sq);  in mlx5e_rl_query()
1326 			struct mlx5e_sq *sq;  in mlx5e_rl_refresh_channel_params()  local
1329 			sq = channel->sq;  in mlx5e_rl_refresh_channel_params()
1331 			if (sq == NULL)  in mlx5e_rl_refresh_channel_params()
1335 				mlx5_core_modify_cq_moderation_mode(rl->priv->mdev, &sq->cq.mcq,  in mlx5e_rl_refresh_channel_params()
1340 				mlx5_core_modify_cq_moderation(rl->priv->mdev, &sq->cq.mcq,  in mlx5e_rl_refresh_channel_params()
1360 			struct mlx5e_sq *sq;  in mlx5e_rl_refresh_sq_inline()  local
1363 			sq = channel->sq;  in mlx5e_rl_refresh_sq_inline()
1365 			if (sq == NULL)  in mlx5e_rl_refresh_sq_inline()
1368 			mtx_lock(&sq->lock);  in mlx5e_rl_refresh_sq_inline()
1369 			mlx5e_update_sq_inline(sq);  in mlx5e_rl_refresh_sq_inline()
1370 			mtx_unlock(&sq->lock);  in mlx5e_rl_refresh_sq_inline()