Lines Matching refs:rlw
243 mlx5e_rl_open_channel(struct mlx5e_rl_worker *rlw, int eq_ix, in mlx5e_rl_open_channel() argument
247 struct mlx5e_priv *priv = rlw->priv; in mlx5e_rl_open_channel()
498 mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker *rlw, in mlx5e_rlw_channel_set_rate_locked() argument
501 struct mlx5e_rl_priv_data *rl = &rlw->priv->rl; in mlx5e_rlw_channel_set_rate_locked()
510 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rlw_channel_set_rate_locked()
516 MLX5E_SW2HW_MTU(if_getmtu(rlw->priv->ifp)); in mlx5e_rlw_channel_set_rate_locked()
532 atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL); in mlx5e_rlw_channel_set_rate_locked()
535 error = -mlx5_rl_add_rate(rlw->priv->mdev, in mlx5e_rlw_channel_set_rate_locked()
542 atomic_add_64(&rlw->priv->rl.stats.tx_add_new_rate_failure, 1ULL); in mlx5e_rlw_channel_set_rate_locked()
545 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rlw_channel_set_rate_locked()
552 use_sq_remap = MLX5_CAP_QOS(rlw->priv->mdev, qos_remap_pp) && in mlx5e_rlw_channel_set_rate_locked()
565 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rlw_channel_set_rate_locked()
568 mlx5_rl_remove_rate(rlw->priv->mdev, in mlx5e_rlw_channel_set_rate_locked()
577 rlw->priv->mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR && in mlx5e_rlw_channel_set_rate_locked()
578 pci_channel_offline(rlw->priv->mdev->pdev) == 0) in mlx5e_rlw_channel_set_rate_locked()
582 atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL); in mlx5e_rlw_channel_set_rate_locked()
587 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rlw_channel_set_rate_locked()
596 struct mlx5e_rl_worker *rlw = arg; in mlx5e_rl_worker() local
610 priv = rlw->priv; in mlx5e_rl_worker()
613 ix = (rlw - priv->rl.workers) % in mlx5e_rl_worker()
619 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_worker()
621 struct mlx5e_rl_channel *channel = rlw->channels + x; in mlx5e_rl_worker()
627 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_worker()
630 error = mlx5e_rl_open_channel(rlw, ix, in mlx5e_rl_worker()
634 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_worker()
640 mlx5e_rlw_channel_set_rate_locked(rlw, channel, channel->init_rate); in mlx5e_rl_worker()
643 if (STAILQ_FIRST(&rlw->process_head) == NULL) { in mlx5e_rl_worker()
645 if (rlw->worker_done != 0) in mlx5e_rl_worker()
647 cv_wait(&rlw->cv, &rlw->mtx); in mlx5e_rl_worker()
650 if (rlw->worker_done != 0) in mlx5e_rl_worker()
652 channel = STAILQ_FIRST(&rlw->process_head); in mlx5e_rl_worker()
654 STAILQ_REMOVE_HEAD(&rlw->process_head, entry); in mlx5e_rl_worker()
659 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_worker()
664 error = mlx5e_rl_open_channel(rlw, ix, in mlx5e_rl_worker()
672 atomic_add_64(&rlw->priv->rl.stats.tx_open_queues, 1ULL); in mlx5e_rl_worker()
678 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_worker()
680 error = mlx5e_rlw_channel_set_rate_locked(rlw, channel, in mlx5e_rl_worker()
690 error = mlx5e_rlw_channel_set_rate_locked(rlw, channel, 0); in mlx5e_rl_worker()
702 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_worker()
704 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_worker()
707 STAILQ_INSERT_HEAD(&rlw->index_list_head, channel, entry); in mlx5e_rl_worker()
720 struct mlx5e_rl_channel *channel = rlw->channels + x; in mlx5e_rl_worker()
726 mlx5e_rlw_channel_set_rate_locked(rlw, channel, 0); in mlx5e_rl_worker()
729 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_worker()
731 atomic_add_64(&rlw->priv->rl.stats.tx_open_queues, -1ULL); in mlx5e_rl_worker()
732 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_worker()
736 rlw->worker_done = 0; in mlx5e_rl_worker()
737 cv_broadcast(&rlw->cv); in mlx5e_rl_worker()
738 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_worker()
941 struct mlx5e_rl_worker *rlw = rl->workers + j; in mlx5e_rl_init() local
943 rlw->priv = priv; in mlx5e_rl_init()
945 cv_init(&rlw->cv, "mlx5-worker-cv"); in mlx5e_rl_init()
946 mtx_init(&rlw->mtx, "mlx5-worker-mtx", NULL, MTX_DEF); in mlx5e_rl_init()
947 STAILQ_INIT(&rlw->index_list_head); in mlx5e_rl_init()
948 STAILQ_INIT(&rlw->process_head); in mlx5e_rl_init()
950 rlw->channels = malloc(sizeof(rlw->channels[0]) * in mlx5e_rl_init()
953 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_init()
955 struct mlx5e_rl_channel *channel = rlw->channels + i; in mlx5e_rl_init()
956 channel->worker = rlw; in mlx5e_rl_init()
957 STAILQ_INSERT_TAIL(&rlw->index_list_head, channel, entry); in mlx5e_rl_init()
959 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_init()
997 struct mlx5e_rl_worker *rlw = rl->workers + j; in mlx5e_rl_open_workers() local
1000 error = kproc_kthread_add(mlx5e_rl_worker, rlw, &rl_proc, &rl_thread, in mlx5e_rl_open_workers()
1005 rlw->worker_done = 1; in mlx5e_rl_open_workers()
1025 struct mlx5e_rl_worker *rlw = rl->workers + y; in mlx5e_rl_close_workers() local
1028 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_close_workers()
1029 if (rlw->worker_done == 0) { in mlx5e_rl_close_workers()
1030 rlw->worker_done = 1; in mlx5e_rl_close_workers()
1031 cv_broadcast(&rlw->cv); in mlx5e_rl_close_workers()
1034 rlw->worker_done = 0; in mlx5e_rl_close_workers()
1036 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_close_workers()
1041 struct mlx5e_rl_worker *rlw = rl->workers + y; in mlx5e_rl_close_workers() local
1044 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_close_workers()
1045 while (rlw->worker_done != 0) in mlx5e_rl_close_workers()
1046 cv_wait(&rlw->cv, &rlw->mtx); in mlx5e_rl_close_workers()
1047 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_close_workers()
1088 struct mlx5e_rl_worker *rlw = rl->workers + y; in mlx5e_rl_cleanup() local
1090 cv_destroy(&rlw->cv); in mlx5e_rl_cleanup()
1091 mtx_destroy(&rlw->mtx); in mlx5e_rl_cleanup()
1092 free(rlw->channels, M_MLX5EN); in mlx5e_rl_cleanup()
1100 mlx5e_rlw_queue_channel_locked(struct mlx5e_rl_worker *rlw, in mlx5e_rlw_queue_channel_locked() argument
1103 STAILQ_INSERT_TAIL(&rlw->process_head, channel, entry); in mlx5e_rlw_queue_channel_locked()
1104 cv_broadcast(&rlw->cv); in mlx5e_rlw_queue_channel_locked()
1108 mlx5e_rl_free(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel) in mlx5e_rl_free() argument
1113 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_free()
1120 mlx5e_rlw_queue_channel_locked(rlw, channel); in mlx5e_rl_free()
1125 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_free()
1129 mlx5e_rl_modify(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel, uint64_t rate) in mlx5e_rl_modify() argument
1132 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_modify()
1137 mlx5e_rlw_queue_channel_locked(rlw, channel); in mlx5e_rl_modify()
1142 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_modify()
1148 mlx5e_rl_query(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel, in mlx5e_rl_query() argument
1153 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_rl_query()
1169 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_rl_query()
1175 mlx5e_find_available_tx_ring_index(struct mlx5e_rl_worker *rlw, in mlx5e_find_available_tx_ring_index() argument
1181 MLX5E_RL_WORKER_LOCK(rlw); in mlx5e_find_available_tx_ring_index()
1183 if ((channel = STAILQ_FIRST(&rlw->index_list_head)) != NULL) { in mlx5e_find_available_tx_ring_index()
1186 STAILQ_REMOVE_HEAD(&rlw->index_list_head, entry); in mlx5e_find_available_tx_ring_index()
1188 atomic_add_64(&rlw->priv->rl.stats.tx_active_connections, 1ULL); in mlx5e_find_available_tx_ring_index()
1190 atomic_add_64(&rlw->priv->rl.stats.tx_available_resource_failure, 1ULL); in mlx5e_find_available_tx_ring_index()
1192 MLX5E_RL_WORKER_UNLOCK(rlw); in mlx5e_find_available_tx_ring_index()
1196 mlx5_en_info(rlw->priv->ifp, in mlx5e_find_available_tx_ring_index()
1208 struct mlx5e_rl_worker *rlw; in mlx5e_rl_snd_tag_alloc() local
1221 rlw = priv->rl.workers + ((params->rate_limit.hdr.flowid % 128) % in mlx5e_rl_snd_tag_alloc()
1224 error = mlx5e_find_available_tx_ring_index(rlw, &channel); in mlx5e_rl_snd_tag_alloc()
1228 error = mlx5e_rl_modify(rlw, channel, params->rate_limit.max_rate); in mlx5e_rl_snd_tag_alloc()
1230 mlx5e_rl_free(rlw, channel); in mlx5e_rl_snd_tag_alloc()
1322 struct mlx5e_rl_worker *rlw = rl->workers + y; in mlx5e_rl_refresh_channel_params() local
1328 channel = rlw->channels + x; in mlx5e_rl_refresh_channel_params()
1356 struct mlx5e_rl_worker *rlw = rl->workers + y; in mlx5e_rl_refresh_sq_inline() local
1362 channel = rlw->channels + x; in mlx5e_rl_refresh_sq_inline()