55     struct mlx5e_sq_param *param)  in mlx5e_rl_build_sq_param()  argument
57 	void *sqc = param->sqc;  in mlx5e_rl_build_sq_param()
59 	uint8_t log_sq_size = order_base_2(rl->param.tx_queue_size);  in mlx5e_rl_build_sq_param()
65 	param->wq.linear = 1;  in mlx5e_rl_build_sq_param()
70     struct mlx5e_cq_param *param)  in mlx5e_rl_build_cq_param()  argument
72 	void *cqc = param->cqc;  in mlx5e_rl_build_cq_param()
73 	uint8_t log_sq_size = order_base_2(rl->param.tx_queue_size);  in mlx5e_rl_build_cq_param()
76 	MLX5_SET(cqc, cqc, cq_period, rl->param.tx_coalesce_usecs);  in mlx5e_rl_build_cq_param()
77 	MLX5_SET(cqc, cqc, cq_max_count, rl->param.tx_coalesce_pkts);  in mlx5e_rl_build_cq_param()
80 	switch (rl->param.tx_coalesce_mode) {  in mlx5e_rl_build_cq_param()
105     struct mlx5e_sq_param *param, int ix)  in mlx5e_rl_create_sq()  argument
108 	void *sqc = param->sqc;  in mlx5e_rl_create_sq()
132 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,  in mlx5e_rl_create_sq()
189     struct mlx5e_sq_param *param, int ix)  in mlx5e_rl_open_sq()  argument
193 	err = mlx5e_rl_create_sq(priv, sq, param, ix);  in mlx5e_rl_open_sq()
197 	err = mlx5e_enable_sq(sq, param, &priv->channel[ix].bfreg, priv->rl.tisn);  in mlx5e_rl_open_sq()
235 	sq->cev_factor = priv->rl.param.tx_completion_fact;  in mlx5e_rl_chan_mtx_init()
324 	uint64_t max = rl->param.tx_queue_size /  in mlx5e_rl_sync_tx_completion_fact()
336 	rl->param.tx_completion_fact_max = max;  in mlx5e_rl_sync_tx_completion_fact()
342 	if (rl->param.tx_completion_fact < 1)  in mlx5e_rl_sync_tx_completion_fact()
343 		rl->param.tx_completion_fact = 1;  in mlx5e_rl_sync_tx_completion_fact()
344 	else if (rl->param.tx_completion_fact > max)  in mlx5e_rl_sync_tx_completion_fact()
345 		rl->param.tx_completion_fact = max;  in mlx5e_rl_sync_tx_completion_fact()
395 	for (x = 0; x != rl->param.tx_rates_def; x++) {  in mlx5e_rl_find_best_rate_locked()
413 	if (user_rate > rl->param.tx_limit_max)  in mlx5e_rl_find_best_rate_locked()
414 		user_rate = rl->param.tx_limit_max;  in mlx5e_rl_find_best_rate_locked()
418 	    rl->param.tx_allowed_deviation, 1000ULL))  in mlx5e_rl_find_best_rate_locked()
515 		temp = rl->param.tx_burst_size *  in mlx5e_rlw_channel_set_rate_locked()
620 	for (x = 0; x < priv->rl.param.tx_channels_per_worker_def; x++) {  in mlx5e_rl_worker()
719 	for (x = 0; x < priv->rl.param.tx_channels_per_worker_def; x++) {  in mlx5e_rl_worker()
765 mlx5e_rl_set_default_params(struct mlx5e_rl_params *param,  in mlx5e_rl_set_default_params()  argument
769 	param->tx_worker_threads_def = mdev->priv.eq_table.num_comp_vectors;  in mlx5e_rl_set_default_params()
770 	param->tx_worker_threads_max = MLX5E_RL_MAX_WORKERS;  in mlx5e_rl_set_default_params()
773 	if (param->tx_worker_threads_def == 0 ||  in mlx5e_rl_set_default_params()
774 	    param->tx_worker_threads_def > param->tx_worker_threads_max)  in mlx5e_rl_set_default_params()
775 		param->tx_worker_threads_def = param->tx_worker_threads_max;  in mlx5e_rl_set_default_params()
778 	param->tx_channels_per_worker_def = MLX5E_RL_MAX_SQS /  in mlx5e_rl_set_default_params()
779 	    param->tx_worker_threads_def;  in mlx5e_rl_set_default_params()
780 	param->tx_channels_per_worker_max = MLX5E_RL_MAX_SQS;  in mlx5e_rl_set_default_params()
783 	if (param->tx_channels_per_worker_def > MLX5E_RL_DEF_SQ_PER_WORKER)  in mlx5e_rl_set_default_params()
784 		param->tx_channels_per_worker_def = MLX5E_RL_DEF_SQ_PER_WORKER;  in mlx5e_rl_set_default_params()
787 	param->tx_burst_size = 4;	/* MTUs */  in mlx5e_rl_set_default_params()
800 	param->tx_burst_size_max = 255;  in mlx5e_rl_set_default_params()
803 	param->tx_limit_min = mdev->priv.rl_table.min_rate * 1000ULL;  in mlx5e_rl_set_default_params()
804 	param->tx_limit_max = mdev->priv.rl_table.max_rate * 1000ULL;  in mlx5e_rl_set_default_params()
807 	param->tx_rates_max = mdev->priv.rl_table.max_size;  in mlx5e_rl_set_default_params()
810 	if (param->tx_rates_max > MLX5E_RL_MAX_TX_RATES)  in mlx5e_rl_set_default_params()
811 		param->tx_rates_max = MLX5E_RL_MAX_TX_RATES;  in mlx5e_rl_set_default_params()
814 	param->tx_rates_def = param->tx_rates_max;  in mlx5e_rl_set_default_params()
817 	if (param->tx_limit_max != 0) {  in mlx5e_rl_set_default_params()
822 		param->tx_allowed_deviation_max = -1ULL /  in mlx5e_rl_set_default_params()
823 		    param->tx_limit_max;  in mlx5e_rl_set_default_params()
826 	param->tx_allowed_deviation = 50;	/* 5.0% */  in mlx5e_rl_set_default_params()
829 	param->tx_queue_size = (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);  in mlx5e_rl_set_default_params()
830 	param->tx_coalesce_usecs = MLX5E_RL_TX_COAL_USEC_DEFAULT;  in mlx5e_rl_set_default_params()
831 	param->tx_coalesce_pkts = MLX5E_RL_TX_COAL_PKTS_DEFAULT;  in mlx5e_rl_set_default_params()
832 	param->tx_coalesce_mode = MLX5E_RL_TX_COAL_MODE_DEFAULT;  in mlx5e_rl_set_default_params()
833 	param->tx_completion_fact = MLX5E_RL_TX_COMP_FACT_DEFAULT;  in mlx5e_rl_set_default_params()
875 	mlx5e_rl_set_default_params(&rl->param, priv->mdev);  in mlx5e_rl_init()
909 	    rl->param.tx_worker_threads_def, M_MLX5EN, M_WAITOK | M_ZERO);  in mlx5e_rl_init()
913 	    rl->param.tx_rates_def, M_MLX5EN, M_WAITOK | M_ZERO);  in mlx5e_rl_init()
923 		for (i = 0; i != rl->param.tx_rates_def; i++) {  in mlx5e_rl_init()
940 	for (j = 0; j < rl->param.tx_worker_threads_def; j++) {  in mlx5e_rl_init()
951 		    rl->param.tx_channels_per_worker_def, M_MLX5EN, M_WAITOK | M_ZERO);  in mlx5e_rl_init()
954 		for (i = 0; i < rl->param.tx_channels_per_worker_def; i++) {  in mlx5e_rl_init()
996 	for (j = 0; j < rl->param.tx_worker_threads_def; j++) {  in mlx5e_rl_open_workers()
1024 	for (y = 0; y < rl->param.tx_worker_threads_def; y++) {  in mlx5e_rl_close_workers()
1040 	for (y = 0; y < rl->param.tx_worker_threads_def; y++) {  in mlx5e_rl_close_workers()
1059 	for (x = 0; x != rl->param.tx_rates_def; x++)  in mlx5e_rl_reset_rates()
1087 	for (y = 0; y < rl->param.tx_worker_threads_def; y++) {  in mlx5e_rl_cleanup()
1222 	    priv->rl.param.tx_worker_threads_def);  in mlx5e_rl_snd_tag_alloc()
1285 	sbuf_new_for_sysctl(&sbuf, NULL, 128 * rl->param.tx_rates_def, req);  in mlx5e_rl_sysctl_show_rate_table()
1292 	for (x = 0; x != rl->param.tx_rates_def; x++) {  in mlx5e_rl_sysctl_show_rate_table()
1297 		    x, (unsigned)rl->param.tx_burst_size,  in mlx5e_rl_sysctl_show_rate_table()
1321 	for (y = 0; y != rl->param.tx_worker_threads_def; y++) {  in mlx5e_rl_refresh_channel_params()
1324 		for (x = 0; x != rl->param.tx_channels_per_worker_def; x++) {  in mlx5e_rl_refresh_channel_params()
1336 				    rl->param.tx_coalesce_usecs,  in mlx5e_rl_refresh_channel_params()
1337 				    rl->param.tx_coalesce_pkts,  in mlx5e_rl_refresh_channel_params()
1338 				    rl->param.tx_coalesce_mode);  in mlx5e_rl_refresh_channel_params()
1341 				    rl->param.tx_coalesce_usecs,  in mlx5e_rl_refresh_channel_params()
1342 				    rl->param.tx_coalesce_pkts);  in mlx5e_rl_refresh_channel_params()
1355 	for (y = 0; y != rl->param.tx_worker_threads_def; y++) {  in mlx5e_rl_refresh_sq_inline()
1358 		for (x = 0; x != rl->param.tx_channels_per_worker_def; x++) {  in mlx5e_rl_refresh_sq_inline()
1389 	for (x = 0; x != rl->param.tx_rates_def; x++) {  in mlx5e_rl_tx_limit_add()
1397 	if (x == rl->param.tx_rates_def) {  in mlx5e_rl_tx_limit_add()
1398 		for (x = 0; x != rl->param.tx_rates_def; x++) {  in mlx5e_rl_tx_limit_add()
1423 	for (x = 0; x != rl->param.tx_rates_def; x++) {  in mlx5e_rl_tx_limit_clr()
1432 	if (x == rl->param.tx_rates_def)  in mlx5e_rl_tx_limit_clr()
1454 	value = rl->param.arg[arg2];  in mlx5e_rl_sysctl_handler()
1460 		    value == rl->param.arg[arg2])  in mlx5e_rl_sysctl_handler()
1476 		if (value > rl->param.tx_worker_threads_max)  in mlx5e_rl_sysctl_handler()
1477 			value = rl->param.tx_worker_threads_max;  in mlx5e_rl_sysctl_handler()
1482 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1486 		if (value > rl->param.tx_channels_per_worker_max)  in mlx5e_rl_sysctl_handler()
1487 			value = rl->param.tx_channels_per_worker_max;  in mlx5e_rl_sysctl_handler()
1492 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1496 		if (value > rl->param.tx_rates_max)  in mlx5e_rl_sysctl_handler()
1497 			value = rl->param.tx_rates_max;  in mlx5e_rl_sysctl_handler()
1502 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1513 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1528 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1545 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1571 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1587 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1607 		if (value > rl->param.tx_allowed_deviation_max)  in mlx5e_rl_sysctl_handler()
1608 			value = rl->param.tx_allowed_deviation_max;  in mlx5e_rl_sysctl_handler()
1609 		else if (value < rl->param.tx_allowed_deviation_min)  in mlx5e_rl_sysctl_handler()
1610 			value = rl->param.tx_allowed_deviation_min;  in mlx5e_rl_sysctl_handler()
1613 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()
1619 		if (value > rl->param.tx_burst_size_max)  in mlx5e_rl_sysctl_handler()
1620 			value = rl->param.tx_burst_size_max;  in mlx5e_rl_sysctl_handler()
1621 		else if (value < rl->param.tx_burst_size_min)  in mlx5e_rl_sysctl_handler()
1622 			value = rl->param.tx_burst_size_min;  in mlx5e_rl_sysctl_handler()
1625 		rl->param.arg[arg2] = value;  in mlx5e_rl_sysctl_handler()