xref: /freebsd/sys/dev/cxgbe/t4_sched.c (revision ec8004dd419d8c8acfc9025dd050f141c949d53a)
12204b427SNavdeep Parhar /*-
22204b427SNavdeep Parhar  * Copyright (c) 2017 Chelsio Communications, Inc.
32204b427SNavdeep Parhar  * All rights reserved.
42204b427SNavdeep Parhar  * Written by: Navdeep Parhar <np@FreeBSD.org>
52204b427SNavdeep Parhar  *
62204b427SNavdeep Parhar  * Redistribution and use in source and binary forms, with or without
72204b427SNavdeep Parhar  * modification, are permitted provided that the following conditions
82204b427SNavdeep Parhar  * are met:
92204b427SNavdeep Parhar  * 1. Redistributions of source code must retain the above copyright
102204b427SNavdeep Parhar  *    notice, this list of conditions and the following disclaimer.
112204b427SNavdeep Parhar  * 2. Redistributions in binary form must reproduce the above copyright
122204b427SNavdeep Parhar  *    notice, this list of conditions and the following disclaimer in the
132204b427SNavdeep Parhar  *    documentation and/or other materials provided with the distribution.
142204b427SNavdeep Parhar  *
152204b427SNavdeep Parhar  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
162204b427SNavdeep Parhar  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
172204b427SNavdeep Parhar  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
182204b427SNavdeep Parhar  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
192204b427SNavdeep Parhar  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
202204b427SNavdeep Parhar  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
212204b427SNavdeep Parhar  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
222204b427SNavdeep Parhar  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
232204b427SNavdeep Parhar  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
242204b427SNavdeep Parhar  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
252204b427SNavdeep Parhar  * SUCH DAMAGE.
262204b427SNavdeep Parhar  */
272204b427SNavdeep Parhar 
282204b427SNavdeep Parhar #include <sys/cdefs.h>
292204b427SNavdeep Parhar __FBSDID("$FreeBSD$");
302204b427SNavdeep Parhar 
312204b427SNavdeep Parhar #include "opt_inet.h"
322204b427SNavdeep Parhar #include "opt_inet6.h"
3367e07112SNavdeep Parhar #include "opt_ratelimit.h"
342204b427SNavdeep Parhar 
352204b427SNavdeep Parhar #include <sys/types.h>
362204b427SNavdeep Parhar #include <sys/malloc.h>
372204b427SNavdeep Parhar #include <sys/queue.h>
382204b427SNavdeep Parhar #include <sys/sbuf.h>
392204b427SNavdeep Parhar #include <sys/taskqueue.h>
402204b427SNavdeep Parhar #include <sys/sysctl.h>
412204b427SNavdeep Parhar 
422204b427SNavdeep Parhar #include "common/common.h"
432204b427SNavdeep Parhar #include "common/t4_regs.h"
442204b427SNavdeep Parhar #include "common/t4_regs_values.h"
452204b427SNavdeep Parhar #include "common/t4_msg.h"
462204b427SNavdeep Parhar 
472204b427SNavdeep Parhar 
482204b427SNavdeep Parhar static int
492204b427SNavdeep Parhar in_range(int val, int lo, int hi)
502204b427SNavdeep Parhar {
512204b427SNavdeep Parhar 
522204b427SNavdeep Parhar 	return (val < 0 || (val <= hi && val >= lo));
532204b427SNavdeep Parhar }
542204b427SNavdeep Parhar 
552204b427SNavdeep Parhar static int
562204b427SNavdeep Parhar set_sched_class_config(struct adapter *sc, int minmax)
572204b427SNavdeep Parhar {
582204b427SNavdeep Parhar 	int rc;
592204b427SNavdeep Parhar 
602204b427SNavdeep Parhar 	if (minmax < 0)
612204b427SNavdeep Parhar 		return (EINVAL);
622204b427SNavdeep Parhar 
632204b427SNavdeep Parhar 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc");
642204b427SNavdeep Parhar 	if (rc)
652204b427SNavdeep Parhar 		return (rc);
6683b5cda1SNavdeep Parhar 	if (hw_off_limits(sc))
6783b5cda1SNavdeep Parhar 		rc = ENXIO;
6883b5cda1SNavdeep Parhar 	else
692204b427SNavdeep Parhar 		rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1);
702204b427SNavdeep Parhar 	end_synchronized_op(sc, 0);
712204b427SNavdeep Parhar 
722204b427SNavdeep Parhar 	return (rc);
732204b427SNavdeep Parhar }
742204b427SNavdeep Parhar 
752204b427SNavdeep Parhar static int
762204b427SNavdeep Parhar set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p,
772204b427SNavdeep Parhar     int sleep_ok)
782204b427SNavdeep Parhar {
792204b427SNavdeep Parhar 	int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode;
802204b427SNavdeep Parhar 	struct port_info *pi;
811979b511SNavdeep Parhar 	struct tx_cl_rl_params *tc, old;
829c3b8b3cSNavdeep Parhar 	bool check_pktsize = false;
832204b427SNavdeep Parhar 
842204b427SNavdeep Parhar 	if (p->level == SCHED_CLASS_LEVEL_CL_RL)
852204b427SNavdeep Parhar 		fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
862204b427SNavdeep Parhar 	else if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
872204b427SNavdeep Parhar 		fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
882204b427SNavdeep Parhar 	else if (p->level == SCHED_CLASS_LEVEL_CH_RL)
892204b427SNavdeep Parhar 		fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
902204b427SNavdeep Parhar 	else
912204b427SNavdeep Parhar 		return (EINVAL);
922204b427SNavdeep Parhar 
939c3b8b3cSNavdeep Parhar 	if (p->level == SCHED_CLASS_LEVEL_CL_RL) {
942204b427SNavdeep Parhar 		if (p->mode == SCHED_CLASS_MODE_CLASS)
952204b427SNavdeep Parhar 			fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
969c3b8b3cSNavdeep Parhar 		else if (p->mode == SCHED_CLASS_MODE_FLOW) {
979c3b8b3cSNavdeep Parhar 			check_pktsize = true;
982204b427SNavdeep Parhar 			fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
999c3b8b3cSNavdeep Parhar 		} else
1002204b427SNavdeep Parhar 			return (EINVAL);
1019c3b8b3cSNavdeep Parhar 	} else
1029c3b8b3cSNavdeep Parhar 		fw_mode = 0;
1032204b427SNavdeep Parhar 
1049c3b8b3cSNavdeep Parhar 	/* Valid channel must always be provided. */
1059c3b8b3cSNavdeep Parhar 	if (p->channel < 0)
1062204b427SNavdeep Parhar 		return (EINVAL);
1072204b427SNavdeep Parhar 	if (!in_range(p->channel, 0, sc->chip_params->nchan - 1))
1082204b427SNavdeep Parhar 		return (ERANGE);
1092204b427SNavdeep Parhar 
1102204b427SNavdeep Parhar 	pi = sc->port[sc->chan_map[p->channel]];
1112204b427SNavdeep Parhar 	if (pi == NULL)
1122204b427SNavdeep Parhar 		return (ENXIO);
1132204b427SNavdeep Parhar 	MPASS(pi->tx_chan == p->channel);
1142204b427SNavdeep Parhar 	top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */
1152204b427SNavdeep Parhar 
1169c3b8b3cSNavdeep Parhar 	if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
1179c3b8b3cSNavdeep Parhar 	    p->level == SCHED_CLASS_LEVEL_CH_RL) {
1182204b427SNavdeep Parhar 		/*
1199c3b8b3cSNavdeep Parhar 		 * Valid rate (mode, unit and values) must be provided.
1202204b427SNavdeep Parhar 		 */
1212204b427SNavdeep Parhar 
1222204b427SNavdeep Parhar 		if (p->minrate < 0)
1232204b427SNavdeep Parhar 			p->minrate = 0;
1249c3b8b3cSNavdeep Parhar 		if (p->maxrate < 0)
1252204b427SNavdeep Parhar 			return (EINVAL);
1269c3b8b3cSNavdeep Parhar 
1279c3b8b3cSNavdeep Parhar 		if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS) {
1289c3b8b3cSNavdeep Parhar 			fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
1299c3b8b3cSNavdeep Parhar 			/* ratemode could be relative (%) or absolute. */
1309c3b8b3cSNavdeep Parhar 			if (p->ratemode == SCHED_CLASS_RATEMODE_REL) {
1319c3b8b3cSNavdeep Parhar 				fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
1329c3b8b3cSNavdeep Parhar 				/* maxrate is % of port bandwidth. */
1339c3b8b3cSNavdeep Parhar 				if (!in_range(p->minrate, 0, 100) ||
1349c3b8b3cSNavdeep Parhar 				    !in_range(p->maxrate, 0, 100)) {
1359c3b8b3cSNavdeep Parhar 					return (ERANGE);
1362204b427SNavdeep Parhar 				}
1379c3b8b3cSNavdeep Parhar 			} else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS) {
1389c3b8b3cSNavdeep Parhar 				fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
1399c3b8b3cSNavdeep Parhar 				/* maxrate is absolute value in kbps. */
1409c3b8b3cSNavdeep Parhar 				if (!in_range(p->minrate, 0, top_speed) ||
1419c3b8b3cSNavdeep Parhar 				    !in_range(p->maxrate, 0, top_speed)) {
1429c3b8b3cSNavdeep Parhar 					return (ERANGE);
1432204b427SNavdeep Parhar 				}
1449c3b8b3cSNavdeep Parhar 			} else
1452204b427SNavdeep Parhar 				return (EINVAL);
1469c3b8b3cSNavdeep Parhar 		} else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS) {
1479c3b8b3cSNavdeep Parhar 			/* maxrate is the absolute value in pps. */
1489c3b8b3cSNavdeep Parhar 			check_pktsize = true;
1499c3b8b3cSNavdeep Parhar 			fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
1509c3b8b3cSNavdeep Parhar 		} else
1519c3b8b3cSNavdeep Parhar 			return (EINVAL);
1529c3b8b3cSNavdeep Parhar 	} else {
1539c3b8b3cSNavdeep Parhar 		MPASS(p->level == SCHED_CLASS_LEVEL_CL_WRR);
1549c3b8b3cSNavdeep Parhar 
1559c3b8b3cSNavdeep Parhar 		/*
1569c3b8b3cSNavdeep Parhar 		 * Valid weight must be provided.
1579c3b8b3cSNavdeep Parhar 		 */
1589c3b8b3cSNavdeep Parhar 		if (p->weight < 0)
1599c3b8b3cSNavdeep Parhar 		       return (EINVAL);
1609c3b8b3cSNavdeep Parhar 		if (!in_range(p->weight, 1, 99))
1619c3b8b3cSNavdeep Parhar 			return (ERANGE);
1629c3b8b3cSNavdeep Parhar 
1639c3b8b3cSNavdeep Parhar 		fw_rateunit = 0;
1649c3b8b3cSNavdeep Parhar 		fw_ratemode = 0;
1659c3b8b3cSNavdeep Parhar 	}
1669c3b8b3cSNavdeep Parhar 
1679c3b8b3cSNavdeep Parhar 	if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
1689c3b8b3cSNavdeep Parhar 	    p->level == SCHED_CLASS_LEVEL_CL_WRR) {
1699c3b8b3cSNavdeep Parhar 		/*
1709c3b8b3cSNavdeep Parhar 		 * Valid scheduling class must be provided.
1719c3b8b3cSNavdeep Parhar 		 */
1729c3b8b3cSNavdeep Parhar 		if (p->cl < 0)
1739c3b8b3cSNavdeep Parhar 			return (EINVAL);
1746beb67c7SNavdeep Parhar 		if (!in_range(p->cl, 0, sc->params.nsched_cls - 1))
1759c3b8b3cSNavdeep Parhar 			return (ERANGE);
1769c3b8b3cSNavdeep Parhar 	}
1779c3b8b3cSNavdeep Parhar 
1789c3b8b3cSNavdeep Parhar 	if (check_pktsize) {
1799c3b8b3cSNavdeep Parhar 		if (p->pktsize < 0)
1809c3b8b3cSNavdeep Parhar 			return (EINVAL);
1819c3b8b3cSNavdeep Parhar 		if (!in_range(p->pktsize, 64, pi->vi[0].ifp->if_mtu))
1829c3b8b3cSNavdeep Parhar 			return (ERANGE);
1832204b427SNavdeep Parhar 	}
1842204b427SNavdeep Parhar 
1852204b427SNavdeep Parhar 	if (p->level == SCHED_CLASS_LEVEL_CL_RL) {
1862204b427SNavdeep Parhar 		tc = &pi->sched_params->cl_rl[p->cl];
1871979b511SNavdeep Parhar 		mtx_lock(&sc->tc_lock);
188*ec8004ddSNavdeep Parhar 		if (tc->refcount > 0 || tc->state == CS_HW_UPDATE_IN_PROGRESS)
1892204b427SNavdeep Parhar 			rc = EBUSY;
1901979b511SNavdeep Parhar 		else {
191*ec8004ddSNavdeep Parhar 			old = *tc;
192*ec8004ddSNavdeep Parhar 
193*ec8004ddSNavdeep Parhar 			tc->flags |= CF_USER;
194*ec8004ddSNavdeep Parhar 			tc->state = CS_HW_UPDATE_IN_PROGRESS;
1952204b427SNavdeep Parhar 			tc->ratemode = fw_ratemode;
1962204b427SNavdeep Parhar 			tc->rateunit = fw_rateunit;
1972204b427SNavdeep Parhar 			tc->mode = fw_mode;
1982204b427SNavdeep Parhar 			tc->maxrate = p->maxrate;
1992204b427SNavdeep Parhar 			tc->pktsize = p->pktsize;
2001979b511SNavdeep Parhar 			rc = 0;
2012204b427SNavdeep Parhar 		}
2021979b511SNavdeep Parhar 		mtx_unlock(&sc->tc_lock);
2031979b511SNavdeep Parhar 		if (rc != 0)
2041979b511SNavdeep Parhar 			return (rc);
2051979b511SNavdeep Parhar 	}
2061979b511SNavdeep Parhar 
2071979b511SNavdeep Parhar 	rc = begin_synchronized_op(sc, NULL,
2081979b511SNavdeep Parhar 	    sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp");
2091979b511SNavdeep Parhar 	if (rc != 0) {
2101979b511SNavdeep Parhar 		if (p->level == SCHED_CLASS_LEVEL_CL_RL) {
2111979b511SNavdeep Parhar 			mtx_lock(&sc->tc_lock);
212*ec8004ddSNavdeep Parhar 			MPASS(tc->refcount == 0);
213*ec8004ddSNavdeep Parhar 			MPASS(tc->flags & CF_USER);
214*ec8004ddSNavdeep Parhar 			MPASS(tc->state == CS_HW_UPDATE_IN_PROGRESS);
2151979b511SNavdeep Parhar 			*tc = old;
2161979b511SNavdeep Parhar 			mtx_unlock(&sc->tc_lock);
2171979b511SNavdeep Parhar 		}
2181979b511SNavdeep Parhar 		return (rc);
2192204b427SNavdeep Parhar 	}
22083b5cda1SNavdeep Parhar 	if (!hw_off_limits(sc)) {
22183b5cda1SNavdeep Parhar 		rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level,
22283b5cda1SNavdeep Parhar 		    fw_mode, fw_rateunit, fw_ratemode, p->channel, p->cl,
22383b5cda1SNavdeep Parhar 		    p->minrate, p->maxrate, p->weight, p->pktsize, 0, sleep_ok);
22483b5cda1SNavdeep Parhar 	}
2252204b427SNavdeep Parhar 	end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD);
2262204b427SNavdeep Parhar 
2271979b511SNavdeep Parhar 	if (p->level == SCHED_CLASS_LEVEL_CL_RL) {
2281979b511SNavdeep Parhar 		mtx_lock(&sc->tc_lock);
2291979b511SNavdeep Parhar 		MPASS(tc->refcount == 0);
230*ec8004ddSNavdeep Parhar 		MPASS(tc->flags & CF_USER);
231*ec8004ddSNavdeep Parhar 		MPASS(tc->state == CS_HW_UPDATE_IN_PROGRESS);
2321979b511SNavdeep Parhar 
2331979b511SNavdeep Parhar 		if (rc == 0)
234*ec8004ddSNavdeep Parhar 			tc->state = CS_HW_CONFIGURED;
235*ec8004ddSNavdeep Parhar 		else {
236*ec8004ddSNavdeep Parhar 			/* parameters failed so we don't park at params_set */
237*ec8004ddSNavdeep Parhar 			tc->state = CS_UNINITIALIZED;
238*ec8004ddSNavdeep Parhar 			tc->flags &= ~CF_USER;
239*ec8004ddSNavdeep Parhar 			CH_ERR(pi, "failed to configure traffic class %d: %d.  "
240*ec8004ddSNavdeep Parhar 			    "params: mode %d, rateunit %d, ratemode %d, "
241*ec8004ddSNavdeep Parhar 			    "channel %d, minrate %d, maxrate %d, pktsize %d, "
242*ec8004ddSNavdeep Parhar 			    "burstsize %d\n", p->cl, rc, fw_mode, fw_rateunit,
243*ec8004ddSNavdeep Parhar 			    fw_ratemode, p->channel, p->minrate, p->maxrate,
244*ec8004ddSNavdeep Parhar 			    p->pktsize, 0);
245*ec8004ddSNavdeep Parhar 		}
2461979b511SNavdeep Parhar 		mtx_unlock(&sc->tc_lock);
2471979b511SNavdeep Parhar 	}
2481979b511SNavdeep Parhar 
2492204b427SNavdeep Parhar 	return (rc);
2502204b427SNavdeep Parhar }
2512204b427SNavdeep Parhar 
2522204b427SNavdeep Parhar static void
2532204b427SNavdeep Parhar update_tx_sched(void *context, int pending)
2542204b427SNavdeep Parhar {
2551979b511SNavdeep Parhar 	int i, j, rc;
2562204b427SNavdeep Parhar 	struct port_info *pi;
2572204b427SNavdeep Parhar 	struct tx_cl_rl_params *tc;
2582204b427SNavdeep Parhar 	struct adapter *sc = context;
2596beb67c7SNavdeep Parhar 	const int n = sc->params.nsched_cls;
2602204b427SNavdeep Parhar 
2612204b427SNavdeep Parhar 	mtx_lock(&sc->tc_lock);
2622204b427SNavdeep Parhar 	for_each_port(sc, i) {
2632204b427SNavdeep Parhar 		pi = sc->port[i];
2642204b427SNavdeep Parhar 		tc = &pi->sched_params->cl_rl[0];
2652204b427SNavdeep Parhar 		for (j = 0; j < n; j++, tc++) {
2662204b427SNavdeep Parhar 			MPASS(mtx_owned(&sc->tc_lock));
267*ec8004ddSNavdeep Parhar 			if (tc->state != CS_HW_UPDATE_REQUESTED)
2682204b427SNavdeep Parhar 				continue;
2692204b427SNavdeep Parhar 			mtx_unlock(&sc->tc_lock);
2702204b427SNavdeep Parhar 
2712204b427SNavdeep Parhar 			if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
2722204b427SNavdeep Parhar 			    "t4utxs") != 0) {
2732204b427SNavdeep Parhar 				mtx_lock(&sc->tc_lock);
2742204b427SNavdeep Parhar 				continue;
2752204b427SNavdeep Parhar 			}
2761979b511SNavdeep Parhar 			rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED,
2771979b511SNavdeep Parhar 			    FW_SCHED_PARAMS_LEVEL_CL_RL, tc->mode, tc->rateunit,
2781979b511SNavdeep Parhar 			    tc->ratemode, pi->tx_chan, j, 0, tc->maxrate, 0,
27909a7189fSNavdeep Parhar 			    tc->pktsize, tc->burstsize, 1);
2802204b427SNavdeep Parhar 			end_synchronized_op(sc, 0);
2812204b427SNavdeep Parhar 
2822204b427SNavdeep Parhar 			mtx_lock(&sc->tc_lock);
283*ec8004ddSNavdeep Parhar 			MPASS(tc->state == CS_HW_UPDATE_REQUESTED);
284*ec8004ddSNavdeep Parhar 			if (rc == 0) {
285*ec8004ddSNavdeep Parhar 				tc->state = CS_HW_CONFIGURED;
286*ec8004ddSNavdeep Parhar 				continue;
287*ec8004ddSNavdeep Parhar 			}
288*ec8004ddSNavdeep Parhar 			/* parameters failed so we try to avoid params_set */
289*ec8004ddSNavdeep Parhar 			if (tc->refcount > 0)
290*ec8004ddSNavdeep Parhar 				tc->state = CS_PARAMS_SET;
2911979b511SNavdeep Parhar 			else
292*ec8004ddSNavdeep Parhar 				tc->state = CS_UNINITIALIZED;
293*ec8004ddSNavdeep Parhar 			CH_ERR(pi, "failed to configure traffic class %d: %d.  "
294*ec8004ddSNavdeep Parhar 			    "params: mode %d, rateunit %d, ratemode %d, "
295*ec8004ddSNavdeep Parhar 			    "channel %d, minrate %d, maxrate %d, pktsize %d, "
296*ec8004ddSNavdeep Parhar 			    "burstsize %d\n", j, rc, tc->mode, tc->rateunit,
297*ec8004ddSNavdeep Parhar 			    tc->ratemode, pi->tx_chan, 0, tc->maxrate,
298*ec8004ddSNavdeep Parhar 			    tc->pktsize, tc->burstsize);
2992204b427SNavdeep Parhar 		}
3002204b427SNavdeep Parhar 	}
3012204b427SNavdeep Parhar 	mtx_unlock(&sc->tc_lock);
3022204b427SNavdeep Parhar }
3032204b427SNavdeep Parhar 
3042204b427SNavdeep Parhar int
3052204b427SNavdeep Parhar t4_set_sched_class(struct adapter *sc, struct t4_sched_params *p)
3062204b427SNavdeep Parhar {
3072204b427SNavdeep Parhar 
3082204b427SNavdeep Parhar 	if (p->type != SCHED_CLASS_TYPE_PACKET)
3092204b427SNavdeep Parhar 		return (EINVAL);
3102204b427SNavdeep Parhar 
3112204b427SNavdeep Parhar 	if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
3122204b427SNavdeep Parhar 		return (set_sched_class_config(sc, p->u.config.minmax));
3132204b427SNavdeep Parhar 
3142204b427SNavdeep Parhar 	if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
3152204b427SNavdeep Parhar 		return (set_sched_class_params(sc, &p->u.params, 1));
3162204b427SNavdeep Parhar 
3172204b427SNavdeep Parhar 	return (EINVAL);
3182204b427SNavdeep Parhar }
3192204b427SNavdeep Parhar 
3201979b511SNavdeep Parhar static int
3211979b511SNavdeep Parhar bind_txq_to_traffic_class(struct adapter *sc, struct sge_txq *txq, int idx)
3221979b511SNavdeep Parhar {
3231979b511SNavdeep Parhar 	struct tx_cl_rl_params *tc0, *tc;
3241979b511SNavdeep Parhar 	int rc, old_idx;
3251979b511SNavdeep Parhar 	uint32_t fw_mnem, fw_class;
3261979b511SNavdeep Parhar 
32743bbae19SNavdeep Parhar 	if (!(txq->eq.flags & EQ_HW_ALLOCATED))
32843bbae19SNavdeep Parhar 		return (ENXIO);
3291979b511SNavdeep Parhar 
3301979b511SNavdeep Parhar 	mtx_lock(&sc->tc_lock);
3311979b511SNavdeep Parhar 	if (txq->tc_idx == -2) {
3321979b511SNavdeep Parhar 		rc = EBUSY;	/* Another bind/unbind in progress already. */
3331979b511SNavdeep Parhar 		goto done;
3341979b511SNavdeep Parhar 	}
3351979b511SNavdeep Parhar 	if (idx == txq->tc_idx) {
3361979b511SNavdeep Parhar 		rc = 0;		/* No change, nothing to do. */
3371979b511SNavdeep Parhar 		goto done;
3381979b511SNavdeep Parhar 	}
3391979b511SNavdeep Parhar 
3401979b511SNavdeep Parhar 	tc0 = &sc->port[txq->eq.tx_chan]->sched_params->cl_rl[0];
3411979b511SNavdeep Parhar 	if (idx != -1) {
3421979b511SNavdeep Parhar 		/*
3431979b511SNavdeep Parhar 		 * Bind to a different class at index idx.
3441979b511SNavdeep Parhar 		 */
3451979b511SNavdeep Parhar 		tc = &tc0[idx];
346*ec8004ddSNavdeep Parhar 		if (tc->state != CS_HW_CONFIGURED) {
3471979b511SNavdeep Parhar 			rc = ENXIO;
3481979b511SNavdeep Parhar 			goto done;
3491979b511SNavdeep Parhar 		} else {
3501979b511SNavdeep Parhar 			/*
3511979b511SNavdeep Parhar 			 * Ok to proceed.  Place a reference on the new class
3521979b511SNavdeep Parhar 			 * while still holding on to the reference on the
3531979b511SNavdeep Parhar 			 * previous class, if any.
3541979b511SNavdeep Parhar 			 */
3551979b511SNavdeep Parhar 			tc->refcount++;
3561979b511SNavdeep Parhar 		}
3571979b511SNavdeep Parhar 	}
3581979b511SNavdeep Parhar 	/* Mark as busy before letting go of the lock. */
3591979b511SNavdeep Parhar 	old_idx = txq->tc_idx;
3601979b511SNavdeep Parhar 	txq->tc_idx = -2;
3611979b511SNavdeep Parhar 	mtx_unlock(&sc->tc_lock);
3621979b511SNavdeep Parhar 
3631979b511SNavdeep Parhar 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4btxq");
364*ec8004ddSNavdeep Parhar 	if (rc == 0) {
3651979b511SNavdeep Parhar 		fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
3661979b511SNavdeep Parhar 		    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH) |
3671979b511SNavdeep Parhar 		    V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
3681979b511SNavdeep Parhar 		fw_class = idx < 0 ? 0xffffffff : idx;
369*ec8004ddSNavdeep Parhar 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_mnem,
370*ec8004ddSNavdeep Parhar 		    &fw_class);
3711979b511SNavdeep Parhar 		end_synchronized_op(sc, 0);
372*ec8004ddSNavdeep Parhar 	}
3731979b511SNavdeep Parhar 
3741979b511SNavdeep Parhar 	mtx_lock(&sc->tc_lock);
3751979b511SNavdeep Parhar 	MPASS(txq->tc_idx == -2);
3761979b511SNavdeep Parhar 	if (rc == 0) {
3771979b511SNavdeep Parhar 		/*
3781979b511SNavdeep Parhar 		 * Unbind, bind, or bind to a different class succeeded.  Remove
3791979b511SNavdeep Parhar 		 * the reference on the old traffic class, if any.
3801979b511SNavdeep Parhar 		 */
3811979b511SNavdeep Parhar 		if (old_idx != -1) {
3821979b511SNavdeep Parhar 			tc = &tc0[old_idx];
3831979b511SNavdeep Parhar 			MPASS(tc->refcount > 0);
3841979b511SNavdeep Parhar 			tc->refcount--;
3851979b511SNavdeep Parhar 		}
3861979b511SNavdeep Parhar 		txq->tc_idx = idx;
3871979b511SNavdeep Parhar 	} else {
3881979b511SNavdeep Parhar 		/*
3891979b511SNavdeep Parhar 		 * Unbind, bind, or bind to a different class failed.  Remove
3901979b511SNavdeep Parhar 		 * the anticipatory reference on the new traffic class, if any.
3911979b511SNavdeep Parhar 		 */
3921979b511SNavdeep Parhar 		if (idx != -1) {
3931979b511SNavdeep Parhar 			tc = &tc0[idx];
3941979b511SNavdeep Parhar 			MPASS(tc->refcount > 0);
3951979b511SNavdeep Parhar 			tc->refcount--;
3961979b511SNavdeep Parhar 		}
3971979b511SNavdeep Parhar 		txq->tc_idx = old_idx;
3981979b511SNavdeep Parhar 	}
3991979b511SNavdeep Parhar done:
4006beb67c7SNavdeep Parhar 	MPASS(txq->tc_idx >= -1 && txq->tc_idx < sc->params.nsched_cls);
4011979b511SNavdeep Parhar 	mtx_unlock(&sc->tc_lock);
4021979b511SNavdeep Parhar 	return (rc);
4031979b511SNavdeep Parhar }
4041979b511SNavdeep Parhar 
4052204b427SNavdeep Parhar int
4062204b427SNavdeep Parhar t4_set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
4072204b427SNavdeep Parhar {
4082204b427SNavdeep Parhar 	struct port_info *pi = NULL;
4092204b427SNavdeep Parhar 	struct vi_info *vi;
4102204b427SNavdeep Parhar 	struct sge_txq *txq;
4112204b427SNavdeep Parhar 	int i, rc;
4122204b427SNavdeep Parhar 
4131979b511SNavdeep Parhar 	if (p->port >= sc->params.nports)
4141979b511SNavdeep Parhar 		return (EINVAL);
4152204b427SNavdeep Parhar 
4161979b511SNavdeep Parhar 	/*
4171979b511SNavdeep Parhar 	 * XXX: cxgbetool allows the user to specify the physical port only.  So
4181979b511SNavdeep Parhar 	 * we always operate on the main VI.
4191979b511SNavdeep Parhar 	 */
4202204b427SNavdeep Parhar 	pi = sc->port[p->port];
4212204b427SNavdeep Parhar 	vi = &pi->vi[0];
4221979b511SNavdeep Parhar 
4231979b511SNavdeep Parhar 	/* Checking VI_INIT_DONE outside a synch-op is a harmless race here. */
4241979b511SNavdeep Parhar 	if (!(vi->flags & VI_INIT_DONE))
4251979b511SNavdeep Parhar 		return (EAGAIN);
426eb6f5d6eSNavdeep Parhar 	MPASS(vi->ntxq > 0);
4272204b427SNavdeep Parhar 
4282204b427SNavdeep Parhar 	if (!in_range(p->queue, 0, vi->ntxq - 1) ||
4296beb67c7SNavdeep Parhar 	    !in_range(p->cl, 0, sc->params.nsched_cls - 1))
4301979b511SNavdeep Parhar 		return (EINVAL);
4312204b427SNavdeep Parhar 
4321979b511SNavdeep Parhar 	if (p->queue < 0) {
4332204b427SNavdeep Parhar 		/*
4342204b427SNavdeep Parhar 		 * Change the scheduling on all the TX queues for the
4352204b427SNavdeep Parhar 		 * interface.
4362204b427SNavdeep Parhar 		 */
4372204b427SNavdeep Parhar 		for_each_txq(vi, i, txq) {
4381979b511SNavdeep Parhar 			rc = bind_txq_to_traffic_class(sc, txq, p->cl);
4391979b511SNavdeep Parhar 			if (rc != 0)
4401979b511SNavdeep Parhar 				break;
4411979b511SNavdeep Parhar 		}
4421979b511SNavdeep Parhar 	} else {
4431979b511SNavdeep Parhar 		/*
4441979b511SNavdeep Parhar 		 * If op.queue is non-negative, then we're only changing the
4451979b511SNavdeep Parhar 		 * scheduling on a single specified TX queue.
4461979b511SNavdeep Parhar 		 */
4471979b511SNavdeep Parhar 		txq = &sc->sge.txq[vi->first_txq + p->queue];
4481979b511SNavdeep Parhar 		rc = bind_txq_to_traffic_class(sc, txq, p->cl);
4492204b427SNavdeep Parhar 	}
4502204b427SNavdeep Parhar 
4512204b427SNavdeep Parhar 	return (rc);
4522204b427SNavdeep Parhar }
4532204b427SNavdeep Parhar 
4542204b427SNavdeep Parhar int
4552204b427SNavdeep Parhar t4_init_tx_sched(struct adapter *sc)
4562204b427SNavdeep Parhar {
457*ec8004ddSNavdeep Parhar 	int i;
4586beb67c7SNavdeep Parhar 	const int n = sc->params.nsched_cls;
4592204b427SNavdeep Parhar 	struct port_info *pi;
4602204b427SNavdeep Parhar 
4612204b427SNavdeep Parhar 	mtx_init(&sc->tc_lock, "tx_sched lock", NULL, MTX_DEF);
4622204b427SNavdeep Parhar 	TASK_INIT(&sc->tc_task, 0, update_tx_sched, sc);
4632204b427SNavdeep Parhar 	for_each_port(sc, i) {
4642204b427SNavdeep Parhar 		pi = sc->port[i];
4652204b427SNavdeep Parhar 		pi->sched_params = malloc(sizeof(*pi->sched_params) +
466*ec8004ddSNavdeep Parhar 		    n * sizeof(struct tx_cl_rl_params), M_CXGBE, M_ZERO | M_WAITOK);
4672204b427SNavdeep Parhar 	}
4682204b427SNavdeep Parhar 
4692204b427SNavdeep Parhar 	return (0);
4702204b427SNavdeep Parhar }
4712204b427SNavdeep Parhar 
4722204b427SNavdeep Parhar int
4732204b427SNavdeep Parhar t4_free_tx_sched(struct adapter *sc)
4742204b427SNavdeep Parhar {
4752204b427SNavdeep Parhar 	int i;
4762204b427SNavdeep Parhar 
4772204b427SNavdeep Parhar 	taskqueue_drain(taskqueue_thread, &sc->tc_task);
4782204b427SNavdeep Parhar 
479cc2050c5SNavdeep Parhar 	for_each_port(sc, i) {
480cc2050c5SNavdeep Parhar 		if (sc->port[i] != NULL)
4812204b427SNavdeep Parhar 			free(sc->port[i]->sched_params, M_CXGBE);
482cc2050c5SNavdeep Parhar 	}
4832204b427SNavdeep Parhar 
4842204b427SNavdeep Parhar 	if (mtx_initialized(&sc->tc_lock))
4852204b427SNavdeep Parhar 		mtx_destroy(&sc->tc_lock);
4862204b427SNavdeep Parhar 
4872204b427SNavdeep Parhar 	return (0);
4882204b427SNavdeep Parhar }
4892204b427SNavdeep Parhar 
4902204b427SNavdeep Parhar void
4912204b427SNavdeep Parhar t4_update_tx_sched(struct adapter *sc)
4922204b427SNavdeep Parhar {
4932204b427SNavdeep Parhar 
4942204b427SNavdeep Parhar 	taskqueue_enqueue(taskqueue_thread, &sc->tc_task);
4952204b427SNavdeep Parhar }
4962204b427SNavdeep Parhar 
4972204b427SNavdeep Parhar int
4982204b427SNavdeep Parhar t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate,
4992204b427SNavdeep Parhar     int *tc_idx)
5002204b427SNavdeep Parhar {
501*ec8004ddSNavdeep Parhar 	int rc = 0, fa, fa2, i, pktsize, burstsize;
5021979b511SNavdeep Parhar 	bool update;
5032204b427SNavdeep Parhar 	struct tx_cl_rl_params *tc;
5041979b511SNavdeep Parhar 	struct port_info *pi;
5052204b427SNavdeep Parhar 
5062204b427SNavdeep Parhar 	MPASS(port_id >= 0 && port_id < sc->params.nports);
5072204b427SNavdeep Parhar 
5081979b511SNavdeep Parhar 	pi = sc->port[port_id];
50909a7189fSNavdeep Parhar 	if (pi->sched_params->pktsize > 0)
51009a7189fSNavdeep Parhar 		pktsize = pi->sched_params->pktsize;
51109a7189fSNavdeep Parhar 	else
51209a7189fSNavdeep Parhar 		pktsize = pi->vi[0].ifp->if_mtu;
51309a7189fSNavdeep Parhar 	if (pi->sched_params->burstsize > 0)
51409a7189fSNavdeep Parhar 		burstsize = pi->sched_params->burstsize;
51509a7189fSNavdeep Parhar 	else
51609a7189fSNavdeep Parhar 		burstsize = pktsize * 4;
5171979b511SNavdeep Parhar 	tc = &pi->sched_params->cl_rl[0];
51809a7189fSNavdeep Parhar 
5191979b511SNavdeep Parhar 	update = false;
520*ec8004ddSNavdeep Parhar 	fa = fa2 = -1;
5212204b427SNavdeep Parhar 	mtx_lock(&sc->tc_lock);
5226beb67c7SNavdeep Parhar 	for (i = 0; i < sc->params.nsched_cls; i++, tc++) {
523*ec8004ddSNavdeep Parhar 		if (tc->state >= CS_PARAMS_SET &&
524*ec8004ddSNavdeep Parhar 		    tc->ratemode == FW_SCHED_PARAMS_RATE_ABS &&
5252204b427SNavdeep Parhar 		    tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE &&
5262204b427SNavdeep Parhar 		    tc->mode == FW_SCHED_PARAMS_MODE_FLOW &&
52709a7189fSNavdeep Parhar 		    tc->maxrate == maxrate && tc->pktsize == pktsize &&
52809a7189fSNavdeep Parhar 		    tc->burstsize == burstsize) {
5292204b427SNavdeep Parhar 			tc->refcount++;
5302204b427SNavdeep Parhar 			*tc_idx = i;
531*ec8004ddSNavdeep Parhar 			if (tc->state == CS_PARAMS_SET) {
532*ec8004ddSNavdeep Parhar 				tc->state = CS_HW_UPDATE_REQUESTED;
5331979b511SNavdeep Parhar 				update = true;
5341979b511SNavdeep Parhar 			}
5352204b427SNavdeep Parhar 			goto done;
5362204b427SNavdeep Parhar 		}
537*ec8004ddSNavdeep Parhar 
538*ec8004ddSNavdeep Parhar 		if (fa < 0 && tc->state == CS_UNINITIALIZED) {
539*ec8004ddSNavdeep Parhar 			MPASS(tc->refcount == 0);
540*ec8004ddSNavdeep Parhar 			fa = i;		/* first available, never used. */
541*ec8004ddSNavdeep Parhar 		}
542*ec8004ddSNavdeep Parhar 		if (fa2 < 0 && tc->refcount == 0 && !(tc->flags & CF_USER)) {
543*ec8004ddSNavdeep Parhar 			fa2 = i;	/* first available, used previously.  */
544*ec8004ddSNavdeep Parhar 		}
5452204b427SNavdeep Parhar 	}
5462204b427SNavdeep Parhar 	/* Not found */
5476beb67c7SNavdeep Parhar 	MPASS(i == sc->params.nsched_cls);
548*ec8004ddSNavdeep Parhar 	if (fa == -1)
549*ec8004ddSNavdeep Parhar 		fa = fa2;
550*ec8004ddSNavdeep Parhar 	if (fa == -1) {
551*ec8004ddSNavdeep Parhar 		*tc_idx = -1;
552*ec8004ddSNavdeep Parhar 		rc = ENOSPC;
553*ec8004ddSNavdeep Parhar 	} else {
554*ec8004ddSNavdeep Parhar 		MPASS(fa >= 0 && fa < sc->params.nsched_cls);
5551979b511SNavdeep Parhar 		tc = &pi->sched_params->cl_rl[fa];
556*ec8004ddSNavdeep Parhar 		MPASS(!(tc->flags & CF_USER));
557*ec8004ddSNavdeep Parhar 		MPASS(tc->refcount == 0);
558*ec8004ddSNavdeep Parhar 
5592204b427SNavdeep Parhar 		tc->refcount = 1;
560*ec8004ddSNavdeep Parhar 		tc->state = CS_HW_UPDATE_REQUESTED;
5612204b427SNavdeep Parhar 		tc->ratemode = FW_SCHED_PARAMS_RATE_ABS;
5622204b427SNavdeep Parhar 		tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
5632204b427SNavdeep Parhar 		tc->mode = FW_SCHED_PARAMS_MODE_FLOW;
5642204b427SNavdeep Parhar 		tc->maxrate = maxrate;
56509a7189fSNavdeep Parhar 		tc->pktsize = pktsize;
56609a7189fSNavdeep Parhar 		tc->burstsize = burstsize;
5672204b427SNavdeep Parhar 		*tc_idx = fa;
5681979b511SNavdeep Parhar 		update = true;
5692204b427SNavdeep Parhar 	}
5702204b427SNavdeep Parhar done:
5712204b427SNavdeep Parhar 	mtx_unlock(&sc->tc_lock);
572*ec8004ddSNavdeep Parhar 	if (update)
5731979b511SNavdeep Parhar 		t4_update_tx_sched(sc);
5742204b427SNavdeep Parhar 	return (rc);
5752204b427SNavdeep Parhar }
5762204b427SNavdeep Parhar 
5772204b427SNavdeep Parhar void
5781979b511SNavdeep Parhar t4_release_cl_rl(struct adapter *sc, int port_id, int tc_idx)
5792204b427SNavdeep Parhar {
5802204b427SNavdeep Parhar 	struct tx_cl_rl_params *tc;
5812204b427SNavdeep Parhar 
5822204b427SNavdeep Parhar 	MPASS(port_id >= 0 && port_id < sc->params.nports);
5836beb67c7SNavdeep Parhar 	MPASS(tc_idx >= 0 && tc_idx < sc->params.nsched_cls);
5842204b427SNavdeep Parhar 
5852204b427SNavdeep Parhar 	mtx_lock(&sc->tc_lock);
5862204b427SNavdeep Parhar 	tc = &sc->port[port_id]->sched_params->cl_rl[tc_idx];
5872204b427SNavdeep Parhar 	MPASS(tc->refcount > 0);
5882204b427SNavdeep Parhar 	tc->refcount--;
5892204b427SNavdeep Parhar 	mtx_unlock(&sc->tc_lock);
5902204b427SNavdeep Parhar }
59167e07112SNavdeep Parhar 
5921979b511SNavdeep Parhar int
5931979b511SNavdeep Parhar sysctl_tc(SYSCTL_HANDLER_ARGS)
5941979b511SNavdeep Parhar {
5951979b511SNavdeep Parhar 	struct vi_info *vi = arg1;
59643bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
5971979b511SNavdeep Parhar 	struct sge_txq *txq;
5981979b511SNavdeep Parhar 	int qidx = arg2, rc, tc_idx;
5991979b511SNavdeep Parhar 
60043bbae19SNavdeep Parhar 	MPASS(qidx >= vi->first_txq && qidx < vi->first_txq + vi->ntxq);
6011979b511SNavdeep Parhar 
60243bbae19SNavdeep Parhar 	txq = &sc->sge.txq[qidx];
6031979b511SNavdeep Parhar 	tc_idx = txq->tc_idx;
6041979b511SNavdeep Parhar 	rc = sysctl_handle_int(oidp, &tc_idx, 0, req);
6051979b511SNavdeep Parhar 	if (rc != 0 || req->newptr == NULL)
6061979b511SNavdeep Parhar 		return (rc);
6071979b511SNavdeep Parhar 
6081979b511SNavdeep Parhar 	if (sc->flags & IS_VF)
6091979b511SNavdeep Parhar 		return (EPERM);
6106beb67c7SNavdeep Parhar 	if (!in_range(tc_idx, 0, sc->params.nsched_cls - 1))
6111979b511SNavdeep Parhar 		return (EINVAL);
6121979b511SNavdeep Parhar 
6131979b511SNavdeep Parhar 	return (bind_txq_to_traffic_class(sc, txq, tc_idx));
6141979b511SNavdeep Parhar }
6151979b511SNavdeep Parhar 
6161979b511SNavdeep Parhar int
6171979b511SNavdeep Parhar sysctl_tc_params(SYSCTL_HANDLER_ARGS)
6181979b511SNavdeep Parhar {
6191979b511SNavdeep Parhar 	struct adapter *sc = arg1;
6201979b511SNavdeep Parhar 	struct tx_cl_rl_params tc;
6211979b511SNavdeep Parhar 	struct sbuf *sb;
6221979b511SNavdeep Parhar 	int i, rc, port_id, mbps, gbps;
6231979b511SNavdeep Parhar 
6241979b511SNavdeep Parhar 	rc = sysctl_wire_old_buffer(req, 0);
6251979b511SNavdeep Parhar 	if (rc != 0)
6261979b511SNavdeep Parhar 		return (rc);
6271979b511SNavdeep Parhar 
6281979b511SNavdeep Parhar 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6291979b511SNavdeep Parhar 	if (sb == NULL)
6301979b511SNavdeep Parhar 		return (ENOMEM);
6311979b511SNavdeep Parhar 
6321979b511SNavdeep Parhar 	port_id = arg2 >> 16;
6331979b511SNavdeep Parhar 	MPASS(port_id < sc->params.nports);
6341979b511SNavdeep Parhar 	MPASS(sc->port[port_id] != NULL);
6351979b511SNavdeep Parhar 	i = arg2 & 0xffff;
6366beb67c7SNavdeep Parhar 	MPASS(i < sc->params.nsched_cls);
6371979b511SNavdeep Parhar 
6381979b511SNavdeep Parhar 	mtx_lock(&sc->tc_lock);
6391979b511SNavdeep Parhar 	tc = sc->port[port_id]->sched_params->cl_rl[i];
6401979b511SNavdeep Parhar 	mtx_unlock(&sc->tc_lock);
6411979b511SNavdeep Parhar 
642*ec8004ddSNavdeep Parhar 	if (tc.state < CS_PARAMS_SET) {
643*ec8004ddSNavdeep Parhar 		sbuf_printf(sb, "uninitialized");
644*ec8004ddSNavdeep Parhar 		goto done;
645*ec8004ddSNavdeep Parhar 	}
646*ec8004ddSNavdeep Parhar 
6471979b511SNavdeep Parhar 	switch (tc.rateunit) {
6481979b511SNavdeep Parhar 	case SCHED_CLASS_RATEUNIT_BITS:
6491979b511SNavdeep Parhar 		switch (tc.ratemode) {
6501979b511SNavdeep Parhar 		case SCHED_CLASS_RATEMODE_REL:
6511979b511SNavdeep Parhar 			/* XXX: top speed or actual link speed? */
6521979b511SNavdeep Parhar 			gbps = port_top_speed(sc->port[port_id]);
6531979b511SNavdeep Parhar 			sbuf_printf(sb, "%u%% of %uGbps", tc.maxrate, gbps);
6541979b511SNavdeep Parhar 			break;
6551979b511SNavdeep Parhar 		case SCHED_CLASS_RATEMODE_ABS:
6561979b511SNavdeep Parhar 			mbps = tc.maxrate / 1000;
6571979b511SNavdeep Parhar 			gbps = tc.maxrate / 1000000;
6581979b511SNavdeep Parhar 			if (tc.maxrate == gbps * 1000000)
6591979b511SNavdeep Parhar 				sbuf_printf(sb, "%uGbps", gbps);
6601979b511SNavdeep Parhar 			else if (tc.maxrate == mbps * 1000)
6611979b511SNavdeep Parhar 				sbuf_printf(sb, "%uMbps", mbps);
6621979b511SNavdeep Parhar 			else
6631979b511SNavdeep Parhar 				sbuf_printf(sb, "%uKbps", tc.maxrate);
6641979b511SNavdeep Parhar 			break;
6651979b511SNavdeep Parhar 		default:
6661979b511SNavdeep Parhar 			rc = ENXIO;
6671979b511SNavdeep Parhar 			goto done;
6681979b511SNavdeep Parhar 		}
6691979b511SNavdeep Parhar 		break;
6701979b511SNavdeep Parhar 	case SCHED_CLASS_RATEUNIT_PKTS:
6711979b511SNavdeep Parhar 		sbuf_printf(sb, "%upps", tc.maxrate);
6721979b511SNavdeep Parhar 		break;
6731979b511SNavdeep Parhar 	default:
6741979b511SNavdeep Parhar 		rc = ENXIO;
6751979b511SNavdeep Parhar 		goto done;
6761979b511SNavdeep Parhar 	}
6771979b511SNavdeep Parhar 
6781979b511SNavdeep Parhar 	switch (tc.mode) {
6791979b511SNavdeep Parhar 	case SCHED_CLASS_MODE_CLASS:
680*ec8004ddSNavdeep Parhar 		/* Note that pktsize and burstsize are not used in this mode. */
6811979b511SNavdeep Parhar 		sbuf_printf(sb, " aggregate");
6821979b511SNavdeep Parhar 		break;
6831979b511SNavdeep Parhar 	case SCHED_CLASS_MODE_FLOW:
6841979b511SNavdeep Parhar 		sbuf_printf(sb, " per-flow");
6858a684e1fSNavdeep Parhar 		if (tc.pktsize > 0)
6868a684e1fSNavdeep Parhar 			sbuf_printf(sb, " pkt-size %u", tc.pktsize);
6878a684e1fSNavdeep Parhar 		if (tc.burstsize > 0)
6888a684e1fSNavdeep Parhar 			sbuf_printf(sb, " burst-size %u", tc.burstsize);
6891979b511SNavdeep Parhar 		break;
6901979b511SNavdeep Parhar 	default:
6911979b511SNavdeep Parhar 		rc = ENXIO;
6921979b511SNavdeep Parhar 		goto done;
6931979b511SNavdeep Parhar 	}
6941979b511SNavdeep Parhar 
6951979b511SNavdeep Parhar done:
6961979b511SNavdeep Parhar 	if (rc == 0)
6971979b511SNavdeep Parhar 		rc = sbuf_finish(sb);
6981979b511SNavdeep Parhar 	sbuf_delete(sb);
6991979b511SNavdeep Parhar 
7001979b511SNavdeep Parhar 	return (rc);
7011979b511SNavdeep Parhar }
7021979b511SNavdeep Parhar 
70367e07112SNavdeep Parhar #ifdef RATELIMIT
70467e07112SNavdeep Parhar void
70567e07112SNavdeep Parhar t4_init_etid_table(struct adapter *sc)
70667e07112SNavdeep Parhar {
70767e07112SNavdeep Parhar 	int i;
70867e07112SNavdeep Parhar 	struct tid_info *t;
70967e07112SNavdeep Parhar 
71067e07112SNavdeep Parhar 	if (!is_ethoffload(sc))
71167e07112SNavdeep Parhar 		return;
71267e07112SNavdeep Parhar 
71367e07112SNavdeep Parhar 	t = &sc->tids;
71467e07112SNavdeep Parhar 	MPASS(t->netids > 0);
71567e07112SNavdeep Parhar 
71667e07112SNavdeep Parhar 	mtx_init(&t->etid_lock, "etid lock", NULL, MTX_DEF);
71767e07112SNavdeep Parhar 	t->etid_tab = malloc(sizeof(*t->etid_tab) * t->netids, M_CXGBE,
71867e07112SNavdeep Parhar 			M_ZERO | M_WAITOK);
71967e07112SNavdeep Parhar 	t->efree = t->etid_tab;
72067e07112SNavdeep Parhar 	t->etids_in_use = 0;
72167e07112SNavdeep Parhar 	for (i = 1; i < t->netids; i++)
72267e07112SNavdeep Parhar 		t->etid_tab[i - 1].next = &t->etid_tab[i];
72367e07112SNavdeep Parhar 	t->etid_tab[t->netids - 1].next = NULL;
72467e07112SNavdeep Parhar }
72567e07112SNavdeep Parhar 
72667e07112SNavdeep Parhar void
72767e07112SNavdeep Parhar t4_free_etid_table(struct adapter *sc)
72867e07112SNavdeep Parhar {
72967e07112SNavdeep Parhar 	struct tid_info *t;
73067e07112SNavdeep Parhar 
73167e07112SNavdeep Parhar 	if (!is_ethoffload(sc))
73267e07112SNavdeep Parhar 		return;
73367e07112SNavdeep Parhar 
73467e07112SNavdeep Parhar 	t = &sc->tids;
73567e07112SNavdeep Parhar 	MPASS(t->netids > 0);
73667e07112SNavdeep Parhar 
73767e07112SNavdeep Parhar 	free(t->etid_tab, M_CXGBE);
73867e07112SNavdeep Parhar 	t->etid_tab = NULL;
73967e07112SNavdeep Parhar 
74067e07112SNavdeep Parhar 	if (mtx_initialized(&t->etid_lock))
74167e07112SNavdeep Parhar 		mtx_destroy(&t->etid_lock);
74267e07112SNavdeep Parhar }
74367e07112SNavdeep Parhar 
74467e07112SNavdeep Parhar /* etid services */
745e38a50e8SJohn Baldwin static int alloc_etid(struct adapter *, struct cxgbe_rate_tag *);
74667e07112SNavdeep Parhar static void free_etid(struct adapter *, int);
74767e07112SNavdeep Parhar 
74867e07112SNavdeep Parhar static int
749e38a50e8SJohn Baldwin alloc_etid(struct adapter *sc, struct cxgbe_rate_tag *cst)
75067e07112SNavdeep Parhar {
75167e07112SNavdeep Parhar 	struct tid_info *t = &sc->tids;
75267e07112SNavdeep Parhar 	int etid = -1;
75367e07112SNavdeep Parhar 
75467e07112SNavdeep Parhar 	mtx_lock(&t->etid_lock);
75567e07112SNavdeep Parhar 	if (t->efree) {
75667e07112SNavdeep Parhar 		union etid_entry *p = t->efree;
75767e07112SNavdeep Parhar 
75867e07112SNavdeep Parhar 		etid = p - t->etid_tab + t->etid_base;
75967e07112SNavdeep Parhar 		t->efree = p->next;
76067e07112SNavdeep Parhar 		p->cst = cst;
76167e07112SNavdeep Parhar 		t->etids_in_use++;
76267e07112SNavdeep Parhar 	}
76367e07112SNavdeep Parhar 	mtx_unlock(&t->etid_lock);
76467e07112SNavdeep Parhar 	return (etid);
76567e07112SNavdeep Parhar }
76667e07112SNavdeep Parhar 
767e38a50e8SJohn Baldwin struct cxgbe_rate_tag *
76867e07112SNavdeep Parhar lookup_etid(struct adapter *sc, int etid)
76967e07112SNavdeep Parhar {
77067e07112SNavdeep Parhar 	struct tid_info *t = &sc->tids;
77167e07112SNavdeep Parhar 
77267e07112SNavdeep Parhar 	return (t->etid_tab[etid - t->etid_base].cst);
77367e07112SNavdeep Parhar }
77467e07112SNavdeep Parhar 
77567e07112SNavdeep Parhar static void
77667e07112SNavdeep Parhar free_etid(struct adapter *sc, int etid)
77767e07112SNavdeep Parhar {
77867e07112SNavdeep Parhar 	struct tid_info *t = &sc->tids;
77967e07112SNavdeep Parhar 	union etid_entry *p = &t->etid_tab[etid - t->etid_base];
78067e07112SNavdeep Parhar 
78167e07112SNavdeep Parhar 	mtx_lock(&t->etid_lock);
78267e07112SNavdeep Parhar 	p->next = t->efree;
78367e07112SNavdeep Parhar 	t->efree = p;
78467e07112SNavdeep Parhar 	t->etids_in_use--;
78567e07112SNavdeep Parhar 	mtx_unlock(&t->etid_lock);
78667e07112SNavdeep Parhar }
78767e07112SNavdeep Parhar 
78867e07112SNavdeep Parhar int
789e38a50e8SJohn Baldwin cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
79067e07112SNavdeep Parhar     struct m_snd_tag **pt)
79167e07112SNavdeep Parhar {
79267e07112SNavdeep Parhar 	int rc, schedcl;
79367e07112SNavdeep Parhar 	struct vi_info *vi = ifp->if_softc;
79467e07112SNavdeep Parhar 	struct port_info *pi = vi->pi;
79567e07112SNavdeep Parhar 	struct adapter *sc = pi->adapter;
796e38a50e8SJohn Baldwin 	struct cxgbe_rate_tag *cst;
79767e07112SNavdeep Parhar 
798e38a50e8SJohn Baldwin 	MPASS(params->hdr.type == IF_SND_TAG_TYPE_RATE_LIMIT);
79967e07112SNavdeep Parhar 
80067e07112SNavdeep Parhar 	rc = t4_reserve_cl_rl_kbps(sc, pi->port_id,
80167e07112SNavdeep Parhar 	    (params->rate_limit.max_rate * 8ULL / 1000), &schedcl);
80267e07112SNavdeep Parhar 	if (rc != 0)
80367e07112SNavdeep Parhar 		return (rc);
8046beb67c7SNavdeep Parhar 	MPASS(schedcl >= 0 && schedcl < sc->params.nsched_cls);
80567e07112SNavdeep Parhar 
80667e07112SNavdeep Parhar 	cst = malloc(sizeof(*cst), M_CXGBE, M_ZERO | M_NOWAIT);
80767e07112SNavdeep Parhar 	if (cst == NULL) {
80867e07112SNavdeep Parhar failed:
8091979b511SNavdeep Parhar 		t4_release_cl_rl(sc, pi->port_id, schedcl);
81067e07112SNavdeep Parhar 		return (ENOMEM);
81167e07112SNavdeep Parhar 	}
81267e07112SNavdeep Parhar 
81367e07112SNavdeep Parhar 	cst->etid = alloc_etid(sc, cst);
81467e07112SNavdeep Parhar 	if (cst->etid < 0) {
81567e07112SNavdeep Parhar 		free(cst, M_CXGBE);
81667e07112SNavdeep Parhar 		goto failed;
81767e07112SNavdeep Parhar 	}
81867e07112SNavdeep Parhar 
81967e07112SNavdeep Parhar 	mtx_init(&cst->lock, "cst_lock", NULL, MTX_DEF);
820786099deSNavdeep Parhar 	mbufq_init(&cst->pending_tx, INT_MAX);
821786099deSNavdeep Parhar 	mbufq_init(&cst->pending_fwack, INT_MAX);
82256fb710fSJohn Baldwin 	m_snd_tag_init(&cst->com, ifp, IF_SND_TAG_TYPE_RATE_LIMIT);
823786099deSNavdeep Parhar 	cst->flags |= EO_FLOWC_PENDING | EO_SND_TAG_REF;
82467e07112SNavdeep Parhar 	cst->adapter = sc;
82567e07112SNavdeep Parhar 	cst->port_id = pi->port_id;
82667e07112SNavdeep Parhar 	cst->schedcl = schedcl;
82767e07112SNavdeep Parhar 	cst->max_rate = params->rate_limit.max_rate;
828ac8ec5feSNavdeep Parhar 	cst->tx_credits = sc->params.eo_wr_cred;
82967e07112SNavdeep Parhar 	cst->tx_total = cst->tx_credits;
830786099deSNavdeep Parhar 	cst->plen = 0;
831c0236bd9SNavdeep Parhar 	cst->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
832edb518f4SNavdeep Parhar 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) |
833edb518f4SNavdeep Parhar 	    V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
83467e07112SNavdeep Parhar 
83567e07112SNavdeep Parhar 	/*
83667e07112SNavdeep Parhar 	 * Queues will be selected later when the connection flowid is available.
83767e07112SNavdeep Parhar 	 */
83867e07112SNavdeep Parhar 
83956fb710fSJohn Baldwin 	*pt = &cst->com;
84067e07112SNavdeep Parhar 	return (0);
84167e07112SNavdeep Parhar }
84267e07112SNavdeep Parhar 
84367e07112SNavdeep Parhar /*
84467e07112SNavdeep Parhar  * Change in parameters, no change in ifp.
84567e07112SNavdeep Parhar  */
84667e07112SNavdeep Parhar int
847e38a50e8SJohn Baldwin cxgbe_rate_tag_modify(struct m_snd_tag *mst,
84867e07112SNavdeep Parhar     union if_snd_tag_modify_params *params)
84967e07112SNavdeep Parhar {
85067e07112SNavdeep Parhar 	int rc, schedcl;
851e38a50e8SJohn Baldwin 	struct cxgbe_rate_tag *cst = mst_to_crt(mst);
85267e07112SNavdeep Parhar 	struct adapter *sc = cst->adapter;
85367e07112SNavdeep Parhar 
85467e07112SNavdeep Parhar 	/* XXX: is schedcl -1 ok here? */
8556beb67c7SNavdeep Parhar 	MPASS(cst->schedcl >= 0 && cst->schedcl < sc->params.nsched_cls);
85667e07112SNavdeep Parhar 
857786099deSNavdeep Parhar 	mtx_lock(&cst->lock);
858786099deSNavdeep Parhar 	MPASS(cst->flags & EO_SND_TAG_REF);
85967e07112SNavdeep Parhar 	rc = t4_reserve_cl_rl_kbps(sc, cst->port_id,
86067e07112SNavdeep Parhar 	    (params->rate_limit.max_rate * 8ULL / 1000), &schedcl);
86167e07112SNavdeep Parhar 	if (rc != 0)
86267e07112SNavdeep Parhar 		return (rc);
8636beb67c7SNavdeep Parhar 	MPASS(schedcl >= 0 && schedcl < sc->params.nsched_cls);
8641979b511SNavdeep Parhar 	t4_release_cl_rl(sc, cst->port_id, cst->schedcl);
86567e07112SNavdeep Parhar 	cst->schedcl = schedcl;
86667e07112SNavdeep Parhar 	cst->max_rate = params->rate_limit.max_rate;
867786099deSNavdeep Parhar 	mtx_unlock(&cst->lock);
86867e07112SNavdeep Parhar 
86967e07112SNavdeep Parhar 	return (0);
87067e07112SNavdeep Parhar }
87167e07112SNavdeep Parhar 
87267e07112SNavdeep Parhar int
873e38a50e8SJohn Baldwin cxgbe_rate_tag_query(struct m_snd_tag *mst,
87467e07112SNavdeep Parhar     union if_snd_tag_query_params *params)
87567e07112SNavdeep Parhar {
876e38a50e8SJohn Baldwin 	struct cxgbe_rate_tag *cst = mst_to_crt(mst);
87767e07112SNavdeep Parhar 
87867e07112SNavdeep Parhar 	params->rate_limit.max_rate = cst->max_rate;
87967e07112SNavdeep Parhar 
88067e07112SNavdeep Parhar #define CST_TO_MST_QLEVEL_SCALE (IF_SND_QUEUE_LEVEL_MAX / cst->tx_total)
88167e07112SNavdeep Parhar 	params->rate_limit.queue_level =
88267e07112SNavdeep Parhar 		(cst->tx_total - cst->tx_credits) * CST_TO_MST_QLEVEL_SCALE;
88367e07112SNavdeep Parhar 
88467e07112SNavdeep Parhar 	return (0);
88567e07112SNavdeep Parhar }
88667e07112SNavdeep Parhar 
887786099deSNavdeep Parhar /*
888786099deSNavdeep Parhar  * Unlocks cst and frees it.
889786099deSNavdeep Parhar  */
89067e07112SNavdeep Parhar void
891e38a50e8SJohn Baldwin cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *cst)
89267e07112SNavdeep Parhar {
89367e07112SNavdeep Parhar 	struct adapter *sc = cst->adapter;
89467e07112SNavdeep Parhar 
895786099deSNavdeep Parhar 	mtx_assert(&cst->lock, MA_OWNED);
896786099deSNavdeep Parhar 	MPASS((cst->flags & EO_SND_TAG_REF) == 0);
897786099deSNavdeep Parhar 	MPASS(cst->tx_credits == cst->tx_total);
898786099deSNavdeep Parhar 	MPASS(cst->plen == 0);
899786099deSNavdeep Parhar 	MPASS(mbufq_first(&cst->pending_tx) == NULL);
900786099deSNavdeep Parhar 	MPASS(mbufq_first(&cst->pending_fwack) == NULL);
901786099deSNavdeep Parhar 
90267e07112SNavdeep Parhar 	if (cst->etid >= 0)
90367e07112SNavdeep Parhar 		free_etid(sc, cst->etid);
90467e07112SNavdeep Parhar 	if (cst->schedcl != -1)
9051979b511SNavdeep Parhar 		t4_release_cl_rl(sc, cst->port_id, cst->schedcl);
906786099deSNavdeep Parhar 	mtx_unlock(&cst->lock);
90767e07112SNavdeep Parhar 	mtx_destroy(&cst->lock);
90867e07112SNavdeep Parhar 	free(cst, M_CXGBE);
90967e07112SNavdeep Parhar }
910786099deSNavdeep Parhar 
911786099deSNavdeep Parhar void
912e38a50e8SJohn Baldwin cxgbe_rate_tag_free(struct m_snd_tag *mst)
913786099deSNavdeep Parhar {
914e38a50e8SJohn Baldwin 	struct cxgbe_rate_tag *cst = mst_to_crt(mst);
915786099deSNavdeep Parhar 
916786099deSNavdeep Parhar 	mtx_lock(&cst->lock);
917786099deSNavdeep Parhar 
918786099deSNavdeep Parhar 	/* The kernel is done with the snd_tag.  Remove its reference. */
919786099deSNavdeep Parhar 	MPASS(cst->flags & EO_SND_TAG_REF);
920786099deSNavdeep Parhar 	cst->flags &= ~EO_SND_TAG_REF;
921786099deSNavdeep Parhar 
922786099deSNavdeep Parhar 	if (cst->ncompl == 0) {
923786099deSNavdeep Parhar 		/*
924786099deSNavdeep Parhar 		 * No fw4_ack in flight.  Free the tag right away if there are
925786099deSNavdeep Parhar 		 * no outstanding credits.  Request the firmware to return all
926786099deSNavdeep Parhar 		 * credits for the etid otherwise.
927786099deSNavdeep Parhar 		 */
928786099deSNavdeep Parhar 		if (cst->tx_credits == cst->tx_total) {
929e38a50e8SJohn Baldwin 			cxgbe_rate_tag_free_locked(cst);
930786099deSNavdeep Parhar 			return;	/* cst is gone. */
931786099deSNavdeep Parhar 		}
932786099deSNavdeep Parhar 		send_etid_flush_wr(cst);
933786099deSNavdeep Parhar 	}
934786099deSNavdeep Parhar 	mtx_unlock(&cst->lock);
935786099deSNavdeep Parhar }
93620abea66SRandall Stewart 
93720abea66SRandall Stewart void
938b8b01d9bSNavdeep Parhar cxgbe_ratelimit_query(struct ifnet *ifp, struct if_ratelimit_query_results *q)
93920abea66SRandall Stewart {
940b8b01d9bSNavdeep Parhar 	struct vi_info *vi = ifp->if_softc;
941b8b01d9bSNavdeep Parhar 	struct adapter *sc = vi->adapter;
942b8b01d9bSNavdeep Parhar 
94320abea66SRandall Stewart 	q->rate_table = NULL;
94420abea66SRandall Stewart 	q->flags = RT_IS_SELECTABLE;
945b8b01d9bSNavdeep Parhar 	/*
946b8b01d9bSNavdeep Parhar 	 * Absolute max limits from the firmware configuration.  Practical
947b8b01d9bSNavdeep Parhar 	 * limits depend on the burstsize, pktsize (ifp->if_mtu ultimately) and
948b8b01d9bSNavdeep Parhar 	 * the card's cclk.
949b8b01d9bSNavdeep Parhar 	 */
950b8b01d9bSNavdeep Parhar 	q->max_flows = sc->tids.netids;
9516beb67c7SNavdeep Parhar 	q->number_of_rates = sc->params.nsched_cls;
952b8b01d9bSNavdeep Parhar 	q->min_segment_burst = 4; /* matches PKTSCHED_BURST in the firmware. */
953b8b01d9bSNavdeep Parhar 
954b8b01d9bSNavdeep Parhar #if 1
955b8b01d9bSNavdeep Parhar 	if (chip_id(sc) < CHELSIO_T6) {
956b8b01d9bSNavdeep Parhar 		/* Based on testing by rrs@ with a T580 at burstsize = 4. */
957b8b01d9bSNavdeep Parhar 		MPASS(q->min_segment_burst == 4);
958e2e43aafSNavdeep Parhar 		q->max_flows = min(4000, q->max_flows);
959b8b01d9bSNavdeep Parhar 	} else {
960b8b01d9bSNavdeep Parhar 		/* XXX: TBD, carried forward from T5 for now. */
961e2e43aafSNavdeep Parhar 		q->max_flows = min(4000, q->max_flows);
962b8b01d9bSNavdeep Parhar 	}
963b8b01d9bSNavdeep Parhar 
964b8b01d9bSNavdeep Parhar 	/*
965b8b01d9bSNavdeep Parhar 	 * XXX: tcp_ratelimit.c grabs all available rates on link-up before it
966b8b01d9bSNavdeep Parhar 	 * even knows whether hw pacing will be used or not.  This prevents
967b8b01d9bSNavdeep Parhar 	 * other consumers like SO_MAX_PACING_RATE or those using cxgbetool or
968b8b01d9bSNavdeep Parhar 	 * the private ioctls from using any of traffic classes.
969b8b01d9bSNavdeep Parhar 	 *
970b8b01d9bSNavdeep Parhar 	 * Underreport the number of rates to tcp_ratelimit so that it doesn't
971b8b01d9bSNavdeep Parhar 	 * hog all of them.  This can be removed if/when tcp_ratelimit switches
972b8b01d9bSNavdeep Parhar 	 * to making its allocations on first-use rather than link-up.  There is
973b8b01d9bSNavdeep Parhar 	 * nothing wrong with one particular consumer reserving all the classes
974b8b01d9bSNavdeep Parhar 	 * but it should do so only if it'll actually use hw rate limiting.
975b8b01d9bSNavdeep Parhar 	 */
976b8b01d9bSNavdeep Parhar 	q->number_of_rates /= 4;
977b8b01d9bSNavdeep Parhar #endif
97820abea66SRandall Stewart }
97967e07112SNavdeep Parhar #endif
980