12204b427SNavdeep Parhar /*- 22204b427SNavdeep Parhar * Copyright (c) 2017 Chelsio Communications, Inc. 32204b427SNavdeep Parhar * All rights reserved. 42204b427SNavdeep Parhar * Written by: Navdeep Parhar <np@FreeBSD.org> 52204b427SNavdeep Parhar * 62204b427SNavdeep Parhar * Redistribution and use in source and binary forms, with or without 72204b427SNavdeep Parhar * modification, are permitted provided that the following conditions 82204b427SNavdeep Parhar * are met: 92204b427SNavdeep Parhar * 1. Redistributions of source code must retain the above copyright 102204b427SNavdeep Parhar * notice, this list of conditions and the following disclaimer. 112204b427SNavdeep Parhar * 2. Redistributions in binary form must reproduce the above copyright 122204b427SNavdeep Parhar * notice, this list of conditions and the following disclaimer in the 132204b427SNavdeep Parhar * documentation and/or other materials provided with the distribution. 142204b427SNavdeep Parhar * 152204b427SNavdeep Parhar * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 162204b427SNavdeep Parhar * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 172204b427SNavdeep Parhar * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 182204b427SNavdeep Parhar * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 192204b427SNavdeep Parhar * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 202204b427SNavdeep Parhar * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 212204b427SNavdeep Parhar * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 222204b427SNavdeep Parhar * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 232204b427SNavdeep Parhar * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 242204b427SNavdeep Parhar * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 252204b427SNavdeep Parhar * SUCH DAMAGE. 262204b427SNavdeep Parhar */ 272204b427SNavdeep Parhar 282204b427SNavdeep Parhar #include <sys/cdefs.h> 292204b427SNavdeep Parhar __FBSDID("$FreeBSD$"); 302204b427SNavdeep Parhar 312204b427SNavdeep Parhar #include "opt_inet.h" 322204b427SNavdeep Parhar #include "opt_inet6.h" 3367e07112SNavdeep Parhar #include "opt_ratelimit.h" 342204b427SNavdeep Parhar 352204b427SNavdeep Parhar #include <sys/types.h> 362204b427SNavdeep Parhar #include <sys/malloc.h> 372204b427SNavdeep Parhar #include <sys/queue.h> 382204b427SNavdeep Parhar #include <sys/sbuf.h> 392204b427SNavdeep Parhar #include <sys/taskqueue.h> 402204b427SNavdeep Parhar #include <sys/sysctl.h> 412204b427SNavdeep Parhar 422204b427SNavdeep Parhar #include "common/common.h" 432204b427SNavdeep Parhar #include "common/t4_regs.h" 442204b427SNavdeep Parhar #include "common/t4_regs_values.h" 452204b427SNavdeep Parhar #include "common/t4_msg.h" 462204b427SNavdeep Parhar 472204b427SNavdeep Parhar 482204b427SNavdeep Parhar static int 492204b427SNavdeep Parhar in_range(int val, int lo, int hi) 502204b427SNavdeep Parhar { 512204b427SNavdeep Parhar 522204b427SNavdeep Parhar return (val < 0 || (val <= hi && val >= lo)); 532204b427SNavdeep Parhar } 542204b427SNavdeep Parhar 552204b427SNavdeep Parhar static int 562204b427SNavdeep Parhar set_sched_class_config(struct adapter *sc, int minmax) 572204b427SNavdeep Parhar { 582204b427SNavdeep Parhar int rc; 592204b427SNavdeep Parhar 602204b427SNavdeep Parhar if (minmax < 0) 612204b427SNavdeep Parhar return (EINVAL); 622204b427SNavdeep Parhar 632204b427SNavdeep Parhar rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc"); 642204b427SNavdeep Parhar if (rc) 652204b427SNavdeep Parhar return (rc); 6683b5cda1SNavdeep Parhar if (hw_off_limits(sc)) 6783b5cda1SNavdeep Parhar rc = ENXIO; 6883b5cda1SNavdeep Parhar else 692204b427SNavdeep Parhar rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1); 702204b427SNavdeep Parhar end_synchronized_op(sc, 0); 712204b427SNavdeep Parhar 722204b427SNavdeep Parhar return (rc); 732204b427SNavdeep Parhar } 742204b427SNavdeep Parhar 752204b427SNavdeep Parhar static int 762204b427SNavdeep Parhar set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p, 772204b427SNavdeep Parhar int sleep_ok) 782204b427SNavdeep Parhar { 792204b427SNavdeep Parhar int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode; 802204b427SNavdeep Parhar struct port_info *pi; 811979b511SNavdeep Parhar struct tx_cl_rl_params *tc, old; 829c3b8b3cSNavdeep Parhar bool check_pktsize = false; 832204b427SNavdeep Parhar 842204b427SNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL) 852204b427SNavdeep Parhar fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 862204b427SNavdeep Parhar else if (p->level == SCHED_CLASS_LEVEL_CL_WRR) 872204b427SNavdeep Parhar fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 882204b427SNavdeep Parhar else if (p->level == SCHED_CLASS_LEVEL_CH_RL) 892204b427SNavdeep Parhar fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 902204b427SNavdeep Parhar else 912204b427SNavdeep Parhar return (EINVAL); 922204b427SNavdeep Parhar 939c3b8b3cSNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL) { 942204b427SNavdeep Parhar if (p->mode == SCHED_CLASS_MODE_CLASS) 952204b427SNavdeep Parhar fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 969c3b8b3cSNavdeep Parhar else if (p->mode == SCHED_CLASS_MODE_FLOW) { 979c3b8b3cSNavdeep Parhar check_pktsize = true; 982204b427SNavdeep Parhar fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 999c3b8b3cSNavdeep Parhar } else 1002204b427SNavdeep Parhar return (EINVAL); 1019c3b8b3cSNavdeep Parhar } else 1029c3b8b3cSNavdeep Parhar fw_mode = 0; 1032204b427SNavdeep Parhar 1049c3b8b3cSNavdeep Parhar /* Valid channel must always be provided. */ 1059c3b8b3cSNavdeep Parhar if (p->channel < 0) 1062204b427SNavdeep Parhar return (EINVAL); 1072204b427SNavdeep Parhar if (!in_range(p->channel, 0, sc->chip_params->nchan - 1)) 1082204b427SNavdeep Parhar return (ERANGE); 1092204b427SNavdeep Parhar 1102204b427SNavdeep Parhar pi = sc->port[sc->chan_map[p->channel]]; 1112204b427SNavdeep Parhar if (pi == NULL) 1122204b427SNavdeep Parhar return (ENXIO); 1132204b427SNavdeep Parhar MPASS(pi->tx_chan == p->channel); 1142204b427SNavdeep Parhar top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */ 1152204b427SNavdeep Parhar 1169c3b8b3cSNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL || 1179c3b8b3cSNavdeep Parhar p->level == SCHED_CLASS_LEVEL_CH_RL) { 1182204b427SNavdeep Parhar /* 1199c3b8b3cSNavdeep Parhar * Valid rate (mode, unit and values) must be provided. 1202204b427SNavdeep Parhar */ 1212204b427SNavdeep Parhar 1222204b427SNavdeep Parhar if (p->minrate < 0) 1232204b427SNavdeep Parhar p->minrate = 0; 1249c3b8b3cSNavdeep Parhar if (p->maxrate < 0) 1252204b427SNavdeep Parhar return (EINVAL); 1269c3b8b3cSNavdeep Parhar 1279c3b8b3cSNavdeep Parhar if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS) { 1289c3b8b3cSNavdeep Parhar fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 1299c3b8b3cSNavdeep Parhar /* ratemode could be relative (%) or absolute. */ 1309c3b8b3cSNavdeep Parhar if (p->ratemode == SCHED_CLASS_RATEMODE_REL) { 1319c3b8b3cSNavdeep Parhar fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 1329c3b8b3cSNavdeep Parhar /* maxrate is % of port bandwidth. */ 1339c3b8b3cSNavdeep Parhar if (!in_range(p->minrate, 0, 100) || 1349c3b8b3cSNavdeep Parhar !in_range(p->maxrate, 0, 100)) { 1359c3b8b3cSNavdeep Parhar return (ERANGE); 1362204b427SNavdeep Parhar } 1379c3b8b3cSNavdeep Parhar } else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS) { 1389c3b8b3cSNavdeep Parhar fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 1399c3b8b3cSNavdeep Parhar /* maxrate is absolute value in kbps. */ 1409c3b8b3cSNavdeep Parhar if (!in_range(p->minrate, 0, top_speed) || 1419c3b8b3cSNavdeep Parhar !in_range(p->maxrate, 0, top_speed)) { 1429c3b8b3cSNavdeep Parhar return (ERANGE); 1432204b427SNavdeep Parhar } 1449c3b8b3cSNavdeep Parhar } else 1452204b427SNavdeep Parhar return (EINVAL); 1469c3b8b3cSNavdeep Parhar } else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS) { 1479c3b8b3cSNavdeep Parhar /* maxrate is the absolute value in pps. */ 1489c3b8b3cSNavdeep Parhar check_pktsize = true; 1499c3b8b3cSNavdeep Parhar fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 1509c3b8b3cSNavdeep Parhar } else 1519c3b8b3cSNavdeep Parhar return (EINVAL); 1529c3b8b3cSNavdeep Parhar } else { 1539c3b8b3cSNavdeep Parhar MPASS(p->level == SCHED_CLASS_LEVEL_CL_WRR); 1549c3b8b3cSNavdeep Parhar 1559c3b8b3cSNavdeep Parhar /* 1569c3b8b3cSNavdeep Parhar * Valid weight must be provided. 1579c3b8b3cSNavdeep Parhar */ 1589c3b8b3cSNavdeep Parhar if (p->weight < 0) 1599c3b8b3cSNavdeep Parhar return (EINVAL); 1609c3b8b3cSNavdeep Parhar if (!in_range(p->weight, 1, 99)) 1619c3b8b3cSNavdeep Parhar return (ERANGE); 1629c3b8b3cSNavdeep Parhar 1639c3b8b3cSNavdeep Parhar fw_rateunit = 0; 1649c3b8b3cSNavdeep Parhar fw_ratemode = 0; 1659c3b8b3cSNavdeep Parhar } 1669c3b8b3cSNavdeep Parhar 1679c3b8b3cSNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL || 1689c3b8b3cSNavdeep Parhar p->level == SCHED_CLASS_LEVEL_CL_WRR) { 1699c3b8b3cSNavdeep Parhar /* 1709c3b8b3cSNavdeep Parhar * Valid scheduling class must be provided. 1719c3b8b3cSNavdeep Parhar */ 1729c3b8b3cSNavdeep Parhar if (p->cl < 0) 1739c3b8b3cSNavdeep Parhar return (EINVAL); 1746beb67c7SNavdeep Parhar if (!in_range(p->cl, 0, sc->params.nsched_cls - 1)) 1759c3b8b3cSNavdeep Parhar return (ERANGE); 1769c3b8b3cSNavdeep Parhar } 1779c3b8b3cSNavdeep Parhar 1789c3b8b3cSNavdeep Parhar if (check_pktsize) { 1799c3b8b3cSNavdeep Parhar if (p->pktsize < 0) 1809c3b8b3cSNavdeep Parhar return (EINVAL); 1819c3b8b3cSNavdeep Parhar if (!in_range(p->pktsize, 64, pi->vi[0].ifp->if_mtu)) 1829c3b8b3cSNavdeep Parhar return (ERANGE); 1832204b427SNavdeep Parhar } 1842204b427SNavdeep Parhar 1852204b427SNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL) { 1862204b427SNavdeep Parhar tc = &pi->sched_params->cl_rl[p->cl]; 1871979b511SNavdeep Parhar mtx_lock(&sc->tc_lock); 188*ec8004ddSNavdeep Parhar if (tc->refcount > 0 || tc->state == CS_HW_UPDATE_IN_PROGRESS) 1892204b427SNavdeep Parhar rc = EBUSY; 1901979b511SNavdeep Parhar else { 191*ec8004ddSNavdeep Parhar old = *tc; 192*ec8004ddSNavdeep Parhar 193*ec8004ddSNavdeep Parhar tc->flags |= CF_USER; 194*ec8004ddSNavdeep Parhar tc->state = CS_HW_UPDATE_IN_PROGRESS; 1952204b427SNavdeep Parhar tc->ratemode = fw_ratemode; 1962204b427SNavdeep Parhar tc->rateunit = fw_rateunit; 1972204b427SNavdeep Parhar tc->mode = fw_mode; 1982204b427SNavdeep Parhar tc->maxrate = p->maxrate; 1992204b427SNavdeep Parhar tc->pktsize = p->pktsize; 2001979b511SNavdeep Parhar rc = 0; 2012204b427SNavdeep Parhar } 2021979b511SNavdeep Parhar mtx_unlock(&sc->tc_lock); 2031979b511SNavdeep Parhar if (rc != 0) 2041979b511SNavdeep Parhar return (rc); 2051979b511SNavdeep Parhar } 2061979b511SNavdeep Parhar 2071979b511SNavdeep Parhar rc = begin_synchronized_op(sc, NULL, 2081979b511SNavdeep Parhar sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp"); 2091979b511SNavdeep Parhar if (rc != 0) { 2101979b511SNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL) { 2111979b511SNavdeep Parhar mtx_lock(&sc->tc_lock); 212*ec8004ddSNavdeep Parhar MPASS(tc->refcount == 0); 213*ec8004ddSNavdeep Parhar MPASS(tc->flags & CF_USER); 214*ec8004ddSNavdeep Parhar MPASS(tc->state == CS_HW_UPDATE_IN_PROGRESS); 2151979b511SNavdeep Parhar *tc = old; 2161979b511SNavdeep Parhar mtx_unlock(&sc->tc_lock); 2171979b511SNavdeep Parhar } 2181979b511SNavdeep Parhar return (rc); 2192204b427SNavdeep Parhar } 22083b5cda1SNavdeep Parhar if (!hw_off_limits(sc)) { 22183b5cda1SNavdeep Parhar rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, 22283b5cda1SNavdeep Parhar fw_mode, fw_rateunit, fw_ratemode, p->channel, p->cl, 22383b5cda1SNavdeep Parhar p->minrate, p->maxrate, p->weight, p->pktsize, 0, sleep_ok); 22483b5cda1SNavdeep Parhar } 2252204b427SNavdeep Parhar end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD); 2262204b427SNavdeep Parhar 2271979b511SNavdeep Parhar if (p->level == SCHED_CLASS_LEVEL_CL_RL) { 2281979b511SNavdeep Parhar mtx_lock(&sc->tc_lock); 2291979b511SNavdeep Parhar MPASS(tc->refcount == 0); 230*ec8004ddSNavdeep Parhar MPASS(tc->flags & CF_USER); 231*ec8004ddSNavdeep Parhar MPASS(tc->state == CS_HW_UPDATE_IN_PROGRESS); 2321979b511SNavdeep Parhar 2331979b511SNavdeep Parhar if (rc == 0) 234*ec8004ddSNavdeep Parhar tc->state = CS_HW_CONFIGURED; 235*ec8004ddSNavdeep Parhar else { 236*ec8004ddSNavdeep Parhar /* parameters failed so we don't park at params_set */ 237*ec8004ddSNavdeep Parhar tc->state = CS_UNINITIALIZED; 238*ec8004ddSNavdeep Parhar tc->flags &= ~CF_USER; 239*ec8004ddSNavdeep Parhar CH_ERR(pi, "failed to configure traffic class %d: %d. " 240*ec8004ddSNavdeep Parhar "params: mode %d, rateunit %d, ratemode %d, " 241*ec8004ddSNavdeep Parhar "channel %d, minrate %d, maxrate %d, pktsize %d, " 242*ec8004ddSNavdeep Parhar "burstsize %d\n", p->cl, rc, fw_mode, fw_rateunit, 243*ec8004ddSNavdeep Parhar fw_ratemode, p->channel, p->minrate, p->maxrate, 244*ec8004ddSNavdeep Parhar p->pktsize, 0); 245*ec8004ddSNavdeep Parhar } 2461979b511SNavdeep Parhar mtx_unlock(&sc->tc_lock); 2471979b511SNavdeep Parhar } 2481979b511SNavdeep Parhar 2492204b427SNavdeep Parhar return (rc); 2502204b427SNavdeep Parhar } 2512204b427SNavdeep Parhar 2522204b427SNavdeep Parhar static void 2532204b427SNavdeep Parhar update_tx_sched(void *context, int pending) 2542204b427SNavdeep Parhar { 2551979b511SNavdeep Parhar int i, j, rc; 2562204b427SNavdeep Parhar struct port_info *pi; 2572204b427SNavdeep Parhar struct tx_cl_rl_params *tc; 2582204b427SNavdeep Parhar struct adapter *sc = context; 2596beb67c7SNavdeep Parhar const int n = sc->params.nsched_cls; 2602204b427SNavdeep Parhar 2612204b427SNavdeep Parhar mtx_lock(&sc->tc_lock); 2622204b427SNavdeep Parhar for_each_port(sc, i) { 2632204b427SNavdeep Parhar pi = sc->port[i]; 2642204b427SNavdeep Parhar tc = &pi->sched_params->cl_rl[0]; 2652204b427SNavdeep Parhar for (j = 0; j < n; j++, tc++) { 2662204b427SNavdeep Parhar MPASS(mtx_owned(&sc->tc_lock)); 267*ec8004ddSNavdeep Parhar if (tc->state != CS_HW_UPDATE_REQUESTED) 2682204b427SNavdeep Parhar continue; 2692204b427SNavdeep Parhar mtx_unlock(&sc->tc_lock); 2702204b427SNavdeep Parhar 2712204b427SNavdeep Parhar if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 2722204b427SNavdeep Parhar "t4utxs") != 0) { 2732204b427SNavdeep Parhar mtx_lock(&sc->tc_lock); 2742204b427SNavdeep Parhar continue; 2752204b427SNavdeep Parhar } 2761979b511SNavdeep Parhar rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, 2771979b511SNavdeep Parhar FW_SCHED_PARAMS_LEVEL_CL_RL, tc->mode, tc->rateunit, 2781979b511SNavdeep Parhar tc->ratemode, pi->tx_chan, j, 0, tc->maxrate, 0, 27909a7189fSNavdeep Parhar tc->pktsize, tc->burstsize, 1); 2802204b427SNavdeep Parhar end_synchronized_op(sc, 0); 2812204b427SNavdeep Parhar 2822204b427SNavdeep Parhar mtx_lock(&sc->tc_lock); 283*ec8004ddSNavdeep Parhar MPASS(tc->state == CS_HW_UPDATE_REQUESTED); 284*ec8004ddSNavdeep Parhar if (rc == 0) { 285*ec8004ddSNavdeep Parhar tc->state = CS_HW_CONFIGURED; 286*ec8004ddSNavdeep Parhar continue; 287*ec8004ddSNavdeep Parhar } 288*ec8004ddSNavdeep Parhar /* parameters failed so we try to avoid params_set */ 289*ec8004ddSNavdeep Parhar if (tc->refcount > 0) 290*ec8004ddSNavdeep Parhar tc->state = CS_PARAMS_SET; 2911979b511SNavdeep Parhar else 292*ec8004ddSNavdeep Parhar tc->state = CS_UNINITIALIZED; 293*ec8004ddSNavdeep Parhar CH_ERR(pi, "failed to configure traffic class %d: %d. " 294*ec8004ddSNavdeep Parhar "params: mode %d, rateunit %d, ratemode %d, " 295*ec8004ddSNavdeep Parhar "channel %d, minrate %d, maxrate %d, pktsize %d, " 296*ec8004ddSNavdeep Parhar "burstsize %d\n", j, rc, tc->mode, tc->rateunit, 297*ec8004ddSNavdeep Parhar tc->ratemode, pi->tx_chan, 0, tc->maxrate, 298*ec8004ddSNavdeep Parhar tc->pktsize, tc->burstsize); 2992204b427SNavdeep Parhar } 3002204b427SNavdeep Parhar } 3012204b427SNavdeep Parhar mtx_unlock(&sc->tc_lock); 3022204b427SNavdeep Parhar } 3032204b427SNavdeep Parhar 3042204b427SNavdeep Parhar int 3052204b427SNavdeep Parhar t4_set_sched_class(struct adapter *sc, struct t4_sched_params *p) 3062204b427SNavdeep Parhar { 3072204b427SNavdeep Parhar 3082204b427SNavdeep Parhar if (p->type != SCHED_CLASS_TYPE_PACKET) 3092204b427SNavdeep Parhar return (EINVAL); 3102204b427SNavdeep Parhar 3112204b427SNavdeep Parhar if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 3122204b427SNavdeep Parhar return (set_sched_class_config(sc, p->u.config.minmax)); 3132204b427SNavdeep Parhar 3142204b427SNavdeep Parhar if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 3152204b427SNavdeep Parhar return (set_sched_class_params(sc, &p->u.params, 1)); 3162204b427SNavdeep Parhar 3172204b427SNavdeep Parhar return (EINVAL); 3182204b427SNavdeep Parhar } 3192204b427SNavdeep Parhar 3201979b511SNavdeep Parhar static int 3211979b511SNavdeep Parhar bind_txq_to_traffic_class(struct adapter *sc, struct sge_txq *txq, int idx) 3221979b511SNavdeep Parhar { 3231979b511SNavdeep Parhar struct tx_cl_rl_params *tc0, *tc; 3241979b511SNavdeep Parhar int rc, old_idx; 3251979b511SNavdeep Parhar uint32_t fw_mnem, fw_class; 3261979b511SNavdeep Parhar 32743bbae19SNavdeep Parhar if (!(txq->eq.flags & EQ_HW_ALLOCATED)) 32843bbae19SNavdeep Parhar return (ENXIO); 3291979b511SNavdeep Parhar 3301979b511SNavdeep Parhar mtx_lock(&sc->tc_lock); 3311979b511SNavdeep Parhar if (txq->tc_idx == -2) { 3321979b511SNavdeep Parhar rc = EBUSY; /* Another bind/unbind in progress already. */ 3331979b511SNavdeep Parhar goto done; 3341979b511SNavdeep Parhar } 3351979b511SNavdeep Parhar if (idx == txq->tc_idx) { 3361979b511SNavdeep Parhar rc = 0; /* No change, nothing to do. */ 3371979b511SNavdeep Parhar goto done; 3381979b511SNavdeep Parhar } 3391979b511SNavdeep Parhar 3401979b511SNavdeep Parhar tc0 = &sc->port[txq->eq.tx_chan]->sched_params->cl_rl[0]; 3411979b511SNavdeep Parhar if (idx != -1) { 3421979b511SNavdeep Parhar /* 3431979b511SNavdeep Parhar * Bind to a different class at index idx. 3441979b511SNavdeep Parhar */ 3451979b511SNavdeep Parhar tc = &tc0[idx]; 346*ec8004ddSNavdeep Parhar if (tc->state != CS_HW_CONFIGURED) { 3471979b511SNavdeep Parhar rc = ENXIO; 3481979b511SNavdeep Parhar goto done; 3491979b511SNavdeep Parhar } else { 3501979b511SNavdeep Parhar /* 3511979b511SNavdeep Parhar * Ok to proceed. Place a reference on the new class 3521979b511SNavdeep Parhar * while still holding on to the reference on the 3531979b511SNavdeep Parhar * previous class, if any. 3541979b511SNavdeep Parhar */ 3551979b511SNavdeep Parhar tc->refcount++; 3561979b511SNavdeep Parhar } 3571979b511SNavdeep Parhar } 3581979b511SNavdeep Parhar /* Mark as busy before letting go of the lock. */ 3591979b511SNavdeep Parhar old_idx = txq->tc_idx; 3601979b511SNavdeep Parhar txq->tc_idx = -2; 3611979b511SNavdeep Parhar mtx_unlock(&sc->tc_lock); 3621979b511SNavdeep Parhar 3631979b511SNavdeep Parhar rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4btxq"); 364*ec8004ddSNavdeep Parhar if (rc == 0) { 3651979b511SNavdeep Parhar fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 3661979b511SNavdeep Parhar V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH) | 3671979b511SNavdeep Parhar V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 3681979b511SNavdeep Parhar fw_class = idx < 0 ? 0xffffffff : idx; 369*ec8004ddSNavdeep Parhar rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_mnem, 370*ec8004ddSNavdeep Parhar &fw_class); 3711979b511SNavdeep Parhar end_synchronized_op(sc, 0); 372*ec8004ddSNavdeep Parhar } 3731979b511SNavdeep Parhar 3741979b511SNavdeep Parhar mtx_lock(&sc->tc_lock); 3751979b511SNavdeep Parhar MPASS(txq->tc_idx == -2); 3761979b511SNavdeep Parhar if (rc == 0) { 3771979b511SNavdeep Parhar /* 3781979b511SNavdeep Parhar * Unbind, bind, or bind to a different class succeeded. Remove 3791979b511SNavdeep Parhar * the reference on the old traffic class, if any. 3801979b511SNavdeep Parhar */ 3811979b511SNavdeep Parhar if (old_idx != -1) { 3821979b511SNavdeep Parhar tc = &tc0[old_idx]; 3831979b511SNavdeep Parhar MPASS(tc->refcount > 0); 3841979b511SNavdeep Parhar tc->refcount--; 3851979b511SNavdeep Parhar } 3861979b511SNavdeep Parhar txq->tc_idx = idx; 3871979b511SNavdeep Parhar } else { 3881979b511SNavdeep Parhar /* 3891979b511SNavdeep Parhar * Unbind, bind, or bind to a different class failed. Remove 3901979b511SNavdeep Parhar * the anticipatory reference on the new traffic class, if any. 3911979b511SNavdeep Parhar */ 3921979b511SNavdeep Parhar if (idx != -1) { 3931979b511SNavdeep Parhar tc = &tc0[idx]; 3941979b511SNavdeep Parhar MPASS(tc->refcount > 0); 3951979b511SNavdeep Parhar tc->refcount--; 3961979b511SNavdeep Parhar } 3971979b511SNavdeep Parhar txq->tc_idx = old_idx; 3981979b511SNavdeep Parhar } 3991979b511SNavdeep Parhar done: 4006beb67c7SNavdeep Parhar MPASS(txq->tc_idx >= -1 && txq->tc_idx < sc->params.nsched_cls); 4011979b511SNavdeep Parhar mtx_unlock(&sc->tc_lock); 4021979b511SNavdeep Parhar return (rc); 4031979b511SNavdeep Parhar } 4041979b511SNavdeep Parhar 4052204b427SNavdeep Parhar int 4062204b427SNavdeep Parhar t4_set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 4072204b427SNavdeep Parhar { 4082204b427SNavdeep Parhar struct port_info *pi = NULL; 4092204b427SNavdeep Parhar struct vi_info *vi; 4102204b427SNavdeep Parhar struct sge_txq *txq; 4112204b427SNavdeep Parhar int i, rc; 4122204b427SNavdeep Parhar 4131979b511SNavdeep Parhar if (p->port >= sc->params.nports) 4141979b511SNavdeep Parhar return (EINVAL); 4152204b427SNavdeep Parhar 4161979b511SNavdeep Parhar /* 4171979b511SNavdeep Parhar * XXX: cxgbetool allows the user to specify the physical port only. So 4181979b511SNavdeep Parhar * we always operate on the main VI. 4191979b511SNavdeep Parhar */ 4202204b427SNavdeep Parhar pi = sc->port[p->port]; 4212204b427SNavdeep Parhar vi = &pi->vi[0]; 4221979b511SNavdeep Parhar 4231979b511SNavdeep Parhar /* Checking VI_INIT_DONE outside a synch-op is a harmless race here. */ 4241979b511SNavdeep Parhar if (!(vi->flags & VI_INIT_DONE)) 4251979b511SNavdeep Parhar return (EAGAIN); 426eb6f5d6eSNavdeep Parhar MPASS(vi->ntxq > 0); 4272204b427SNavdeep Parhar 4282204b427SNavdeep Parhar if (!in_range(p->queue, 0, vi->ntxq - 1) || 4296beb67c7SNavdeep Parhar !in_range(p->cl, 0, sc->params.nsched_cls - 1)) 4301979b511SNavdeep Parhar return (EINVAL); 4312204b427SNavdeep Parhar 4321979b511SNavdeep Parhar if (p->queue < 0) { 4332204b427SNavdeep Parhar /* 4342204b427SNavdeep Parhar * Change the scheduling on all the TX queues for the 4352204b427SNavdeep Parhar * interface. 4362204b427SNavdeep Parhar */ 4372204b427SNavdeep Parhar for_each_txq(vi, i, txq) { 4381979b511SNavdeep Parhar rc = bind_txq_to_traffic_class(sc, txq, p->cl); 4391979b511SNavdeep Parhar if (rc != 0) 4401979b511SNavdeep Parhar break; 4411979b511SNavdeep Parhar } 4421979b511SNavdeep Parhar } else { 4431979b511SNavdeep Parhar /* 4441979b511SNavdeep Parhar * If op.queue is non-negative, then we're only changing the 4451979b511SNavdeep Parhar * scheduling on a single specified TX queue. 4461979b511SNavdeep Parhar */ 4471979b511SNavdeep Parhar txq = &sc->sge.txq[vi->first_txq + p->queue]; 4481979b511SNavdeep Parhar rc = bind_txq_to_traffic_class(sc, txq, p->cl); 4492204b427SNavdeep Parhar } 4502204b427SNavdeep Parhar 4512204b427SNavdeep Parhar return (rc); 4522204b427SNavdeep Parhar } 4532204b427SNavdeep Parhar 4542204b427SNavdeep Parhar int 4552204b427SNavdeep Parhar t4_init_tx_sched(struct adapter *sc) 4562204b427SNavdeep Parhar { 457*ec8004ddSNavdeep Parhar int i; 4586beb67c7SNavdeep Parhar const int n = sc->params.nsched_cls; 4592204b427SNavdeep Parhar struct port_info *pi; 4602204b427SNavdeep Parhar 4612204b427SNavdeep Parhar mtx_init(&sc->tc_lock, "tx_sched lock", NULL, MTX_DEF); 4622204b427SNavdeep Parhar TASK_INIT(&sc->tc_task, 0, update_tx_sched, sc); 4632204b427SNavdeep Parhar for_each_port(sc, i) { 4642204b427SNavdeep Parhar pi = sc->port[i]; 4652204b427SNavdeep Parhar pi->sched_params = malloc(sizeof(*pi->sched_params) + 466*ec8004ddSNavdeep Parhar n * sizeof(struct tx_cl_rl_params), M_CXGBE, M_ZERO | M_WAITOK); 4672204b427SNavdeep Parhar } 4682204b427SNavdeep Parhar 4692204b427SNavdeep Parhar return (0); 4702204b427SNavdeep Parhar } 4712204b427SNavdeep Parhar 4722204b427SNavdeep Parhar int 4732204b427SNavdeep Parhar t4_free_tx_sched(struct adapter *sc) 4742204b427SNavdeep Parhar { 4752204b427SNavdeep Parhar int i; 4762204b427SNavdeep Parhar 4772204b427SNavdeep Parhar taskqueue_drain(taskqueue_thread, &sc->tc_task); 4782204b427SNavdeep Parhar 479cc2050c5SNavdeep Parhar for_each_port(sc, i) { 480cc2050c5SNavdeep Parhar if (sc->port[i] != NULL) 4812204b427SNavdeep Parhar free(sc->port[i]->sched_params, M_CXGBE); 482cc2050c5SNavdeep Parhar } 4832204b427SNavdeep Parhar 4842204b427SNavdeep Parhar if (mtx_initialized(&sc->tc_lock)) 4852204b427SNavdeep Parhar mtx_destroy(&sc->tc_lock); 4862204b427SNavdeep Parhar 4872204b427SNavdeep Parhar return (0); 4882204b427SNavdeep Parhar } 4892204b427SNavdeep Parhar 4902204b427SNavdeep Parhar void 4912204b427SNavdeep Parhar t4_update_tx_sched(struct adapter *sc) 4922204b427SNavdeep Parhar { 4932204b427SNavdeep Parhar 4942204b427SNavdeep Parhar taskqueue_enqueue(taskqueue_thread, &sc->tc_task); 4952204b427SNavdeep Parhar } 4962204b427SNavdeep Parhar 4972204b427SNavdeep Parhar int 4982204b427SNavdeep Parhar t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate, 4992204b427SNavdeep Parhar int *tc_idx) 5002204b427SNavdeep Parhar { 501*ec8004ddSNavdeep Parhar int rc = 0, fa, fa2, i, pktsize, burstsize; 5021979b511SNavdeep Parhar bool update; 5032204b427SNavdeep Parhar struct tx_cl_rl_params *tc; 5041979b511SNavdeep Parhar struct port_info *pi; 5052204b427SNavdeep Parhar 5062204b427SNavdeep Parhar MPASS(port_id >= 0 && port_id < sc->params.nports); 5072204b427SNavdeep Parhar 5081979b511SNavdeep Parhar pi = sc->port[port_id]; 50909a7189fSNavdeep Parhar if (pi->sched_params->pktsize > 0) 51009a7189fSNavdeep Parhar pktsize = pi->sched_params->pktsize; 51109a7189fSNavdeep Parhar else 51209a7189fSNavdeep Parhar pktsize = pi->vi[0].ifp->if_mtu; 51309a7189fSNavdeep Parhar if (pi->sched_params->burstsize > 0) 51409a7189fSNavdeep Parhar burstsize = pi->sched_params->burstsize; 51509a7189fSNavdeep Parhar else 51609a7189fSNavdeep Parhar burstsize = pktsize * 4; 5171979b511SNavdeep Parhar tc = &pi->sched_params->cl_rl[0]; 51809a7189fSNavdeep Parhar 5191979b511SNavdeep Parhar update = false; 520*ec8004ddSNavdeep Parhar fa = fa2 = -1; 5212204b427SNavdeep Parhar mtx_lock(&sc->tc_lock); 5226beb67c7SNavdeep Parhar for (i = 0; i < sc->params.nsched_cls; i++, tc++) { 523*ec8004ddSNavdeep Parhar if (tc->state >= CS_PARAMS_SET && 524*ec8004ddSNavdeep Parhar tc->ratemode == FW_SCHED_PARAMS_RATE_ABS && 5252204b427SNavdeep Parhar tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE && 5262204b427SNavdeep Parhar tc->mode == FW_SCHED_PARAMS_MODE_FLOW && 52709a7189fSNavdeep Parhar tc->maxrate == maxrate && tc->pktsize == pktsize && 52809a7189fSNavdeep Parhar tc->burstsize == burstsize) { 5292204b427SNavdeep Parhar tc->refcount++; 5302204b427SNavdeep Parhar *tc_idx = i; 531*ec8004ddSNavdeep Parhar if (tc->state == CS_PARAMS_SET) { 532*ec8004ddSNavdeep Parhar tc->state = CS_HW_UPDATE_REQUESTED; 5331979b511SNavdeep Parhar update = true; 5341979b511SNavdeep Parhar } 5352204b427SNavdeep Parhar goto done; 5362204b427SNavdeep Parhar } 537*ec8004ddSNavdeep Parhar 538*ec8004ddSNavdeep Parhar if (fa < 0 && tc->state == CS_UNINITIALIZED) { 539*ec8004ddSNavdeep Parhar MPASS(tc->refcount == 0); 540*ec8004ddSNavdeep Parhar fa = i; /* first available, never used. */ 541*ec8004ddSNavdeep Parhar } 542*ec8004ddSNavdeep Parhar if (fa2 < 0 && tc->refcount == 0 && !(tc->flags & CF_USER)) { 543*ec8004ddSNavdeep Parhar fa2 = i; /* first available, used previously. */ 544*ec8004ddSNavdeep Parhar } 5452204b427SNavdeep Parhar } 5462204b427SNavdeep Parhar /* Not found */ 5476beb67c7SNavdeep Parhar MPASS(i == sc->params.nsched_cls); 548*ec8004ddSNavdeep Parhar if (fa == -1) 549*ec8004ddSNavdeep Parhar fa = fa2; 550*ec8004ddSNavdeep Parhar if (fa == -1) { 551*ec8004ddSNavdeep Parhar *tc_idx = -1; 552*ec8004ddSNavdeep Parhar rc = ENOSPC; 553*ec8004ddSNavdeep Parhar } else { 554*ec8004ddSNavdeep Parhar MPASS(fa >= 0 && fa < sc->params.nsched_cls); 5551979b511SNavdeep Parhar tc = &pi->sched_params->cl_rl[fa]; 556*ec8004ddSNavdeep Parhar MPASS(!(tc->flags & CF_USER)); 557*ec8004ddSNavdeep Parhar MPASS(tc->refcount == 0); 558*ec8004ddSNavdeep Parhar 5592204b427SNavdeep Parhar tc->refcount = 1; 560*ec8004ddSNavdeep Parhar tc->state = CS_HW_UPDATE_REQUESTED; 5612204b427SNavdeep Parhar tc->ratemode = FW_SCHED_PARAMS_RATE_ABS; 5622204b427SNavdeep Parhar tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 5632204b427SNavdeep Parhar tc->mode = FW_SCHED_PARAMS_MODE_FLOW; 5642204b427SNavdeep Parhar tc->maxrate = maxrate; 56509a7189fSNavdeep Parhar tc->pktsize = pktsize; 56609a7189fSNavdeep Parhar tc->burstsize = burstsize; 5672204b427SNavdeep Parhar *tc_idx = fa; 5681979b511SNavdeep Parhar update = true; 5692204b427SNavdeep Parhar } 5702204b427SNavdeep Parhar done: 5712204b427SNavdeep Parhar mtx_unlock(&sc->tc_lock); 572*ec8004ddSNavdeep Parhar if (update) 5731979b511SNavdeep Parhar t4_update_tx_sched(sc); 5742204b427SNavdeep Parhar return (rc); 5752204b427SNavdeep Parhar } 5762204b427SNavdeep Parhar 5772204b427SNavdeep Parhar void 5781979b511SNavdeep Parhar t4_release_cl_rl(struct adapter *sc, int port_id, int tc_idx) 5792204b427SNavdeep Parhar { 5802204b427SNavdeep Parhar struct tx_cl_rl_params *tc; 5812204b427SNavdeep Parhar 5822204b427SNavdeep Parhar MPASS(port_id >= 0 && port_id < sc->params.nports); 5836beb67c7SNavdeep Parhar MPASS(tc_idx >= 0 && tc_idx < sc->params.nsched_cls); 5842204b427SNavdeep Parhar 5852204b427SNavdeep Parhar mtx_lock(&sc->tc_lock); 5862204b427SNavdeep Parhar tc = &sc->port[port_id]->sched_params->cl_rl[tc_idx]; 5872204b427SNavdeep Parhar MPASS(tc->refcount > 0); 5882204b427SNavdeep Parhar tc->refcount--; 5892204b427SNavdeep Parhar mtx_unlock(&sc->tc_lock); 5902204b427SNavdeep Parhar } 59167e07112SNavdeep Parhar 5921979b511SNavdeep Parhar int 5931979b511SNavdeep Parhar sysctl_tc(SYSCTL_HANDLER_ARGS) 5941979b511SNavdeep Parhar { 5951979b511SNavdeep Parhar struct vi_info *vi = arg1; 59643bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 5971979b511SNavdeep Parhar struct sge_txq *txq; 5981979b511SNavdeep Parhar int qidx = arg2, rc, tc_idx; 5991979b511SNavdeep Parhar 60043bbae19SNavdeep Parhar MPASS(qidx >= vi->first_txq && qidx < vi->first_txq + vi->ntxq); 6011979b511SNavdeep Parhar 60243bbae19SNavdeep Parhar txq = &sc->sge.txq[qidx]; 6031979b511SNavdeep Parhar tc_idx = txq->tc_idx; 6041979b511SNavdeep Parhar rc = sysctl_handle_int(oidp, &tc_idx, 0, req); 6051979b511SNavdeep Parhar if (rc != 0 || req->newptr == NULL) 6061979b511SNavdeep Parhar return (rc); 6071979b511SNavdeep Parhar 6081979b511SNavdeep Parhar if (sc->flags & IS_VF) 6091979b511SNavdeep Parhar return (EPERM); 6106beb67c7SNavdeep Parhar if (!in_range(tc_idx, 0, sc->params.nsched_cls - 1)) 6111979b511SNavdeep Parhar return (EINVAL); 6121979b511SNavdeep Parhar 6131979b511SNavdeep Parhar return (bind_txq_to_traffic_class(sc, txq, tc_idx)); 6141979b511SNavdeep Parhar } 6151979b511SNavdeep Parhar 6161979b511SNavdeep Parhar int 6171979b511SNavdeep Parhar sysctl_tc_params(SYSCTL_HANDLER_ARGS) 6181979b511SNavdeep Parhar { 6191979b511SNavdeep Parhar struct adapter *sc = arg1; 6201979b511SNavdeep Parhar struct tx_cl_rl_params tc; 6211979b511SNavdeep Parhar struct sbuf *sb; 6221979b511SNavdeep Parhar int i, rc, port_id, mbps, gbps; 6231979b511SNavdeep Parhar 6241979b511SNavdeep Parhar rc = sysctl_wire_old_buffer(req, 0); 6251979b511SNavdeep Parhar if (rc != 0) 6261979b511SNavdeep Parhar return (rc); 6271979b511SNavdeep Parhar 6281979b511SNavdeep Parhar sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6291979b511SNavdeep Parhar if (sb == NULL) 6301979b511SNavdeep Parhar return (ENOMEM); 6311979b511SNavdeep Parhar 6321979b511SNavdeep Parhar port_id = arg2 >> 16; 6331979b511SNavdeep Parhar MPASS(port_id < sc->params.nports); 6341979b511SNavdeep Parhar MPASS(sc->port[port_id] != NULL); 6351979b511SNavdeep Parhar i = arg2 & 0xffff; 6366beb67c7SNavdeep Parhar MPASS(i < sc->params.nsched_cls); 6371979b511SNavdeep Parhar 6381979b511SNavdeep Parhar mtx_lock(&sc->tc_lock); 6391979b511SNavdeep Parhar tc = sc->port[port_id]->sched_params->cl_rl[i]; 6401979b511SNavdeep Parhar mtx_unlock(&sc->tc_lock); 6411979b511SNavdeep Parhar 642*ec8004ddSNavdeep Parhar if (tc.state < CS_PARAMS_SET) { 643*ec8004ddSNavdeep Parhar sbuf_printf(sb, "uninitialized"); 644*ec8004ddSNavdeep Parhar goto done; 645*ec8004ddSNavdeep Parhar } 646*ec8004ddSNavdeep Parhar 6471979b511SNavdeep Parhar switch (tc.rateunit) { 6481979b511SNavdeep Parhar case SCHED_CLASS_RATEUNIT_BITS: 6491979b511SNavdeep Parhar switch (tc.ratemode) { 6501979b511SNavdeep Parhar case SCHED_CLASS_RATEMODE_REL: 6511979b511SNavdeep Parhar /* XXX: top speed or actual link speed? */ 6521979b511SNavdeep Parhar gbps = port_top_speed(sc->port[port_id]); 6531979b511SNavdeep Parhar sbuf_printf(sb, "%u%% of %uGbps", tc.maxrate, gbps); 6541979b511SNavdeep Parhar break; 6551979b511SNavdeep Parhar case SCHED_CLASS_RATEMODE_ABS: 6561979b511SNavdeep Parhar mbps = tc.maxrate / 1000; 6571979b511SNavdeep Parhar gbps = tc.maxrate / 1000000; 6581979b511SNavdeep Parhar if (tc.maxrate == gbps * 1000000) 6591979b511SNavdeep Parhar sbuf_printf(sb, "%uGbps", gbps); 6601979b511SNavdeep Parhar else if (tc.maxrate == mbps * 1000) 6611979b511SNavdeep Parhar sbuf_printf(sb, "%uMbps", mbps); 6621979b511SNavdeep Parhar else 6631979b511SNavdeep Parhar sbuf_printf(sb, "%uKbps", tc.maxrate); 6641979b511SNavdeep Parhar break; 6651979b511SNavdeep Parhar default: 6661979b511SNavdeep Parhar rc = ENXIO; 6671979b511SNavdeep Parhar goto done; 6681979b511SNavdeep Parhar } 6691979b511SNavdeep Parhar break; 6701979b511SNavdeep Parhar case SCHED_CLASS_RATEUNIT_PKTS: 6711979b511SNavdeep Parhar sbuf_printf(sb, "%upps", tc.maxrate); 6721979b511SNavdeep Parhar break; 6731979b511SNavdeep Parhar default: 6741979b511SNavdeep Parhar rc = ENXIO; 6751979b511SNavdeep Parhar goto done; 6761979b511SNavdeep Parhar } 6771979b511SNavdeep Parhar 6781979b511SNavdeep Parhar switch (tc.mode) { 6791979b511SNavdeep Parhar case SCHED_CLASS_MODE_CLASS: 680*ec8004ddSNavdeep Parhar /* Note that pktsize and burstsize are not used in this mode. */ 6811979b511SNavdeep Parhar sbuf_printf(sb, " aggregate"); 6821979b511SNavdeep Parhar break; 6831979b511SNavdeep Parhar case SCHED_CLASS_MODE_FLOW: 6841979b511SNavdeep Parhar sbuf_printf(sb, " per-flow"); 6858a684e1fSNavdeep Parhar if (tc.pktsize > 0) 6868a684e1fSNavdeep Parhar sbuf_printf(sb, " pkt-size %u", tc.pktsize); 6878a684e1fSNavdeep Parhar if (tc.burstsize > 0) 6888a684e1fSNavdeep Parhar sbuf_printf(sb, " burst-size %u", tc.burstsize); 6891979b511SNavdeep Parhar break; 6901979b511SNavdeep Parhar default: 6911979b511SNavdeep Parhar rc = ENXIO; 6921979b511SNavdeep Parhar goto done; 6931979b511SNavdeep Parhar } 6941979b511SNavdeep Parhar 6951979b511SNavdeep Parhar done: 6961979b511SNavdeep Parhar if (rc == 0) 6971979b511SNavdeep Parhar rc = sbuf_finish(sb); 6981979b511SNavdeep Parhar sbuf_delete(sb); 6991979b511SNavdeep Parhar 7001979b511SNavdeep Parhar return (rc); 7011979b511SNavdeep Parhar } 7021979b511SNavdeep Parhar 70367e07112SNavdeep Parhar #ifdef RATELIMIT 70467e07112SNavdeep Parhar void 70567e07112SNavdeep Parhar t4_init_etid_table(struct adapter *sc) 70667e07112SNavdeep Parhar { 70767e07112SNavdeep Parhar int i; 70867e07112SNavdeep Parhar struct tid_info *t; 70967e07112SNavdeep Parhar 71067e07112SNavdeep Parhar if (!is_ethoffload(sc)) 71167e07112SNavdeep Parhar return; 71267e07112SNavdeep Parhar 71367e07112SNavdeep Parhar t = &sc->tids; 71467e07112SNavdeep Parhar MPASS(t->netids > 0); 71567e07112SNavdeep Parhar 71667e07112SNavdeep Parhar mtx_init(&t->etid_lock, "etid lock", NULL, MTX_DEF); 71767e07112SNavdeep Parhar t->etid_tab = malloc(sizeof(*t->etid_tab) * t->netids, M_CXGBE, 71867e07112SNavdeep Parhar M_ZERO | M_WAITOK); 71967e07112SNavdeep Parhar t->efree = t->etid_tab; 72067e07112SNavdeep Parhar t->etids_in_use = 0; 72167e07112SNavdeep Parhar for (i = 1; i < t->netids; i++) 72267e07112SNavdeep Parhar t->etid_tab[i - 1].next = &t->etid_tab[i]; 72367e07112SNavdeep Parhar t->etid_tab[t->netids - 1].next = NULL; 72467e07112SNavdeep Parhar } 72567e07112SNavdeep Parhar 72667e07112SNavdeep Parhar void 72767e07112SNavdeep Parhar t4_free_etid_table(struct adapter *sc) 72867e07112SNavdeep Parhar { 72967e07112SNavdeep Parhar struct tid_info *t; 73067e07112SNavdeep Parhar 73167e07112SNavdeep Parhar if (!is_ethoffload(sc)) 73267e07112SNavdeep Parhar return; 73367e07112SNavdeep Parhar 73467e07112SNavdeep Parhar t = &sc->tids; 73567e07112SNavdeep Parhar MPASS(t->netids > 0); 73667e07112SNavdeep Parhar 73767e07112SNavdeep Parhar free(t->etid_tab, M_CXGBE); 73867e07112SNavdeep Parhar t->etid_tab = NULL; 73967e07112SNavdeep Parhar 74067e07112SNavdeep Parhar if (mtx_initialized(&t->etid_lock)) 74167e07112SNavdeep Parhar mtx_destroy(&t->etid_lock); 74267e07112SNavdeep Parhar } 74367e07112SNavdeep Parhar 74467e07112SNavdeep Parhar /* etid services */ 745e38a50e8SJohn Baldwin static int alloc_etid(struct adapter *, struct cxgbe_rate_tag *); 74667e07112SNavdeep Parhar static void free_etid(struct adapter *, int); 74767e07112SNavdeep Parhar 74867e07112SNavdeep Parhar static int 749e38a50e8SJohn Baldwin alloc_etid(struct adapter *sc, struct cxgbe_rate_tag *cst) 75067e07112SNavdeep Parhar { 75167e07112SNavdeep Parhar struct tid_info *t = &sc->tids; 75267e07112SNavdeep Parhar int etid = -1; 75367e07112SNavdeep Parhar 75467e07112SNavdeep Parhar mtx_lock(&t->etid_lock); 75567e07112SNavdeep Parhar if (t->efree) { 75667e07112SNavdeep Parhar union etid_entry *p = t->efree; 75767e07112SNavdeep Parhar 75867e07112SNavdeep Parhar etid = p - t->etid_tab + t->etid_base; 75967e07112SNavdeep Parhar t->efree = p->next; 76067e07112SNavdeep Parhar p->cst = cst; 76167e07112SNavdeep Parhar t->etids_in_use++; 76267e07112SNavdeep Parhar } 76367e07112SNavdeep Parhar mtx_unlock(&t->etid_lock); 76467e07112SNavdeep Parhar return (etid); 76567e07112SNavdeep Parhar } 76667e07112SNavdeep Parhar 767e38a50e8SJohn Baldwin struct cxgbe_rate_tag * 76867e07112SNavdeep Parhar lookup_etid(struct adapter *sc, int etid) 76967e07112SNavdeep Parhar { 77067e07112SNavdeep Parhar struct tid_info *t = &sc->tids; 77167e07112SNavdeep Parhar 77267e07112SNavdeep Parhar return (t->etid_tab[etid - t->etid_base].cst); 77367e07112SNavdeep Parhar } 77467e07112SNavdeep Parhar 77567e07112SNavdeep Parhar static void 77667e07112SNavdeep Parhar free_etid(struct adapter *sc, int etid) 77767e07112SNavdeep Parhar { 77867e07112SNavdeep Parhar struct tid_info *t = &sc->tids; 77967e07112SNavdeep Parhar union etid_entry *p = &t->etid_tab[etid - t->etid_base]; 78067e07112SNavdeep Parhar 78167e07112SNavdeep Parhar mtx_lock(&t->etid_lock); 78267e07112SNavdeep Parhar p->next = t->efree; 78367e07112SNavdeep Parhar t->efree = p; 78467e07112SNavdeep Parhar t->etids_in_use--; 78567e07112SNavdeep Parhar mtx_unlock(&t->etid_lock); 78667e07112SNavdeep Parhar } 78767e07112SNavdeep Parhar 78867e07112SNavdeep Parhar int 789e38a50e8SJohn Baldwin cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, 79067e07112SNavdeep Parhar struct m_snd_tag **pt) 79167e07112SNavdeep Parhar { 79267e07112SNavdeep Parhar int rc, schedcl; 79367e07112SNavdeep Parhar struct vi_info *vi = ifp->if_softc; 79467e07112SNavdeep Parhar struct port_info *pi = vi->pi; 79567e07112SNavdeep Parhar struct adapter *sc = pi->adapter; 796e38a50e8SJohn Baldwin struct cxgbe_rate_tag *cst; 79767e07112SNavdeep Parhar 798e38a50e8SJohn Baldwin MPASS(params->hdr.type == IF_SND_TAG_TYPE_RATE_LIMIT); 79967e07112SNavdeep Parhar 80067e07112SNavdeep Parhar rc = t4_reserve_cl_rl_kbps(sc, pi->port_id, 80167e07112SNavdeep Parhar (params->rate_limit.max_rate * 8ULL / 1000), &schedcl); 80267e07112SNavdeep Parhar if (rc != 0) 80367e07112SNavdeep Parhar return (rc); 8046beb67c7SNavdeep Parhar MPASS(schedcl >= 0 && schedcl < sc->params.nsched_cls); 80567e07112SNavdeep Parhar 80667e07112SNavdeep Parhar cst = malloc(sizeof(*cst), M_CXGBE, M_ZERO | M_NOWAIT); 80767e07112SNavdeep Parhar if (cst == NULL) { 80867e07112SNavdeep Parhar failed: 8091979b511SNavdeep Parhar t4_release_cl_rl(sc, pi->port_id, schedcl); 81067e07112SNavdeep Parhar return (ENOMEM); 81167e07112SNavdeep Parhar } 81267e07112SNavdeep Parhar 81367e07112SNavdeep Parhar cst->etid = alloc_etid(sc, cst); 81467e07112SNavdeep Parhar if (cst->etid < 0) { 81567e07112SNavdeep Parhar free(cst, M_CXGBE); 81667e07112SNavdeep Parhar goto failed; 81767e07112SNavdeep Parhar } 81867e07112SNavdeep Parhar 81967e07112SNavdeep Parhar mtx_init(&cst->lock, "cst_lock", NULL, MTX_DEF); 820786099deSNavdeep Parhar mbufq_init(&cst->pending_tx, INT_MAX); 821786099deSNavdeep Parhar mbufq_init(&cst->pending_fwack, INT_MAX); 82256fb710fSJohn Baldwin m_snd_tag_init(&cst->com, ifp, IF_SND_TAG_TYPE_RATE_LIMIT); 823786099deSNavdeep Parhar cst->flags |= EO_FLOWC_PENDING | EO_SND_TAG_REF; 82467e07112SNavdeep Parhar cst->adapter = sc; 82567e07112SNavdeep Parhar cst->port_id = pi->port_id; 82667e07112SNavdeep Parhar cst->schedcl = schedcl; 82767e07112SNavdeep Parhar cst->max_rate = params->rate_limit.max_rate; 828ac8ec5feSNavdeep Parhar cst->tx_credits = sc->params.eo_wr_cred; 82967e07112SNavdeep Parhar cst->tx_total = cst->tx_credits; 830786099deSNavdeep Parhar cst->plen = 0; 831c0236bd9SNavdeep Parhar cst->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 832edb518f4SNavdeep Parhar V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | 833edb518f4SNavdeep Parhar V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 83467e07112SNavdeep Parhar 83567e07112SNavdeep Parhar /* 83667e07112SNavdeep Parhar * Queues will be selected later when the connection flowid is available. 83767e07112SNavdeep Parhar */ 83867e07112SNavdeep Parhar 83956fb710fSJohn Baldwin *pt = &cst->com; 84067e07112SNavdeep Parhar return (0); 84167e07112SNavdeep Parhar } 84267e07112SNavdeep Parhar 84367e07112SNavdeep Parhar /* 84467e07112SNavdeep Parhar * Change in parameters, no change in ifp. 84567e07112SNavdeep Parhar */ 84667e07112SNavdeep Parhar int 847e38a50e8SJohn Baldwin cxgbe_rate_tag_modify(struct m_snd_tag *mst, 84867e07112SNavdeep Parhar union if_snd_tag_modify_params *params) 84967e07112SNavdeep Parhar { 85067e07112SNavdeep Parhar int rc, schedcl; 851e38a50e8SJohn Baldwin struct cxgbe_rate_tag *cst = mst_to_crt(mst); 85267e07112SNavdeep Parhar struct adapter *sc = cst->adapter; 85367e07112SNavdeep Parhar 85467e07112SNavdeep Parhar /* XXX: is schedcl -1 ok here? */ 8556beb67c7SNavdeep Parhar MPASS(cst->schedcl >= 0 && cst->schedcl < sc->params.nsched_cls); 85667e07112SNavdeep Parhar 857786099deSNavdeep Parhar mtx_lock(&cst->lock); 858786099deSNavdeep Parhar MPASS(cst->flags & EO_SND_TAG_REF); 85967e07112SNavdeep Parhar rc = t4_reserve_cl_rl_kbps(sc, cst->port_id, 86067e07112SNavdeep Parhar (params->rate_limit.max_rate * 8ULL / 1000), &schedcl); 86167e07112SNavdeep Parhar if (rc != 0) 86267e07112SNavdeep Parhar return (rc); 8636beb67c7SNavdeep Parhar MPASS(schedcl >= 0 && schedcl < sc->params.nsched_cls); 8641979b511SNavdeep Parhar t4_release_cl_rl(sc, cst->port_id, cst->schedcl); 86567e07112SNavdeep Parhar cst->schedcl = schedcl; 86667e07112SNavdeep Parhar cst->max_rate = params->rate_limit.max_rate; 867786099deSNavdeep Parhar mtx_unlock(&cst->lock); 86867e07112SNavdeep Parhar 86967e07112SNavdeep Parhar return (0); 87067e07112SNavdeep Parhar } 87167e07112SNavdeep Parhar 87267e07112SNavdeep Parhar int 873e38a50e8SJohn Baldwin cxgbe_rate_tag_query(struct m_snd_tag *mst, 87467e07112SNavdeep Parhar union if_snd_tag_query_params *params) 87567e07112SNavdeep Parhar { 876e38a50e8SJohn Baldwin struct cxgbe_rate_tag *cst = mst_to_crt(mst); 87767e07112SNavdeep Parhar 87867e07112SNavdeep Parhar params->rate_limit.max_rate = cst->max_rate; 87967e07112SNavdeep Parhar 88067e07112SNavdeep Parhar #define CST_TO_MST_QLEVEL_SCALE (IF_SND_QUEUE_LEVEL_MAX / cst->tx_total) 88167e07112SNavdeep Parhar params->rate_limit.queue_level = 88267e07112SNavdeep Parhar (cst->tx_total - cst->tx_credits) * CST_TO_MST_QLEVEL_SCALE; 88367e07112SNavdeep Parhar 88467e07112SNavdeep Parhar return (0); 88567e07112SNavdeep Parhar } 88667e07112SNavdeep Parhar 887786099deSNavdeep Parhar /* 888786099deSNavdeep Parhar * Unlocks cst and frees it. 889786099deSNavdeep Parhar */ 89067e07112SNavdeep Parhar void 891e38a50e8SJohn Baldwin cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *cst) 89267e07112SNavdeep Parhar { 89367e07112SNavdeep Parhar struct adapter *sc = cst->adapter; 89467e07112SNavdeep Parhar 895786099deSNavdeep Parhar mtx_assert(&cst->lock, MA_OWNED); 896786099deSNavdeep Parhar MPASS((cst->flags & EO_SND_TAG_REF) == 0); 897786099deSNavdeep Parhar MPASS(cst->tx_credits == cst->tx_total); 898786099deSNavdeep Parhar MPASS(cst->plen == 0); 899786099deSNavdeep Parhar MPASS(mbufq_first(&cst->pending_tx) == NULL); 900786099deSNavdeep Parhar MPASS(mbufq_first(&cst->pending_fwack) == NULL); 901786099deSNavdeep Parhar 90267e07112SNavdeep Parhar if (cst->etid >= 0) 90367e07112SNavdeep Parhar free_etid(sc, cst->etid); 90467e07112SNavdeep Parhar if (cst->schedcl != -1) 9051979b511SNavdeep Parhar t4_release_cl_rl(sc, cst->port_id, cst->schedcl); 906786099deSNavdeep Parhar mtx_unlock(&cst->lock); 90767e07112SNavdeep Parhar mtx_destroy(&cst->lock); 90867e07112SNavdeep Parhar free(cst, M_CXGBE); 90967e07112SNavdeep Parhar } 910786099deSNavdeep Parhar 911786099deSNavdeep Parhar void 912e38a50e8SJohn Baldwin cxgbe_rate_tag_free(struct m_snd_tag *mst) 913786099deSNavdeep Parhar { 914e38a50e8SJohn Baldwin struct cxgbe_rate_tag *cst = mst_to_crt(mst); 915786099deSNavdeep Parhar 916786099deSNavdeep Parhar mtx_lock(&cst->lock); 917786099deSNavdeep Parhar 918786099deSNavdeep Parhar /* The kernel is done with the snd_tag. Remove its reference. */ 919786099deSNavdeep Parhar MPASS(cst->flags & EO_SND_TAG_REF); 920786099deSNavdeep Parhar cst->flags &= ~EO_SND_TAG_REF; 921786099deSNavdeep Parhar 922786099deSNavdeep Parhar if (cst->ncompl == 0) { 923786099deSNavdeep Parhar /* 924786099deSNavdeep Parhar * No fw4_ack in flight. Free the tag right away if there are 925786099deSNavdeep Parhar * no outstanding credits. Request the firmware to return all 926786099deSNavdeep Parhar * credits for the etid otherwise. 927786099deSNavdeep Parhar */ 928786099deSNavdeep Parhar if (cst->tx_credits == cst->tx_total) { 929e38a50e8SJohn Baldwin cxgbe_rate_tag_free_locked(cst); 930786099deSNavdeep Parhar return; /* cst is gone. */ 931786099deSNavdeep Parhar } 932786099deSNavdeep Parhar send_etid_flush_wr(cst); 933786099deSNavdeep Parhar } 934786099deSNavdeep Parhar mtx_unlock(&cst->lock); 935786099deSNavdeep Parhar } 93620abea66SRandall Stewart 93720abea66SRandall Stewart void 938b8b01d9bSNavdeep Parhar cxgbe_ratelimit_query(struct ifnet *ifp, struct if_ratelimit_query_results *q) 93920abea66SRandall Stewart { 940b8b01d9bSNavdeep Parhar struct vi_info *vi = ifp->if_softc; 941b8b01d9bSNavdeep Parhar struct adapter *sc = vi->adapter; 942b8b01d9bSNavdeep Parhar 94320abea66SRandall Stewart q->rate_table = NULL; 94420abea66SRandall Stewart q->flags = RT_IS_SELECTABLE; 945b8b01d9bSNavdeep Parhar /* 946b8b01d9bSNavdeep Parhar * Absolute max limits from the firmware configuration. Practical 947b8b01d9bSNavdeep Parhar * limits depend on the burstsize, pktsize (ifp->if_mtu ultimately) and 948b8b01d9bSNavdeep Parhar * the card's cclk. 949b8b01d9bSNavdeep Parhar */ 950b8b01d9bSNavdeep Parhar q->max_flows = sc->tids.netids; 9516beb67c7SNavdeep Parhar q->number_of_rates = sc->params.nsched_cls; 952b8b01d9bSNavdeep Parhar q->min_segment_burst = 4; /* matches PKTSCHED_BURST in the firmware. */ 953b8b01d9bSNavdeep Parhar 954b8b01d9bSNavdeep Parhar #if 1 955b8b01d9bSNavdeep Parhar if (chip_id(sc) < CHELSIO_T6) { 956b8b01d9bSNavdeep Parhar /* Based on testing by rrs@ with a T580 at burstsize = 4. */ 957b8b01d9bSNavdeep Parhar MPASS(q->min_segment_burst == 4); 958e2e43aafSNavdeep Parhar q->max_flows = min(4000, q->max_flows); 959b8b01d9bSNavdeep Parhar } else { 960b8b01d9bSNavdeep Parhar /* XXX: TBD, carried forward from T5 for now. */ 961e2e43aafSNavdeep Parhar q->max_flows = min(4000, q->max_flows); 962b8b01d9bSNavdeep Parhar } 963b8b01d9bSNavdeep Parhar 964b8b01d9bSNavdeep Parhar /* 965b8b01d9bSNavdeep Parhar * XXX: tcp_ratelimit.c grabs all available rates on link-up before it 966b8b01d9bSNavdeep Parhar * even knows whether hw pacing will be used or not. This prevents 967b8b01d9bSNavdeep Parhar * other consumers like SO_MAX_PACING_RATE or those using cxgbetool or 968b8b01d9bSNavdeep Parhar * the private ioctls from using any of traffic classes. 969b8b01d9bSNavdeep Parhar * 970b8b01d9bSNavdeep Parhar * Underreport the number of rates to tcp_ratelimit so that it doesn't 971b8b01d9bSNavdeep Parhar * hog all of them. This can be removed if/when tcp_ratelimit switches 972b8b01d9bSNavdeep Parhar * to making its allocations on first-use rather than link-up. There is 973b8b01d9bSNavdeep Parhar * nothing wrong with one particular consumer reserving all the classes 974b8b01d9bSNavdeep Parhar * but it should do so only if it'll actually use hw rate limiting. 975b8b01d9bSNavdeep Parhar */ 976b8b01d9bSNavdeep Parhar q->number_of_rates /= 4; 977b8b01d9bSNavdeep Parhar #endif 97820abea66SRandall Stewart } 97967e07112SNavdeep Parhar #endif 980