1 /*- 2 * Copyright (c) 2017 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/types.h> 35 #include <sys/malloc.h> 36 #include <sys/queue.h> 37 #include <sys/sbuf.h> 38 #include <sys/taskqueue.h> 39 #include <sys/sysctl.h> 40 41 #include "common/common.h" 42 #include "common/t4_regs.h" 43 #include "common/t4_regs_values.h" 44 #include "common/t4_msg.h" 45 46 47 static int 48 in_range(int val, int lo, int hi) 49 { 50 51 return (val < 0 || (val <= hi && val >= lo)); 52 } 53 54 static int 55 set_sched_class_config(struct adapter *sc, int minmax) 56 { 57 int rc; 58 59 if (minmax < 0) 60 return (EINVAL); 61 62 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc"); 63 if (rc) 64 return (rc); 65 rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1); 66 end_synchronized_op(sc, 0); 67 68 return (rc); 69 } 70 71 static int 72 set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p, 73 int sleep_ok) 74 { 75 int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode; 76 struct port_info *pi; 77 struct tx_cl_rl_params *tc; 78 79 if (p->level == SCHED_CLASS_LEVEL_CL_RL) 80 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 81 else if (p->level == SCHED_CLASS_LEVEL_CL_WRR) 82 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 83 else if (p->level == SCHED_CLASS_LEVEL_CH_RL) 84 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 85 else 86 return (EINVAL); 87 88 if (p->mode == SCHED_CLASS_MODE_CLASS) 89 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 90 else if (p->mode == SCHED_CLASS_MODE_FLOW) 91 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 92 else 93 return (EINVAL); 94 95 if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS) 96 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 97 else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS) 98 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 99 else 100 return (EINVAL); 101 102 if (p->ratemode == SCHED_CLASS_RATEMODE_REL) 103 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 104 else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS) 105 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 106 else 107 return (EINVAL); 108 109 /* Vet our parameters ... */ 110 if (!in_range(p->channel, 0, sc->chip_params->nchan - 1)) 111 return (ERANGE); 112 113 pi = sc->port[sc->chan_map[p->channel]]; 114 if (pi == NULL) 115 return (ENXIO); 116 MPASS(pi->tx_chan == p->channel); 117 top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */ 118 119 if (!in_range(p->cl, 0, sc->chip_params->nsched_cls) || 120 !in_range(p->minrate, 0, top_speed) || 121 !in_range(p->maxrate, 0, top_speed) || 122 !in_range(p->weight, 0, 100)) 123 return (ERANGE); 124 125 /* 126 * Translate any unset parameters into the firmware's 127 * nomenclature and/or fail the call if the parameters 128 * are required ... 129 */ 130 if (p->rateunit < 0 || p->ratemode < 0 || p->channel < 0 || p->cl < 0) 131 return (EINVAL); 132 133 if (p->minrate < 0) 134 p->minrate = 0; 135 if (p->maxrate < 0) { 136 if (p->level == SCHED_CLASS_LEVEL_CL_RL || 137 p->level == SCHED_CLASS_LEVEL_CH_RL) 138 return (EINVAL); 139 else 140 p->maxrate = 0; 141 } 142 if (p->weight < 0) { 143 if (p->level == SCHED_CLASS_LEVEL_CL_WRR) 144 return (EINVAL); 145 else 146 p->weight = 0; 147 } 148 if (p->pktsize < 0) { 149 if (p->level == SCHED_CLASS_LEVEL_CL_RL || 150 p->level == SCHED_CLASS_LEVEL_CH_RL) 151 return (EINVAL); 152 else 153 p->pktsize = 0; 154 } 155 156 rc = begin_synchronized_op(sc, NULL, 157 sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp"); 158 if (rc) 159 return (rc); 160 if (p->level == SCHED_CLASS_LEVEL_CL_RL) { 161 tc = &pi->sched_params->cl_rl[p->cl]; 162 if (tc->refcount > 0) { 163 rc = EBUSY; 164 goto done; 165 } else { 166 tc->ratemode = fw_ratemode; 167 tc->rateunit = fw_rateunit; 168 tc->mode = fw_mode; 169 tc->maxrate = p->maxrate; 170 tc->pktsize = p->pktsize; 171 } 172 } 173 rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, fw_mode, 174 fw_rateunit, fw_ratemode, p->channel, p->cl, p->minrate, p->maxrate, 175 p->weight, p->pktsize, sleep_ok); 176 if (p->level == SCHED_CLASS_LEVEL_CL_RL && rc != 0) { 177 /* 178 * Unknown state at this point, see parameters in tc for what 179 * was attempted. 180 */ 181 tc->flags |= TX_CLRL_ERROR; 182 } 183 done: 184 end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD); 185 186 return (rc); 187 } 188 189 static void 190 update_tx_sched(void *context, int pending) 191 { 192 int i, j, mode, rateunit, ratemode, maxrate, pktsize, rc; 193 struct port_info *pi; 194 struct tx_cl_rl_params *tc; 195 struct adapter *sc = context; 196 const int n = sc->chip_params->nsched_cls; 197 198 mtx_lock(&sc->tc_lock); 199 for_each_port(sc, i) { 200 pi = sc->port[i]; 201 tc = &pi->sched_params->cl_rl[0]; 202 for (j = 0; j < n; j++, tc++) { 203 MPASS(mtx_owned(&sc->tc_lock)); 204 if ((tc->flags & TX_CLRL_REFRESH) == 0) 205 continue; 206 207 mode = tc->mode; 208 rateunit = tc->rateunit; 209 ratemode = tc->ratemode; 210 maxrate = tc->maxrate; 211 pktsize = tc->pktsize; 212 mtx_unlock(&sc->tc_lock); 213 214 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 215 "t4utxs") != 0) { 216 mtx_lock(&sc->tc_lock); 217 continue; 218 } 219 rc = t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, 220 FW_SCHED_PARAMS_LEVEL_CL_RL, mode, rateunit, 221 ratemode, pi->tx_chan, j, 0, maxrate, 0, pktsize, 222 1); 223 end_synchronized_op(sc, 0); 224 225 mtx_lock(&sc->tc_lock); 226 if (rc != 0) { 227 tc->flags |= TX_CLRL_ERROR; 228 } else if (tc->mode == mode && 229 tc->rateunit == rateunit && 230 tc->maxrate == maxrate && 231 tc->pktsize == tc->pktsize) { 232 tc->flags &= ~(TX_CLRL_REFRESH | TX_CLRL_ERROR); 233 } 234 } 235 } 236 mtx_unlock(&sc->tc_lock); 237 } 238 239 int 240 t4_set_sched_class(struct adapter *sc, struct t4_sched_params *p) 241 { 242 243 if (p->type != SCHED_CLASS_TYPE_PACKET) 244 return (EINVAL); 245 246 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 247 return (set_sched_class_config(sc, p->u.config.minmax)); 248 249 if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 250 return (set_sched_class_params(sc, &p->u.params, 1)); 251 252 return (EINVAL); 253 } 254 255 int 256 t4_set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 257 { 258 struct port_info *pi = NULL; 259 struct vi_info *vi; 260 struct sge_txq *txq; 261 uint32_t fw_mnem, fw_queue, fw_class; 262 int i, rc; 263 264 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 265 if (rc) 266 return (rc); 267 268 if (p->port >= sc->params.nports) { 269 rc = EINVAL; 270 goto done; 271 } 272 273 /* XXX: Only supported for the main VI. */ 274 pi = sc->port[p->port]; 275 vi = &pi->vi[0]; 276 if (!(vi->flags & VI_INIT_DONE)) { 277 /* tx queues not set up yet */ 278 rc = EAGAIN; 279 goto done; 280 } 281 282 if (!in_range(p->queue, 0, vi->ntxq - 1) || 283 !in_range(p->cl, 0, sc->chip_params->nsched_cls - 1)) { 284 rc = EINVAL; 285 goto done; 286 } 287 288 /* 289 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 290 * Scheduling Class in this case). 291 */ 292 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 293 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 294 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 295 296 /* 297 * If op.queue is non-negative, then we're only changing the scheduling 298 * on a single specified TX queue. 299 */ 300 if (p->queue >= 0) { 301 txq = &sc->sge.txq[vi->first_txq + p->queue]; 302 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 303 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 304 &fw_class); 305 goto done; 306 } 307 308 /* 309 * Change the scheduling on all the TX queues for the 310 * interface. 311 */ 312 for_each_txq(vi, i, txq) { 313 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 314 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 315 &fw_class); 316 if (rc) 317 goto done; 318 } 319 320 rc = 0; 321 done: 322 end_synchronized_op(sc, 0); 323 return (rc); 324 } 325 326 int 327 t4_init_tx_sched(struct adapter *sc) 328 { 329 int i, j; 330 const int n = sc->chip_params->nsched_cls; 331 struct port_info *pi; 332 struct tx_cl_rl_params *tc; 333 static const uint32_t init_kbps[] = { 334 100 * 1000, 335 200 * 1000, 336 400 * 1000, 337 500 * 1000, 338 800 * 1000, 339 1000 * 1000, 340 1200 * 1000, 341 1500 * 1000, 342 1800 * 1000, 343 2000 * 1000, 344 2500 * 1000, 345 3000 * 1000, 346 3500 * 1000, 347 4000 * 1000, 348 5000 * 1000, 349 10000 * 1000 350 }; 351 352 mtx_init(&sc->tc_lock, "tx_sched lock", NULL, MTX_DEF); 353 TASK_INIT(&sc->tc_task, 0, update_tx_sched, sc); 354 for_each_port(sc, i) { 355 pi = sc->port[i]; 356 pi->sched_params = malloc(sizeof(*pi->sched_params) + 357 n * sizeof(*tc), M_CXGBE, M_ZERO | M_WAITOK); 358 tc = &pi->sched_params->cl_rl[0]; 359 for (j = 0; j < n; j++, tc++) { 360 tc->refcount = 0; 361 tc->ratemode = FW_SCHED_PARAMS_RATE_ABS; 362 tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 363 tc->mode = FW_SCHED_PARAMS_MODE_FLOW; 364 tc->maxrate = init_kbps[min(j, nitems(init_kbps) - 1)]; 365 tc->pktsize = ETHERMTU; /* XXX */ 366 367 if (t4_sched_params_cl_rl_kbps(sc, pi->tx_chan, j, 368 tc->mode, tc->maxrate, tc->pktsize, 1) == 0) 369 tc->flags = 0; 370 else 371 tc->flags = TX_CLRL_ERROR; 372 } 373 } 374 375 return (0); 376 } 377 378 int 379 t4_free_tx_sched(struct adapter *sc) 380 { 381 int i; 382 383 taskqueue_drain(taskqueue_thread, &sc->tc_task); 384 385 for_each_port(sc, i) { 386 if (sc->port[i] != NULL) 387 free(sc->port[i]->sched_params, M_CXGBE); 388 } 389 390 if (mtx_initialized(&sc->tc_lock)) 391 mtx_destroy(&sc->tc_lock); 392 393 return (0); 394 } 395 396 void 397 t4_update_tx_sched(struct adapter *sc) 398 { 399 400 taskqueue_enqueue(taskqueue_thread, &sc->tc_task); 401 } 402 403 int 404 t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate, 405 int *tc_idx) 406 { 407 int rc = 0, fa = -1, i; 408 struct tx_cl_rl_params *tc; 409 410 MPASS(port_id >= 0 && port_id < sc->params.nports); 411 412 tc = &sc->port[port_id]->sched_params->cl_rl[0]; 413 mtx_lock(&sc->tc_lock); 414 for (i = 0; i < sc->chip_params->nsched_cls; i++, tc++) { 415 if (fa < 0 && tc->refcount == 0) 416 fa = i; 417 418 if (tc->ratemode == FW_SCHED_PARAMS_RATE_ABS && 419 tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE && 420 tc->mode == FW_SCHED_PARAMS_MODE_FLOW && 421 tc->maxrate == maxrate) { 422 tc->refcount++; 423 *tc_idx = i; 424 goto done; 425 } 426 } 427 /* Not found */ 428 MPASS(i == sc->chip_params->nsched_cls); 429 if (fa != -1) { 430 tc = &sc->port[port_id]->sched_params->cl_rl[fa]; 431 tc->flags = TX_CLRL_REFRESH; 432 tc->refcount = 1; 433 tc->ratemode = FW_SCHED_PARAMS_RATE_ABS; 434 tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 435 tc->mode = FW_SCHED_PARAMS_MODE_FLOW; 436 tc->maxrate = maxrate; 437 tc->pktsize = ETHERMTU; /* XXX */ 438 *tc_idx = fa; 439 t4_update_tx_sched(sc); 440 } else { 441 *tc_idx = -1; 442 rc = ENOSPC; 443 } 444 done: 445 mtx_unlock(&sc->tc_lock); 446 return (rc); 447 } 448 449 void 450 t4_release_cl_rl_kbps(struct adapter *sc, int port_id, int tc_idx) 451 { 452 struct tx_cl_rl_params *tc; 453 454 MPASS(port_id >= 0 && port_id < sc->params.nports); 455 MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls); 456 457 mtx_lock(&sc->tc_lock); 458 tc = &sc->port[port_id]->sched_params->cl_rl[tc_idx]; 459 MPASS(tc->refcount > 0); 460 MPASS(tc->ratemode == FW_SCHED_PARAMS_RATE_ABS); 461 MPASS(tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE); 462 MPASS(tc->mode == FW_SCHED_PARAMS_MODE_FLOW); 463 tc->refcount--; 464 mtx_unlock(&sc->tc_lock); 465 } 466