1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * net/sched/sch_ets.c Enhanced Transmission Selection scheduler 4 * 5 * Description 6 * ----------- 7 * 8 * The Enhanced Transmission Selection scheduler is a classful queuing 9 * discipline that merges functionality of PRIO and DRR qdiscs in one scheduler. 10 * ETS makes it easy to configure a set of strict and bandwidth-sharing bands to 11 * implement the transmission selection described in 802.1Qaz. 12 * 13 * Although ETS is technically classful, it's not possible to add and remove 14 * classes at will. Instead one specifies number of classes, how many are 15 * PRIO-like and how many DRR-like, and quanta for the latter. 16 * 17 * Algorithm 18 * --------- 19 * 20 * The strict classes, if any, are tried for traffic first: first band 0, if it 21 * has no traffic then band 1, etc. 22 * 23 * When there is no traffic in any of the strict queues, the bandwidth-sharing 24 * ones are tried next. Each band is assigned a deficit counter, initialized to 25 * "quantum" of that band. ETS maintains a list of active bandwidth-sharing 26 * bands whose qdiscs are non-empty. A packet is dequeued from the band at the 27 * head of the list if the packet size is smaller or equal to the deficit 28 * counter. If the counter is too small, it is increased by "quantum" and the 29 * scheduler moves on to the next band in the active list. 30 */ 31 32 #include <linux/module.h> 33 #include <net/gen_stats.h> 34 #include <net/netlink.h> 35 #include <net/pkt_cls.h> 36 #include <net/pkt_sched.h> 37 #include <net/sch_generic.h> 38 39 struct ets_class { 40 struct list_head alist; /* In struct ets_sched.active. */ 41 struct Qdisc *qdisc; 42 u32 quantum; 43 u32 deficit; 44 struct gnet_stats_basic_sync bstats; 45 struct gnet_stats_queue qstats; 46 }; 47 48 struct ets_sched { 49 struct list_head active; 50 struct tcf_proto __rcu *filter_list; 51 struct tcf_block *block; 52 unsigned int nbands; 53 unsigned int nstrict; 54 u8 prio2band[TC_PRIO_MAX + 1]; 55 struct ets_class classes[TCQ_ETS_MAX_BANDS]; 56 }; 57 58 static const struct nla_policy ets_policy[TCA_ETS_MAX + 1] = { 59 [TCA_ETS_NBANDS] = { .type = NLA_U8 }, 60 [TCA_ETS_NSTRICT] = { .type = NLA_U8 }, 61 [TCA_ETS_QUANTA] = { .type = NLA_NESTED }, 62 [TCA_ETS_PRIOMAP] = { .type = NLA_NESTED }, 63 }; 64 65 static const struct nla_policy ets_priomap_policy[TCA_ETS_MAX + 1] = { 66 [TCA_ETS_PRIOMAP_BAND] = { .type = NLA_U8 }, 67 }; 68 69 static const struct nla_policy ets_quanta_policy[TCA_ETS_MAX + 1] = { 70 [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 }, 71 }; 72 73 static const struct nla_policy ets_class_policy[TCA_ETS_MAX + 1] = { 74 [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 }, 75 }; 76 77 static bool cl_is_active(struct ets_class *cl) 78 { 79 return !list_empty(&cl->alist); 80 } 81 82 static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr, 83 unsigned int *quantum, 84 struct netlink_ext_ack *extack) 85 { 86 *quantum = nla_get_u32(attr); 87 if (!*quantum) { 88 NL_SET_ERR_MSG(extack, "ETS quantum cannot be zero"); 89 return -EINVAL; 90 } 91 return 0; 92 } 93 94 static struct ets_class * 95 ets_class_from_arg(struct Qdisc *sch, unsigned long arg) 96 { 97 struct ets_sched *q = qdisc_priv(sch); 98 99 if (arg == 0 || arg > q->nbands) 100 return NULL; 101 return &q->classes[arg - 1]; 102 } 103 104 static u32 ets_class_id(struct Qdisc *sch, const struct ets_class *cl) 105 { 106 struct ets_sched *q = qdisc_priv(sch); 107 int band = cl - q->classes; 108 109 return TC_H_MAKE(sch->handle, band + 1); 110 } 111 112 static void ets_offload_change(struct Qdisc *sch) 113 { 114 struct net_device *dev = qdisc_dev(sch); 115 struct ets_sched *q = qdisc_priv(sch); 116 struct tc_ets_qopt_offload qopt; 117 unsigned int w_psum_prev = 0; 118 unsigned int quantum; 119 unsigned int w_psum; 120 unsigned int weight; 121 unsigned int i; 122 u64 q_psum = 0; 123 u64 q_sum = 0; 124 125 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 126 return; 127 128 qopt.command = TC_ETS_REPLACE; 129 qopt.handle = sch->handle; 130 qopt.parent = sch->parent; 131 qopt.replace_params.bands = q->nbands; 132 qopt.replace_params.qstats = &sch->qstats; 133 memcpy(&qopt.replace_params.priomap, 134 q->prio2band, sizeof(q->prio2band)); 135 136 for (i = 0; i < q->nbands; i++) 137 q_sum += q->classes[i].quantum; 138 139 for (i = 0; i < q->nbands; i++) { 140 quantum = q->classes[i].quantum; 141 if (quantum) { 142 q_psum += quantum; 143 w_psum = div64_u64(q_psum * 100, q_sum); 144 } else { 145 w_psum = 0; 146 } 147 weight = w_psum - w_psum_prev; 148 w_psum_prev = w_psum; 149 150 qopt.replace_params.quanta[i] = quantum; 151 qopt.replace_params.weights[i] = weight; 152 } 153 154 dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt); 155 } 156 157 static void ets_offload_destroy(struct Qdisc *sch) 158 { 159 struct net_device *dev = qdisc_dev(sch); 160 struct tc_ets_qopt_offload qopt; 161 162 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 163 return; 164 165 qopt.command = TC_ETS_DESTROY; 166 qopt.handle = sch->handle; 167 qopt.parent = sch->parent; 168 dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt); 169 } 170 171 static void ets_offload_graft(struct Qdisc *sch, struct Qdisc *new, 172 struct Qdisc *old, unsigned long arg, 173 struct netlink_ext_ack *extack) 174 { 175 struct net_device *dev = qdisc_dev(sch); 176 struct tc_ets_qopt_offload qopt; 177 178 qopt.command = TC_ETS_GRAFT; 179 qopt.handle = sch->handle; 180 qopt.parent = sch->parent; 181 qopt.graft_params.band = arg - 1; 182 qopt.graft_params.child_handle = new->handle; 183 184 qdisc_offload_graft_helper(dev, sch, new, old, TC_SETUP_QDISC_ETS, 185 &qopt, extack); 186 } 187 188 static int ets_offload_dump(struct Qdisc *sch) 189 { 190 struct tc_ets_qopt_offload qopt; 191 192 qopt.command = TC_ETS_STATS; 193 qopt.handle = sch->handle; 194 qopt.parent = sch->parent; 195 qopt.stats.bstats = &sch->bstats; 196 qopt.stats.qstats = &sch->qstats; 197 198 return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_ETS, &qopt); 199 } 200 201 static bool ets_class_is_strict(struct ets_sched *q, const struct ets_class *cl) 202 { 203 unsigned int band = cl - q->classes; 204 205 return band < q->nstrict; 206 } 207 208 static int ets_class_change(struct Qdisc *sch, u32 classid, u32 parentid, 209 struct nlattr **tca, unsigned long *arg, 210 struct netlink_ext_ack *extack) 211 { 212 struct ets_class *cl = ets_class_from_arg(sch, *arg); 213 struct ets_sched *q = qdisc_priv(sch); 214 struct nlattr *opt = tca[TCA_OPTIONS]; 215 struct nlattr *tb[TCA_ETS_MAX + 1]; 216 unsigned int quantum; 217 int err; 218 219 /* Classes can be added and removed only through Qdisc_ops.change 220 * interface. 221 */ 222 if (!cl) { 223 NL_SET_ERR_MSG(extack, "Fine-grained class addition and removal is not supported"); 224 return -EOPNOTSUPP; 225 } 226 227 if (!opt) { 228 NL_SET_ERR_MSG(extack, "ETS options are required for this operation"); 229 return -EINVAL; 230 } 231 232 err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_class_policy, extack); 233 if (err < 0) 234 return err; 235 236 if (!tb[TCA_ETS_QUANTA_BAND]) 237 /* Nothing to configure. */ 238 return 0; 239 240 if (ets_class_is_strict(q, cl)) { 241 NL_SET_ERR_MSG(extack, "Strict bands do not have a configurable quantum"); 242 return -EINVAL; 243 } 244 245 err = ets_quantum_parse(sch, tb[TCA_ETS_QUANTA_BAND], &quantum, 246 extack); 247 if (err) 248 return err; 249 250 sch_tree_lock(sch); 251 cl->quantum = quantum; 252 sch_tree_unlock(sch); 253 254 ets_offload_change(sch); 255 return 0; 256 } 257 258 static int ets_class_graft(struct Qdisc *sch, unsigned long arg, 259 struct Qdisc *new, struct Qdisc **old, 260 struct netlink_ext_ack *extack) 261 { 262 struct ets_class *cl = ets_class_from_arg(sch, arg); 263 264 if (!new) { 265 new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, 266 ets_class_id(sch, cl), NULL); 267 if (!new) 268 new = &noop_qdisc; 269 else 270 qdisc_hash_add(new, true); 271 } 272 273 *old = qdisc_replace(sch, new, &cl->qdisc); 274 ets_offload_graft(sch, new, *old, arg, extack); 275 return 0; 276 } 277 278 static struct Qdisc *ets_class_leaf(struct Qdisc *sch, unsigned long arg) 279 { 280 struct ets_class *cl = ets_class_from_arg(sch, arg); 281 282 return cl->qdisc; 283 } 284 285 static unsigned long ets_class_find(struct Qdisc *sch, u32 classid) 286 { 287 unsigned long band = TC_H_MIN(classid); 288 struct ets_sched *q = qdisc_priv(sch); 289 290 if (band - 1 >= q->nbands) 291 return 0; 292 return band; 293 } 294 295 static void ets_class_qlen_notify(struct Qdisc *sch, unsigned long arg) 296 { 297 struct ets_class *cl = ets_class_from_arg(sch, arg); 298 struct ets_sched *q = qdisc_priv(sch); 299 300 /* We get notified about zero-length child Qdiscs as well if they are 301 * offloaded. Those aren't on the active list though, so don't attempt 302 * to remove them. 303 */ 304 if (!ets_class_is_strict(q, cl) && sch->q.qlen) 305 list_del_init(&cl->alist); 306 } 307 308 static int ets_class_dump(struct Qdisc *sch, unsigned long arg, 309 struct sk_buff *skb, struct tcmsg *tcm) 310 { 311 struct ets_class *cl = ets_class_from_arg(sch, arg); 312 struct ets_sched *q = qdisc_priv(sch); 313 struct nlattr *nest; 314 315 tcm->tcm_parent = TC_H_ROOT; 316 tcm->tcm_handle = ets_class_id(sch, cl); 317 tcm->tcm_info = cl->qdisc->handle; 318 319 nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 320 if (!nest) 321 goto nla_put_failure; 322 if (!ets_class_is_strict(q, cl)) { 323 if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, cl->quantum)) 324 goto nla_put_failure; 325 } 326 return nla_nest_end(skb, nest); 327 328 nla_put_failure: 329 nla_nest_cancel(skb, nest); 330 return -EMSGSIZE; 331 } 332 333 static int ets_class_dump_stats(struct Qdisc *sch, unsigned long arg, 334 struct gnet_dump *d) 335 { 336 struct ets_class *cl = ets_class_from_arg(sch, arg); 337 struct Qdisc *cl_q = cl->qdisc; 338 339 if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats, true) < 0 || 340 qdisc_qstats_copy(d, cl_q) < 0) 341 return -1; 342 343 return 0; 344 } 345 346 static void ets_qdisc_walk(struct Qdisc *sch, struct qdisc_walker *arg) 347 { 348 struct ets_sched *q = qdisc_priv(sch); 349 int i; 350 351 if (arg->stop) 352 return; 353 354 for (i = 0; i < q->nbands; i++) { 355 if (!tc_qdisc_stats_dump(sch, i + 1, arg)) 356 break; 357 } 358 } 359 360 static struct tcf_block * 361 ets_qdisc_tcf_block(struct Qdisc *sch, unsigned long cl, 362 struct netlink_ext_ack *extack) 363 { 364 struct ets_sched *q = qdisc_priv(sch); 365 366 if (cl) { 367 NL_SET_ERR_MSG(extack, "ETS classid must be zero"); 368 return NULL; 369 } 370 371 return q->block; 372 } 373 374 static unsigned long ets_qdisc_bind_tcf(struct Qdisc *sch, unsigned long parent, 375 u32 classid) 376 { 377 return ets_class_find(sch, classid); 378 } 379 380 static void ets_qdisc_unbind_tcf(struct Qdisc *sch, unsigned long arg) 381 { 382 } 383 384 static struct ets_class *ets_classify(struct sk_buff *skb, struct Qdisc *sch, 385 int *qerr) 386 { 387 struct ets_sched *q = qdisc_priv(sch); 388 u32 band = skb->priority; 389 struct tcf_result res; 390 struct tcf_proto *fl; 391 int err; 392 393 *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; 394 if (TC_H_MAJ(skb->priority) != sch->handle) { 395 fl = rcu_dereference_bh(q->filter_list); 396 err = tcf_classify(skb, NULL, fl, &res, false); 397 #ifdef CONFIG_NET_CLS_ACT 398 switch (err) { 399 case TC_ACT_STOLEN: 400 case TC_ACT_QUEUED: 401 case TC_ACT_TRAP: 402 *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN; 403 fallthrough; 404 case TC_ACT_SHOT: 405 return NULL; 406 } 407 #endif 408 if (!fl || err < 0) { 409 if (TC_H_MAJ(band)) 410 band = 0; 411 return &q->classes[q->prio2band[band & TC_PRIO_MAX]]; 412 } 413 band = res.classid; 414 } 415 band = TC_H_MIN(band) - 1; 416 if (band >= q->nbands) 417 return &q->classes[q->prio2band[0]]; 418 return &q->classes[band]; 419 } 420 421 static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 422 struct sk_buff **to_free) 423 { 424 unsigned int len = qdisc_pkt_len(skb); 425 struct ets_sched *q = qdisc_priv(sch); 426 struct ets_class *cl; 427 int err = 0; 428 429 cl = ets_classify(skb, sch, &err); 430 if (!cl) { 431 if (err & __NET_XMIT_BYPASS) 432 qdisc_qstats_drop(sch); 433 __qdisc_drop(skb, to_free); 434 return err; 435 } 436 437 err = qdisc_enqueue(skb, cl->qdisc, to_free); 438 if (unlikely(err != NET_XMIT_SUCCESS)) { 439 if (net_xmit_drop_count(err)) { 440 cl->qstats.drops++; 441 qdisc_qstats_drop(sch); 442 } 443 return err; 444 } 445 446 if (!cl_is_active(cl) && !ets_class_is_strict(q, cl)) { 447 list_add_tail(&cl->alist, &q->active); 448 cl->deficit = cl->quantum; 449 } 450 451 sch->qstats.backlog += len; 452 sch->q.qlen++; 453 return err; 454 } 455 456 static struct sk_buff * 457 ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb) 458 { 459 qdisc_bstats_update(sch, skb); 460 qdisc_qstats_backlog_dec(sch, skb); 461 sch->q.qlen--; 462 return skb; 463 } 464 465 static struct sk_buff *ets_qdisc_dequeue(struct Qdisc *sch) 466 { 467 struct ets_sched *q = qdisc_priv(sch); 468 struct ets_class *cl; 469 struct sk_buff *skb; 470 unsigned int band; 471 unsigned int len; 472 473 while (1) { 474 for (band = 0; band < q->nstrict; band++) { 475 cl = &q->classes[band]; 476 skb = qdisc_dequeue_peeked(cl->qdisc); 477 if (skb) 478 return ets_qdisc_dequeue_skb(sch, skb); 479 } 480 481 if (list_empty(&q->active)) 482 goto out; 483 484 cl = list_first_entry(&q->active, struct ets_class, alist); 485 skb = cl->qdisc->ops->peek(cl->qdisc); 486 if (!skb) { 487 qdisc_warn_nonwc(__func__, cl->qdisc); 488 goto out; 489 } 490 491 len = qdisc_pkt_len(skb); 492 if (len <= cl->deficit) { 493 cl->deficit -= len; 494 skb = qdisc_dequeue_peeked(cl->qdisc); 495 if (unlikely(!skb)) 496 goto out; 497 if (cl->qdisc->q.qlen == 0) 498 list_del_init(&cl->alist); 499 return ets_qdisc_dequeue_skb(sch, skb); 500 } 501 502 cl->deficit += cl->quantum; 503 list_move_tail(&cl->alist, &q->active); 504 } 505 out: 506 return NULL; 507 } 508 509 static int ets_qdisc_priomap_parse(struct nlattr *priomap_attr, 510 unsigned int nbands, u8 *priomap, 511 struct netlink_ext_ack *extack) 512 { 513 const struct nlattr *attr; 514 int prio = 0; 515 u8 band; 516 int rem; 517 int err; 518 519 err = __nla_validate_nested(priomap_attr, TCA_ETS_MAX, 520 ets_priomap_policy, NL_VALIDATE_STRICT, 521 extack); 522 if (err) 523 return err; 524 525 nla_for_each_nested(attr, priomap_attr, rem) { 526 switch (nla_type(attr)) { 527 case TCA_ETS_PRIOMAP_BAND: 528 if (prio > TC_PRIO_MAX) { 529 NL_SET_ERR_MSG_MOD(extack, "Too many priorities in ETS priomap"); 530 return -EINVAL; 531 } 532 band = nla_get_u8(attr); 533 if (band >= nbands) { 534 NL_SET_ERR_MSG_MOD(extack, "Invalid band number in ETS priomap"); 535 return -EINVAL; 536 } 537 priomap[prio++] = band; 538 break; 539 default: 540 WARN_ON_ONCE(1); /* Validate should have caught this. */ 541 return -EINVAL; 542 } 543 } 544 545 return 0; 546 } 547 548 static int ets_qdisc_quanta_parse(struct Qdisc *sch, struct nlattr *quanta_attr, 549 unsigned int nbands, unsigned int nstrict, 550 unsigned int *quanta, 551 struct netlink_ext_ack *extack) 552 { 553 const struct nlattr *attr; 554 int band = nstrict; 555 int rem; 556 int err; 557 558 err = __nla_validate_nested(quanta_attr, TCA_ETS_MAX, 559 ets_quanta_policy, NL_VALIDATE_STRICT, 560 extack); 561 if (err < 0) 562 return err; 563 564 nla_for_each_nested(attr, quanta_attr, rem) { 565 switch (nla_type(attr)) { 566 case TCA_ETS_QUANTA_BAND: 567 if (band >= nbands) { 568 NL_SET_ERR_MSG_MOD(extack, "ETS quanta has more values than bands"); 569 return -EINVAL; 570 } 571 err = ets_quantum_parse(sch, attr, &quanta[band++], 572 extack); 573 if (err) 574 return err; 575 break; 576 default: 577 WARN_ON_ONCE(1); /* Validate should have caught this. */ 578 return -EINVAL; 579 } 580 } 581 582 return 0; 583 } 584 585 static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, 586 struct netlink_ext_ack *extack) 587 { 588 unsigned int quanta[TCQ_ETS_MAX_BANDS] = {0}; 589 struct Qdisc *queues[TCQ_ETS_MAX_BANDS]; 590 struct ets_sched *q = qdisc_priv(sch); 591 struct nlattr *tb[TCA_ETS_MAX + 1]; 592 unsigned int oldbands = q->nbands; 593 u8 priomap[TC_PRIO_MAX + 1]; 594 unsigned int nstrict = 0; 595 unsigned int nbands; 596 unsigned int i; 597 int err; 598 599 err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_policy, extack); 600 if (err < 0) 601 return err; 602 603 if (!tb[TCA_ETS_NBANDS]) { 604 NL_SET_ERR_MSG_MOD(extack, "Number of bands is a required argument"); 605 return -EINVAL; 606 } 607 nbands = nla_get_u8(tb[TCA_ETS_NBANDS]); 608 if (nbands < 1 || nbands > TCQ_ETS_MAX_BANDS) { 609 NL_SET_ERR_MSG_MOD(extack, "Invalid number of bands"); 610 return -EINVAL; 611 } 612 /* Unless overridden, traffic goes to the last band. */ 613 memset(priomap, nbands - 1, sizeof(priomap)); 614 615 if (tb[TCA_ETS_NSTRICT]) { 616 nstrict = nla_get_u8(tb[TCA_ETS_NSTRICT]); 617 if (nstrict > nbands) { 618 NL_SET_ERR_MSG_MOD(extack, "Invalid number of strict bands"); 619 return -EINVAL; 620 } 621 } 622 623 if (tb[TCA_ETS_PRIOMAP]) { 624 err = ets_qdisc_priomap_parse(tb[TCA_ETS_PRIOMAP], 625 nbands, priomap, extack); 626 if (err) 627 return err; 628 } 629 630 if (tb[TCA_ETS_QUANTA]) { 631 err = ets_qdisc_quanta_parse(sch, tb[TCA_ETS_QUANTA], 632 nbands, nstrict, quanta, extack); 633 if (err) 634 return err; 635 } 636 /* If there are more bands than strict + quanta provided, the remaining 637 * ones are ETS with quantum of MTU. Initialize the missing values here. 638 */ 639 for (i = nstrict; i < nbands; i++) { 640 if (!quanta[i]) 641 quanta[i] = psched_mtu(qdisc_dev(sch)); 642 } 643 644 /* Before commit, make sure we can allocate all new qdiscs */ 645 for (i = oldbands; i < nbands; i++) { 646 queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, 647 ets_class_id(sch, &q->classes[i]), 648 extack); 649 if (!queues[i]) { 650 while (i > oldbands) 651 qdisc_put(queues[--i]); 652 return -ENOMEM; 653 } 654 } 655 656 sch_tree_lock(sch); 657 658 for (i = nbands; i < oldbands; i++) { 659 if (cl_is_active(&q->classes[i])) 660 list_del_init(&q->classes[i].alist); 661 qdisc_purge_queue(q->classes[i].qdisc); 662 } 663 664 WRITE_ONCE(q->nbands, nbands); 665 for (i = nstrict; i < q->nstrict; i++) { 666 if (q->classes[i].qdisc->q.qlen) { 667 list_add_tail(&q->classes[i].alist, &q->active); 668 q->classes[i].deficit = quanta[i]; 669 } 670 } 671 for (i = q->nstrict; i < nstrict; i++) { 672 if (cl_is_active(&q->classes[i])) 673 list_del_init(&q->classes[i].alist); 674 } 675 WRITE_ONCE(q->nstrict, nstrict); 676 memcpy(q->prio2band, priomap, sizeof(priomap)); 677 678 for (i = 0; i < q->nbands; i++) 679 WRITE_ONCE(q->classes[i].quantum, quanta[i]); 680 681 for (i = oldbands; i < q->nbands; i++) { 682 q->classes[i].qdisc = queues[i]; 683 if (q->classes[i].qdisc != &noop_qdisc) 684 qdisc_hash_add(q->classes[i].qdisc, true); 685 } 686 687 sch_tree_unlock(sch); 688 689 ets_offload_change(sch); 690 for (i = q->nbands; i < oldbands; i++) { 691 qdisc_put(q->classes[i].qdisc); 692 q->classes[i].qdisc = NULL; 693 WRITE_ONCE(q->classes[i].quantum, 0); 694 q->classes[i].deficit = 0; 695 gnet_stats_basic_sync_init(&q->classes[i].bstats); 696 memset(&q->classes[i].qstats, 0, sizeof(q->classes[i].qstats)); 697 } 698 return 0; 699 } 700 701 static int ets_qdisc_init(struct Qdisc *sch, struct nlattr *opt, 702 struct netlink_ext_ack *extack) 703 { 704 struct ets_sched *q = qdisc_priv(sch); 705 int err, i; 706 707 if (!opt) 708 return -EINVAL; 709 710 err = tcf_block_get(&q->block, &q->filter_list, sch, extack); 711 if (err) 712 return err; 713 714 INIT_LIST_HEAD(&q->active); 715 for (i = 0; i < TCQ_ETS_MAX_BANDS; i++) 716 INIT_LIST_HEAD(&q->classes[i].alist); 717 718 return ets_qdisc_change(sch, opt, extack); 719 } 720 721 static void ets_qdisc_reset(struct Qdisc *sch) 722 { 723 struct ets_sched *q = qdisc_priv(sch); 724 int band; 725 726 for (band = q->nstrict; band < q->nbands; band++) { 727 if (q->classes[band].qdisc->q.qlen) 728 list_del_init(&q->classes[band].alist); 729 } 730 for (band = 0; band < q->nbands; band++) 731 qdisc_reset(q->classes[band].qdisc); 732 } 733 734 static void ets_qdisc_destroy(struct Qdisc *sch) 735 { 736 struct ets_sched *q = qdisc_priv(sch); 737 int band; 738 739 ets_offload_destroy(sch); 740 tcf_block_put(q->block); 741 for (band = 0; band < q->nbands; band++) 742 qdisc_put(q->classes[band].qdisc); 743 } 744 745 static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb) 746 { 747 struct ets_sched *q = qdisc_priv(sch); 748 struct nlattr *opts; 749 struct nlattr *nest; 750 u8 nbands, nstrict; 751 int band; 752 int prio; 753 int err; 754 755 err = ets_offload_dump(sch); 756 if (err) 757 return err; 758 759 opts = nla_nest_start_noflag(skb, TCA_OPTIONS); 760 if (!opts) 761 goto nla_err; 762 763 nbands = READ_ONCE(q->nbands); 764 if (nla_put_u8(skb, TCA_ETS_NBANDS, nbands)) 765 goto nla_err; 766 767 nstrict = READ_ONCE(q->nstrict); 768 if (nstrict && nla_put_u8(skb, TCA_ETS_NSTRICT, nstrict)) 769 goto nla_err; 770 771 if (nbands > nstrict) { 772 nest = nla_nest_start(skb, TCA_ETS_QUANTA); 773 if (!nest) 774 goto nla_err; 775 776 for (band = nstrict; band < nbands; band++) { 777 if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, 778 READ_ONCE(q->classes[band].quantum))) 779 goto nla_err; 780 } 781 782 nla_nest_end(skb, nest); 783 } 784 785 nest = nla_nest_start(skb, TCA_ETS_PRIOMAP); 786 if (!nest) 787 goto nla_err; 788 789 for (prio = 0; prio <= TC_PRIO_MAX; prio++) { 790 if (nla_put_u8(skb, TCA_ETS_PRIOMAP_BAND, 791 READ_ONCE(q->prio2band[prio]))) 792 goto nla_err; 793 } 794 795 nla_nest_end(skb, nest); 796 797 return nla_nest_end(skb, opts); 798 799 nla_err: 800 nla_nest_cancel(skb, opts); 801 return -EMSGSIZE; 802 } 803 804 static const struct Qdisc_class_ops ets_class_ops = { 805 .change = ets_class_change, 806 .graft = ets_class_graft, 807 .leaf = ets_class_leaf, 808 .find = ets_class_find, 809 .qlen_notify = ets_class_qlen_notify, 810 .dump = ets_class_dump, 811 .dump_stats = ets_class_dump_stats, 812 .walk = ets_qdisc_walk, 813 .tcf_block = ets_qdisc_tcf_block, 814 .bind_tcf = ets_qdisc_bind_tcf, 815 .unbind_tcf = ets_qdisc_unbind_tcf, 816 }; 817 818 static struct Qdisc_ops ets_qdisc_ops __read_mostly = { 819 .cl_ops = &ets_class_ops, 820 .id = "ets", 821 .priv_size = sizeof(struct ets_sched), 822 .enqueue = ets_qdisc_enqueue, 823 .dequeue = ets_qdisc_dequeue, 824 .peek = qdisc_peek_dequeued, 825 .change = ets_qdisc_change, 826 .init = ets_qdisc_init, 827 .reset = ets_qdisc_reset, 828 .destroy = ets_qdisc_destroy, 829 .dump = ets_qdisc_dump, 830 .owner = THIS_MODULE, 831 }; 832 MODULE_ALIAS_NET_SCH("ets"); 833 834 static int __init ets_init(void) 835 { 836 return register_qdisc(&ets_qdisc_ops); 837 } 838 839 static void __exit ets_exit(void) 840 { 841 unregister_qdisc(&ets_qdisc_ops); 842 } 843 844 module_init(ets_init); 845 module_exit(ets_exit); 846 MODULE_LICENSE("GPL"); 847 MODULE_DESCRIPTION("Enhanced Transmission Selection(ETS) scheduler"); 848