1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * net/sched/sch_ets.c Enhanced Transmission Selection scheduler 4 * 5 * Description 6 * ----------- 7 * 8 * The Enhanced Transmission Selection scheduler is a classful queuing 9 * discipline that merges functionality of PRIO and DRR qdiscs in one scheduler. 10 * ETS makes it easy to configure a set of strict and bandwidth-sharing bands to 11 * implement the transmission selection described in 802.1Qaz. 12 * 13 * Although ETS is technically classful, it's not possible to add and remove 14 * classes at will. Instead one specifies number of classes, how many are 15 * PRIO-like and how many DRR-like, and quanta for the latter. 16 * 17 * Algorithm 18 * --------- 19 * 20 * The strict classes, if any, are tried for traffic first: first band 0, if it 21 * has no traffic then band 1, etc. 22 * 23 * When there is no traffic in any of the strict queues, the bandwidth-sharing 24 * ones are tried next. Each band is assigned a deficit counter, initialized to 25 * "quantum" of that band. ETS maintains a list of active bandwidth-sharing 26 * bands whose qdiscs are non-empty. A packet is dequeued from the band at the 27 * head of the list if the packet size is smaller or equal to the deficit 28 * counter. If the counter is too small, it is increased by "quantum" and the 29 * scheduler moves on to the next band in the active list. 30 */ 31 32 #include <linux/module.h> 33 #include <net/gen_stats.h> 34 #include <net/netlink.h> 35 #include <net/pkt_cls.h> 36 #include <net/pkt_sched.h> 37 #include <net/sch_generic.h> 38 39 struct ets_class { 40 struct list_head alist; /* In struct ets_sched.active. */ 41 struct Qdisc *qdisc; 42 u32 quantum; 43 u32 deficit; 44 struct gnet_stats_basic_sync bstats; 45 struct gnet_stats_queue qstats; 46 }; 47 48 struct ets_sched { 49 struct list_head active; 50 struct tcf_proto __rcu *filter_list; 51 struct tcf_block *block; 52 unsigned int nbands; 53 unsigned int nstrict; 54 u8 prio2band[TC_PRIO_MAX + 1]; 55 struct ets_class classes[TCQ_ETS_MAX_BANDS]; 56 }; 57 58 static const struct nla_policy ets_policy[TCA_ETS_MAX + 1] = { 59 [TCA_ETS_NBANDS] = { .type = NLA_U8 }, 60 [TCA_ETS_NSTRICT] = { .type = NLA_U8 }, 61 [TCA_ETS_QUANTA] = { .type = NLA_NESTED }, 62 [TCA_ETS_PRIOMAP] = { .type = NLA_NESTED }, 63 }; 64 65 static const struct nla_policy ets_priomap_policy[TCA_ETS_MAX + 1] = { 66 [TCA_ETS_PRIOMAP_BAND] = { .type = NLA_U8 }, 67 }; 68 69 static const struct nla_policy ets_quanta_policy[TCA_ETS_MAX + 1] = { 70 [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 }, 71 }; 72 73 static const struct nla_policy ets_class_policy[TCA_ETS_MAX + 1] = { 74 [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 }, 75 }; 76 77 static bool cl_is_active(struct ets_class *cl) 78 { 79 return !list_empty(&cl->alist); 80 } 81 82 static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr, 83 unsigned int *quantum, 84 struct netlink_ext_ack *extack) 85 { 86 *quantum = clamp_t(u32, nla_get_u32(attr), 256, 1 << 20); 87 return 0; 88 } 89 90 static struct ets_class * 91 ets_class_from_arg(struct Qdisc *sch, unsigned long arg) 92 { 93 struct ets_sched *q = qdisc_priv(sch); 94 95 if (arg == 0 || arg > q->nbands) 96 return NULL; 97 return &q->classes[arg - 1]; 98 } 99 100 static u32 ets_class_id(struct Qdisc *sch, const struct ets_class *cl) 101 { 102 struct ets_sched *q = qdisc_priv(sch); 103 int band = cl - q->classes; 104 105 return TC_H_MAKE(sch->handle, band + 1); 106 } 107 108 static void ets_offload_change(struct Qdisc *sch) 109 { 110 struct net_device *dev = qdisc_dev(sch); 111 struct ets_sched *q = qdisc_priv(sch); 112 struct tc_ets_qopt_offload qopt; 113 unsigned int w_psum_prev = 0; 114 unsigned int quantum; 115 unsigned int w_psum; 116 unsigned int weight; 117 unsigned int i; 118 u64 q_psum = 0; 119 u64 q_sum = 0; 120 121 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 122 return; 123 124 qopt.command = TC_ETS_REPLACE; 125 qopt.handle = sch->handle; 126 qopt.parent = sch->parent; 127 qopt.replace_params.bands = q->nbands; 128 qopt.replace_params.qstats = &sch->qstats; 129 memcpy(&qopt.replace_params.priomap, 130 q->prio2band, sizeof(q->prio2band)); 131 132 for (i = 0; i < q->nbands; i++) 133 q_sum += q->classes[i].quantum; 134 135 for (i = 0; i < q->nbands; i++) { 136 quantum = q->classes[i].quantum; 137 if (quantum) { 138 q_psum += quantum; 139 w_psum = div64_u64(q_psum * 100, q_sum); 140 } else { 141 w_psum = 0; 142 } 143 weight = w_psum - w_psum_prev; 144 w_psum_prev = w_psum; 145 146 qopt.replace_params.quanta[i] = quantum; 147 qopt.replace_params.weights[i] = weight; 148 } 149 150 dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt); 151 } 152 153 static void ets_offload_destroy(struct Qdisc *sch) 154 { 155 struct net_device *dev = qdisc_dev(sch); 156 struct tc_ets_qopt_offload qopt; 157 158 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 159 return; 160 161 qopt.command = TC_ETS_DESTROY; 162 qopt.handle = sch->handle; 163 qopt.parent = sch->parent; 164 dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt); 165 } 166 167 static void ets_offload_graft(struct Qdisc *sch, struct Qdisc *new, 168 struct Qdisc *old, unsigned long arg, 169 struct netlink_ext_ack *extack) 170 { 171 struct net_device *dev = qdisc_dev(sch); 172 struct tc_ets_qopt_offload qopt; 173 174 qopt.command = TC_ETS_GRAFT; 175 qopt.handle = sch->handle; 176 qopt.parent = sch->parent; 177 qopt.graft_params.band = arg - 1; 178 qopt.graft_params.child_handle = new->handle; 179 180 qdisc_offload_graft_helper(dev, sch, new, old, TC_SETUP_QDISC_ETS, 181 &qopt, extack); 182 } 183 184 static int ets_offload_dump(struct Qdisc *sch) 185 { 186 struct tc_ets_qopt_offload qopt; 187 188 qopt.command = TC_ETS_STATS; 189 qopt.handle = sch->handle; 190 qopt.parent = sch->parent; 191 qopt.stats.bstats = &sch->bstats; 192 qopt.stats.qstats = &sch->qstats; 193 194 return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_ETS, &qopt); 195 } 196 197 static bool ets_class_is_strict(struct ets_sched *q, const struct ets_class *cl) 198 { 199 unsigned int band = cl - q->classes; 200 201 return band < q->nstrict; 202 } 203 204 static int ets_class_change(struct Qdisc *sch, u32 classid, u32 parentid, 205 struct nlattr **tca, unsigned long *arg, 206 struct netlink_ext_ack *extack) 207 { 208 struct ets_class *cl = ets_class_from_arg(sch, *arg); 209 struct ets_sched *q = qdisc_priv(sch); 210 struct nlattr *opt = tca[TCA_OPTIONS]; 211 struct nlattr *tb[TCA_ETS_MAX + 1]; 212 unsigned int quantum; 213 int err; 214 215 /* Classes can be added and removed only through Qdisc_ops.change 216 * interface. 217 */ 218 if (!cl) { 219 NL_SET_ERR_MSG(extack, "Fine-grained class addition and removal is not supported"); 220 return -EOPNOTSUPP; 221 } 222 223 if (!opt) { 224 NL_SET_ERR_MSG(extack, "ETS options are required for this operation"); 225 return -EINVAL; 226 } 227 228 err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_class_policy, extack); 229 if (err < 0) 230 return err; 231 232 if (!tb[TCA_ETS_QUANTA_BAND]) 233 /* Nothing to configure. */ 234 return 0; 235 236 if (ets_class_is_strict(q, cl)) { 237 NL_SET_ERR_MSG(extack, "Strict bands do not have a configurable quantum"); 238 return -EINVAL; 239 } 240 241 err = ets_quantum_parse(sch, tb[TCA_ETS_QUANTA_BAND], &quantum, 242 extack); 243 if (err) 244 return err; 245 246 WRITE_ONCE(cl->quantum, quantum); 247 248 ets_offload_change(sch); 249 return 0; 250 } 251 252 static int ets_class_graft(struct Qdisc *sch, unsigned long arg, 253 struct Qdisc *new, struct Qdisc **old, 254 struct netlink_ext_ack *extack) 255 { 256 struct ets_class *cl = ets_class_from_arg(sch, arg); 257 258 if (!new) { 259 new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, 260 ets_class_id(sch, cl), NULL); 261 if (!new) 262 new = &noop_qdisc; 263 else 264 qdisc_hash_add(new, true); 265 } 266 267 *old = qdisc_replace(sch, new, &cl->qdisc); 268 ets_offload_graft(sch, new, *old, arg, extack); 269 return 0; 270 } 271 272 static struct Qdisc *ets_class_leaf(struct Qdisc *sch, unsigned long arg) 273 { 274 struct ets_class *cl = ets_class_from_arg(sch, arg); 275 276 return cl->qdisc; 277 } 278 279 static unsigned long ets_class_find(struct Qdisc *sch, u32 classid) 280 { 281 unsigned long band = TC_H_MIN(classid); 282 struct ets_sched *q = qdisc_priv(sch); 283 284 if (band - 1 >= q->nbands) 285 return 0; 286 return band; 287 } 288 289 static void ets_class_qlen_notify(struct Qdisc *sch, unsigned long arg) 290 { 291 struct ets_class *cl = ets_class_from_arg(sch, arg); 292 struct ets_sched *q = qdisc_priv(sch); 293 294 /* We get notified about zero-length child Qdiscs as well if they are 295 * offloaded. Those aren't on the active list though, so don't attempt 296 * to remove them. 297 */ 298 if (!ets_class_is_strict(q, cl) && sch->q.qlen) 299 list_del_init(&cl->alist); 300 } 301 302 static int ets_class_dump(struct Qdisc *sch, unsigned long arg, 303 struct sk_buff *skb, struct tcmsg *tcm) 304 { 305 struct ets_class *cl = ets_class_from_arg(sch, arg); 306 struct ets_sched *q = qdisc_priv(sch); 307 struct nlattr *nest; 308 309 tcm->tcm_parent = TC_H_ROOT; 310 tcm->tcm_handle = ets_class_id(sch, cl); 311 tcm->tcm_info = cl->qdisc->handle; 312 313 nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 314 if (!nest) 315 goto nla_put_failure; 316 if (!ets_class_is_strict(q, cl)) { 317 if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, READ_ONCE(cl->quantum))) 318 goto nla_put_failure; 319 } 320 return nla_nest_end(skb, nest); 321 322 nla_put_failure: 323 nla_nest_cancel(skb, nest); 324 return -EMSGSIZE; 325 } 326 327 static int ets_class_dump_stats(struct Qdisc *sch, unsigned long arg, 328 struct gnet_dump *d) 329 { 330 struct ets_class *cl = ets_class_from_arg(sch, arg); 331 struct Qdisc *cl_q = cl->qdisc; 332 333 if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats, true) < 0 || 334 qdisc_qstats_copy(d, cl_q) < 0) 335 return -1; 336 337 return 0; 338 } 339 340 static void ets_qdisc_walk(struct Qdisc *sch, struct qdisc_walker *arg) 341 { 342 struct ets_sched *q = qdisc_priv(sch); 343 int i; 344 345 if (arg->stop) 346 return; 347 348 for (i = 0; i < q->nbands; i++) { 349 if (!tc_qdisc_stats_dump(sch, i + 1, arg)) 350 break; 351 } 352 } 353 354 static struct tcf_block * 355 ets_qdisc_tcf_block(struct Qdisc *sch, unsigned long cl, 356 struct netlink_ext_ack *extack) 357 { 358 struct ets_sched *q = qdisc_priv(sch); 359 360 if (cl) { 361 NL_SET_ERR_MSG(extack, "ETS classid must be zero"); 362 return NULL; 363 } 364 365 return q->block; 366 } 367 368 static unsigned long ets_qdisc_bind_tcf(struct Qdisc *sch, unsigned long parent, 369 u32 classid) 370 { 371 return ets_class_find(sch, classid); 372 } 373 374 static void ets_qdisc_unbind_tcf(struct Qdisc *sch, unsigned long arg) 375 { 376 } 377 378 static struct ets_class *ets_classify(struct sk_buff *skb, struct Qdisc *sch, 379 int *qerr) 380 { 381 struct ets_sched *q = qdisc_priv(sch); 382 u32 band = skb->priority; 383 struct tcf_result res; 384 struct tcf_proto *fl; 385 int err; 386 387 *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; 388 if (TC_H_MAJ(skb->priority) != sch->handle) { 389 fl = rcu_dereference_bh(q->filter_list); 390 err = tcf_classify_qdisc(skb, fl, &res, false); 391 #ifdef CONFIG_NET_CLS_ACT 392 switch (err) { 393 case TC_ACT_STOLEN: 394 case TC_ACT_QUEUED: 395 case TC_ACT_TRAP: 396 *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN; 397 fallthrough; 398 case TC_ACT_SHOT: 399 return NULL; 400 } 401 #endif 402 if (!fl || err < 0) { 403 if (TC_H_MAJ(band)) 404 band = 0; 405 return &q->classes[q->prio2band[band & TC_PRIO_MAX]]; 406 } 407 band = res.classid; 408 } 409 band = TC_H_MIN(band) - 1; 410 if (band >= q->nbands) 411 return &q->classes[q->prio2band[0]]; 412 return &q->classes[band]; 413 } 414 415 static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 416 struct sk_buff **to_free) 417 { 418 unsigned int len = qdisc_pkt_len(skb); 419 struct ets_sched *q = qdisc_priv(sch); 420 struct ets_class *cl; 421 int err = 0; 422 423 cl = ets_classify(skb, sch, &err); 424 if (!cl) { 425 if (err & __NET_XMIT_BYPASS) 426 qdisc_qstats_drop(sch); 427 __qdisc_drop(skb, to_free); 428 return err; 429 } 430 431 err = qdisc_enqueue(skb, cl->qdisc, to_free); 432 if (unlikely(err != NET_XMIT_SUCCESS)) { 433 if (net_xmit_drop_count(err)) { 434 cl->qstats.drops++; 435 qdisc_qstats_drop(sch); 436 } 437 return err; 438 } 439 440 if (!cl_is_active(cl) && !ets_class_is_strict(q, cl)) { 441 list_add_tail(&cl->alist, &q->active); 442 cl->deficit = READ_ONCE(cl->quantum); 443 } 444 445 qstats_backlog_add(sch, len); 446 qdisc_qlen_inc(sch); 447 return err; 448 } 449 450 static struct sk_buff * 451 ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb) 452 { 453 qdisc_bstats_update(sch, skb); 454 qdisc_qstats_backlog_dec(sch, skb); 455 qdisc_qlen_dec(sch); 456 return skb; 457 } 458 459 static struct sk_buff *ets_qdisc_dequeue(struct Qdisc *sch) 460 { 461 struct ets_sched *q = qdisc_priv(sch); 462 struct ets_class *cl; 463 struct sk_buff *skb; 464 unsigned int band; 465 unsigned int len; 466 467 while (1) { 468 for (band = 0; band < q->nstrict; band++) { 469 cl = &q->classes[band]; 470 skb = qdisc_dequeue_peeked(cl->qdisc); 471 if (skb) 472 return ets_qdisc_dequeue_skb(sch, skb); 473 } 474 475 if (list_empty(&q->active)) 476 goto out; 477 478 cl = list_first_entry(&q->active, struct ets_class, alist); 479 skb = cl->qdisc->ops->peek(cl->qdisc); 480 if (!skb) { 481 qdisc_warn_nonwc(__func__, cl->qdisc); 482 goto out; 483 } 484 485 len = qdisc_pkt_len(skb); 486 if (len <= cl->deficit) { 487 cl->deficit -= len; 488 skb = qdisc_dequeue_peeked(cl->qdisc); 489 if (unlikely(!skb)) 490 goto out; 491 if (cl->qdisc->q.qlen == 0) 492 list_del_init(&cl->alist); 493 return ets_qdisc_dequeue_skb(sch, skb); 494 } 495 496 cl->deficit += READ_ONCE(cl->quantum); 497 list_move_tail(&cl->alist, &q->active); 498 } 499 out: 500 return NULL; 501 } 502 503 static int ets_qdisc_priomap_parse(struct nlattr *priomap_attr, 504 unsigned int nbands, u8 *priomap, 505 struct netlink_ext_ack *extack) 506 { 507 const struct nlattr *attr; 508 int prio = 0; 509 u8 band; 510 int rem; 511 int err; 512 513 err = __nla_validate_nested(priomap_attr, TCA_ETS_MAX, 514 ets_priomap_policy, NL_VALIDATE_STRICT, 515 extack); 516 if (err) 517 return err; 518 519 nla_for_each_nested(attr, priomap_attr, rem) { 520 switch (nla_type(attr)) { 521 case TCA_ETS_PRIOMAP_BAND: 522 if (prio > TC_PRIO_MAX) { 523 NL_SET_ERR_MSG_MOD(extack, "Too many priorities in ETS priomap"); 524 return -EINVAL; 525 } 526 band = nla_get_u8(attr); 527 if (band >= nbands) { 528 NL_SET_ERR_MSG_MOD(extack, "Invalid band number in ETS priomap"); 529 return -EINVAL; 530 } 531 priomap[prio++] = band; 532 break; 533 default: 534 WARN_ON_ONCE(1); /* Validate should have caught this. */ 535 return -EINVAL; 536 } 537 } 538 539 return 0; 540 } 541 542 static int ets_qdisc_quanta_parse(struct Qdisc *sch, struct nlattr *quanta_attr, 543 unsigned int nbands, unsigned int nstrict, 544 unsigned int *quanta, 545 struct netlink_ext_ack *extack) 546 { 547 const struct nlattr *attr; 548 int band = nstrict; 549 int rem; 550 int err; 551 552 err = __nla_validate_nested(quanta_attr, TCA_ETS_MAX, 553 ets_quanta_policy, NL_VALIDATE_STRICT, 554 extack); 555 if (err < 0) 556 return err; 557 558 nla_for_each_nested(attr, quanta_attr, rem) { 559 switch (nla_type(attr)) { 560 case TCA_ETS_QUANTA_BAND: 561 if (band >= nbands) { 562 NL_SET_ERR_MSG_MOD(extack, "ETS quanta has more values than bands"); 563 return -EINVAL; 564 } 565 err = ets_quantum_parse(sch, attr, &quanta[band++], 566 extack); 567 if (err) 568 return err; 569 break; 570 default: 571 WARN_ON_ONCE(1); /* Validate should have caught this. */ 572 return -EINVAL; 573 } 574 } 575 576 return 0; 577 } 578 579 static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, 580 struct netlink_ext_ack *extack) 581 { 582 unsigned int quanta[TCQ_ETS_MAX_BANDS] = {0}; 583 struct Qdisc *queues[TCQ_ETS_MAX_BANDS]; 584 struct ets_sched *q = qdisc_priv(sch); 585 struct nlattr *tb[TCA_ETS_MAX + 1]; 586 unsigned int oldbands = q->nbands; 587 u8 priomap[TC_PRIO_MAX + 1]; 588 unsigned int nstrict = 0; 589 unsigned int nbands; 590 unsigned int i; 591 int err; 592 593 err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_policy, extack); 594 if (err < 0) 595 return err; 596 597 if (!tb[TCA_ETS_NBANDS]) { 598 NL_SET_ERR_MSG_MOD(extack, "Number of bands is a required argument"); 599 return -EINVAL; 600 } 601 nbands = nla_get_u8(tb[TCA_ETS_NBANDS]); 602 if (nbands < 1 || nbands > TCQ_ETS_MAX_BANDS) { 603 NL_SET_ERR_MSG_MOD(extack, "Invalid number of bands"); 604 return -EINVAL; 605 } 606 /* Unless overridden, traffic goes to the last band. */ 607 memset(priomap, nbands - 1, sizeof(priomap)); 608 609 if (tb[TCA_ETS_NSTRICT]) { 610 nstrict = nla_get_u8(tb[TCA_ETS_NSTRICT]); 611 if (nstrict > nbands) { 612 NL_SET_ERR_MSG_MOD(extack, "Invalid number of strict bands"); 613 return -EINVAL; 614 } 615 } 616 617 if (tb[TCA_ETS_PRIOMAP]) { 618 err = ets_qdisc_priomap_parse(tb[TCA_ETS_PRIOMAP], 619 nbands, priomap, extack); 620 if (err) 621 return err; 622 } 623 624 if (tb[TCA_ETS_QUANTA]) { 625 err = ets_qdisc_quanta_parse(sch, tb[TCA_ETS_QUANTA], 626 nbands, nstrict, quanta, extack); 627 if (err) 628 return err; 629 } 630 /* If there are more bands than strict + quanta provided, the remaining 631 * ones are ETS with quantum of max(MTU, 256). Initialize the missing 632 * values here. 633 */ 634 for (i = nstrict; i < nbands; i++) { 635 if (!quanta[i]) 636 quanta[i] = clamp_t(u32, (u32)psched_mtu(qdisc_dev(sch)), 637 256, 1 << 20); 638 } 639 640 /* Before commit, make sure we can allocate all new qdiscs */ 641 for (i = oldbands; i < nbands; i++) { 642 queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, 643 ets_class_id(sch, &q->classes[i]), 644 extack); 645 if (!queues[i]) { 646 while (i > oldbands) 647 qdisc_put(queues[--i]); 648 return -ENOMEM; 649 } 650 } 651 652 sch_tree_lock(sch); 653 654 for (i = nbands; i < oldbands; i++) { 655 if (cl_is_active(&q->classes[i])) 656 list_del_init(&q->classes[i].alist); 657 qdisc_purge_queue(q->classes[i].qdisc); 658 } 659 660 WRITE_ONCE(q->nbands, nbands); 661 for (i = nstrict; i < q->nstrict; i++) { 662 if (q->classes[i].qdisc->q.qlen) { 663 list_add_tail(&q->classes[i].alist, &q->active); 664 q->classes[i].deficit = quanta[i]; 665 } 666 } 667 for (i = q->nstrict; i < nstrict; i++) { 668 if (cl_is_active(&q->classes[i])) 669 list_del_init(&q->classes[i].alist); 670 } 671 WRITE_ONCE(q->nstrict, nstrict); 672 memcpy(q->prio2band, priomap, sizeof(priomap)); 673 674 for (i = 0; i < q->nbands; i++) 675 WRITE_ONCE(q->classes[i].quantum, quanta[i]); 676 677 for (i = oldbands; i < q->nbands; i++) { 678 q->classes[i].qdisc = queues[i]; 679 if (q->classes[i].qdisc != &noop_qdisc) 680 qdisc_hash_add(q->classes[i].qdisc, true); 681 } 682 683 sch_tree_unlock(sch); 684 685 ets_offload_change(sch); 686 for (i = q->nbands; i < oldbands; i++) { 687 qdisc_put(q->classes[i].qdisc); 688 q->classes[i].qdisc = NULL; 689 WRITE_ONCE(q->classes[i].quantum, 0); 690 q->classes[i].deficit = 0; 691 gnet_stats_basic_sync_init(&q->classes[i].bstats); 692 memset(&q->classes[i].qstats, 0, sizeof(q->classes[i].qstats)); 693 } 694 return 0; 695 } 696 697 static int ets_qdisc_init(struct Qdisc *sch, struct nlattr *opt, 698 struct netlink_ext_ack *extack) 699 { 700 struct ets_sched *q = qdisc_priv(sch); 701 int err, i; 702 703 if (!opt) 704 return -EINVAL; 705 706 err = tcf_block_get(&q->block, &q->filter_list, sch, extack); 707 if (err) 708 return err; 709 710 INIT_LIST_HEAD(&q->active); 711 for (i = 0; i < TCQ_ETS_MAX_BANDS; i++) 712 INIT_LIST_HEAD(&q->classes[i].alist); 713 714 return ets_qdisc_change(sch, opt, extack); 715 } 716 717 static void ets_qdisc_reset(struct Qdisc *sch) 718 { 719 struct ets_sched *q = qdisc_priv(sch); 720 int band; 721 722 for (band = q->nstrict; band < q->nbands; band++) { 723 if (q->classes[band].qdisc->q.qlen) 724 list_del_init(&q->classes[band].alist); 725 } 726 for (band = 0; band < q->nbands; band++) 727 qdisc_reset(q->classes[band].qdisc); 728 } 729 730 static void ets_qdisc_destroy(struct Qdisc *sch) 731 { 732 struct ets_sched *q = qdisc_priv(sch); 733 int band; 734 735 ets_offload_destroy(sch); 736 tcf_block_put(q->block); 737 for (band = 0; band < q->nbands; band++) 738 qdisc_put(q->classes[band].qdisc); 739 } 740 741 static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb) 742 { 743 struct ets_sched *q = qdisc_priv(sch); 744 struct nlattr *opts; 745 struct nlattr *nest; 746 u8 nbands, nstrict; 747 int band; 748 int prio; 749 int err; 750 751 err = ets_offload_dump(sch); 752 if (err) 753 return err; 754 755 opts = nla_nest_start_noflag(skb, TCA_OPTIONS); 756 if (!opts) 757 goto nla_err; 758 759 nbands = READ_ONCE(q->nbands); 760 if (nla_put_u8(skb, TCA_ETS_NBANDS, nbands)) 761 goto nla_err; 762 763 nstrict = READ_ONCE(q->nstrict); 764 if (nstrict && nla_put_u8(skb, TCA_ETS_NSTRICT, nstrict)) 765 goto nla_err; 766 767 if (nbands > nstrict) { 768 nest = nla_nest_start(skb, TCA_ETS_QUANTA); 769 if (!nest) 770 goto nla_err; 771 772 for (band = nstrict; band < nbands; band++) { 773 if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, 774 READ_ONCE(q->classes[band].quantum))) 775 goto nla_err; 776 } 777 778 nla_nest_end(skb, nest); 779 } 780 781 nest = nla_nest_start(skb, TCA_ETS_PRIOMAP); 782 if (!nest) 783 goto nla_err; 784 785 for (prio = 0; prio <= TC_PRIO_MAX; prio++) { 786 if (nla_put_u8(skb, TCA_ETS_PRIOMAP_BAND, 787 READ_ONCE(q->prio2band[prio]))) 788 goto nla_err; 789 } 790 791 nla_nest_end(skb, nest); 792 793 return nla_nest_end(skb, opts); 794 795 nla_err: 796 nla_nest_cancel(skb, opts); 797 return -EMSGSIZE; 798 } 799 800 static const struct Qdisc_class_ops ets_class_ops = { 801 .change = ets_class_change, 802 .graft = ets_class_graft, 803 .leaf = ets_class_leaf, 804 .find = ets_class_find, 805 .qlen_notify = ets_class_qlen_notify, 806 .dump = ets_class_dump, 807 .dump_stats = ets_class_dump_stats, 808 .walk = ets_qdisc_walk, 809 .tcf_block = ets_qdisc_tcf_block, 810 .bind_tcf = ets_qdisc_bind_tcf, 811 .unbind_tcf = ets_qdisc_unbind_tcf, 812 }; 813 814 static struct Qdisc_ops ets_qdisc_ops __read_mostly = { 815 .cl_ops = &ets_class_ops, 816 .id = "ets", 817 .priv_size = sizeof(struct ets_sched), 818 .enqueue = ets_qdisc_enqueue, 819 .dequeue = ets_qdisc_dequeue, 820 .peek = qdisc_peek_dequeued, 821 .change = ets_qdisc_change, 822 .init = ets_qdisc_init, 823 .reset = ets_qdisc_reset, 824 .destroy = ets_qdisc_destroy, 825 .dump = ets_qdisc_dump, 826 .owner = THIS_MODULE, 827 }; 828 MODULE_ALIAS_NET_SCH("ets"); 829 830 static int __init ets_init(void) 831 { 832 return register_qdisc(&ets_qdisc_ops); 833 } 834 835 static void __exit ets_exit(void) 836 { 837 unregister_qdisc(&ets_qdisc_ops); 838 } 839 840 module_init(ets_init); 841 module_exit(ets_exit); 842 MODULE_LICENSE("GPL"); 843 MODULE_DESCRIPTION("Enhanced Transmission Selection(ETS) scheduler"); 844