1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/sch_api.c Packet scheduler API. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 * Fixes: 8 * 9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired. 10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support 12 */ 13 14 #include <linux/module.h> 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/string.h> 18 #include <linux/errno.h> 19 #include <linux/skbuff.h> 20 #include <linux/init.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/kmod.h> 24 #include <linux/list.h> 25 #include <linux/hrtimer.h> 26 #include <linux/slab.h> 27 #include <linux/hashtable.h> 28 29 #include <net/net_namespace.h> 30 #include <net/sock.h> 31 #include <net/netlink.h> 32 #include <net/pkt_sched.h> 33 #include <net/pkt_cls.h> 34 #include <net/tc_wrapper.h> 35 36 #include <trace/events/qdisc.h> 37 38 /* 39 40 Short review. 41 ------------- 42 43 This file consists of two interrelated parts: 44 45 1. queueing disciplines manager frontend. 46 2. traffic classes manager frontend. 47 48 Generally, queueing discipline ("qdisc") is a black box, 49 which is able to enqueue packets and to dequeue them (when 50 device is ready to send something) in order and at times 51 determined by algorithm hidden in it. 52 53 qdisc's are divided to two categories: 54 - "queues", which have no internal structure visible from outside. 55 - "schedulers", which split all the packets to "traffic classes", 56 using "packet classifiers" (look at cls_api.c) 57 58 In turn, classes may have child qdiscs (as rule, queues) 59 attached to them etc. etc. etc. 60 61 The goal of the routines in this file is to translate 62 information supplied by user in the form of handles 63 to more intelligible for kernel form, to make some sanity 64 checks and part of work, which is common to all qdiscs 65 and to provide rtnetlink notifications. 66 67 All real intelligent work is done inside qdisc modules. 68 69 70 71 Every discipline has two major routines: enqueue and dequeue. 72 73 ---dequeue 74 75 dequeue usually returns a skb to send. It is allowed to return NULL, 76 but it does not mean that queue is empty, it just means that 77 discipline does not want to send anything this time. 78 Queue is really empty if q->q.qlen == 0. 79 For complicated disciplines with multiple queues q->q is not 80 real packet queue, but however q->q.qlen must be valid. 81 82 ---enqueue 83 84 enqueue returns 0, if packet was enqueued successfully. 85 If packet (this one or another one) was dropped, it returns 86 not zero error code. 87 NET_XMIT_DROP - this packet dropped 88 Expected action: do not backoff, but wait until queue will clear. 89 NET_XMIT_CN - probably this packet enqueued, but another one dropped. 90 Expected action: backoff or ignore 91 92 Auxiliary routines: 93 94 ---peek 95 96 like dequeue but without removing a packet from the queue 97 98 ---reset 99 100 returns qdisc to initial state: purge all buffers, clear all 101 timers, counters (except for statistics) etc. 102 103 ---init 104 105 initializes newly created qdisc. 106 107 ---destroy 108 109 destroys resources allocated by init and during lifetime of qdisc. 110 111 ---change 112 113 changes qdisc parameters. 114 */ 115 116 /* Protects list of registered TC modules. It is pure SMP lock. */ 117 static DEFINE_RWLOCK(qdisc_mod_lock); 118 119 120 /************************************************ 121 * Queueing disciplines manipulation. * 122 ************************************************/ 123 124 125 /* The list of all installed queueing disciplines. */ 126 127 static struct Qdisc_ops *qdisc_base; 128 129 /* Register/unregister queueing discipline */ 130 131 int register_qdisc(struct Qdisc_ops *qops) 132 { 133 struct Qdisc_ops *q, **qp; 134 int rc = -EEXIST; 135 136 write_lock(&qdisc_mod_lock); 137 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 138 if (!strcmp(qops->id, q->id)) 139 goto out; 140 141 if (qops->enqueue == NULL) 142 qops->enqueue = noop_qdisc_ops.enqueue; 143 if (qops->peek == NULL) { 144 if (qops->dequeue == NULL) 145 qops->peek = noop_qdisc_ops.peek; 146 else 147 goto out_einval; 148 } 149 if (qops->dequeue == NULL) 150 qops->dequeue = noop_qdisc_ops.dequeue; 151 152 if (qops->cl_ops) { 153 const struct Qdisc_class_ops *cops = qops->cl_ops; 154 155 if (!(cops->find && cops->walk && cops->leaf)) 156 goto out_einval; 157 158 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf)) 159 goto out_einval; 160 } 161 162 qops->next = NULL; 163 *qp = qops; 164 rc = 0; 165 out: 166 write_unlock(&qdisc_mod_lock); 167 return rc; 168 169 out_einval: 170 rc = -EINVAL; 171 goto out; 172 } 173 EXPORT_SYMBOL(register_qdisc); 174 175 void unregister_qdisc(struct Qdisc_ops *qops) 176 { 177 struct Qdisc_ops *q, **qp; 178 int err = -ENOENT; 179 180 write_lock(&qdisc_mod_lock); 181 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 182 if (q == qops) 183 break; 184 if (q) { 185 *qp = q->next; 186 q->next = NULL; 187 err = 0; 188 } 189 write_unlock(&qdisc_mod_lock); 190 191 WARN(err, "unregister qdisc(%s) failed\n", qops->id); 192 } 193 EXPORT_SYMBOL(unregister_qdisc); 194 195 /* Get default qdisc if not otherwise specified */ 196 void qdisc_get_default(char *name, size_t len) 197 { 198 read_lock(&qdisc_mod_lock); 199 strscpy(name, default_qdisc_ops->id, len); 200 read_unlock(&qdisc_mod_lock); 201 } 202 203 static struct Qdisc_ops *qdisc_lookup_default(const char *name) 204 { 205 struct Qdisc_ops *q = NULL; 206 207 for (q = qdisc_base; q; q = q->next) { 208 if (!strcmp(name, q->id)) { 209 if (!try_module_get(q->owner)) 210 q = NULL; 211 break; 212 } 213 } 214 215 return q; 216 } 217 218 /* Set new default qdisc to use */ 219 int qdisc_set_default(const char *name) 220 { 221 const struct Qdisc_ops *ops; 222 223 if (!capable(CAP_NET_ADMIN)) 224 return -EPERM; 225 226 write_lock(&qdisc_mod_lock); 227 ops = qdisc_lookup_default(name); 228 if (!ops) { 229 /* Not found, drop lock and try to load module */ 230 write_unlock(&qdisc_mod_lock); 231 request_module("sch_%s", name); 232 write_lock(&qdisc_mod_lock); 233 234 ops = qdisc_lookup_default(name); 235 } 236 237 if (ops) { 238 /* Set new default */ 239 module_put(default_qdisc_ops->owner); 240 default_qdisc_ops = ops; 241 } 242 write_unlock(&qdisc_mod_lock); 243 244 return ops ? 0 : -ENOENT; 245 } 246 247 #ifdef CONFIG_NET_SCH_DEFAULT 248 /* Set default value from kernel config */ 249 static int __init sch_default_qdisc(void) 250 { 251 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH); 252 } 253 late_initcall(sch_default_qdisc); 254 #endif 255 256 /* We know handle. Find qdisc among all qdisc's attached to device 257 * (root qdisc, all its children, children of children etc.) 258 * Note: caller either uses rtnl or rcu_read_lock() 259 */ 260 261 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle) 262 { 263 struct Qdisc *q; 264 265 if (!qdisc_dev(root)) 266 return (root->handle == handle ? root : NULL); 267 268 if (!(root->flags & TCQ_F_BUILTIN) && 269 root->handle == handle) 270 return root; 271 272 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle, 273 lockdep_rtnl_is_held()) { 274 if (q->handle == handle) 275 return q; 276 } 277 return NULL; 278 } 279 280 void qdisc_hash_add(struct Qdisc *q, bool invisible) 281 { 282 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { 283 ASSERT_RTNL(); 284 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle); 285 if (invisible) 286 q->flags |= TCQ_F_INVISIBLE; 287 } 288 } 289 EXPORT_SYMBOL(qdisc_hash_add); 290 291 void qdisc_hash_del(struct Qdisc *q) 292 { 293 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { 294 ASSERT_RTNL(); 295 hash_del_rcu(&q->hash); 296 } 297 } 298 EXPORT_SYMBOL(qdisc_hash_del); 299 300 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) 301 { 302 struct Qdisc *q; 303 304 if (!handle) 305 return NULL; 306 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle); 307 if (q) 308 goto out; 309 310 if (dev_ingress_queue(dev)) 311 q = qdisc_match_from_root( 312 rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping), 313 handle); 314 out: 315 return q; 316 } 317 318 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle) 319 { 320 struct netdev_queue *nq; 321 struct Qdisc *q; 322 323 if (!handle) 324 return NULL; 325 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle); 326 if (q) 327 goto out; 328 329 nq = dev_ingress_queue_rcu(dev); 330 if (nq) 331 q = qdisc_match_from_root(rcu_dereference(nq->qdisc_sleeping), 332 handle); 333 out: 334 return q; 335 } 336 337 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) 338 { 339 unsigned long cl; 340 const struct Qdisc_class_ops *cops = p->ops->cl_ops; 341 342 if (cops == NULL) 343 return NULL; 344 cl = cops->find(p, classid); 345 346 if (cl == 0) 347 return NULL; 348 return cops->leaf(p, cl); 349 } 350 351 /* Find queueing discipline by name */ 352 353 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind) 354 { 355 struct Qdisc_ops *q = NULL; 356 357 if (kind) { 358 read_lock(&qdisc_mod_lock); 359 for (q = qdisc_base; q; q = q->next) { 360 if (nla_strcmp(kind, q->id) == 0) { 361 if (!try_module_get(q->owner)) 362 q = NULL; 363 break; 364 } 365 } 366 read_unlock(&qdisc_mod_lock); 367 } 368 return q; 369 } 370 371 /* The linklayer setting were not transferred from iproute2, in older 372 * versions, and the rate tables lookup systems have been dropped in 373 * the kernel. To keep backward compatible with older iproute2 tc 374 * utils, we detect the linklayer setting by detecting if the rate 375 * table were modified. 376 * 377 * For linklayer ATM table entries, the rate table will be aligned to 378 * 48 bytes, thus some table entries will contain the same value. The 379 * mpu (min packet unit) is also encoded into the old rate table, thus 380 * starting from the mpu, we find low and high table entries for 381 * mapping this cell. If these entries contain the same value, when 382 * the rate tables have been modified for linklayer ATM. 383 * 384 * This is done by rounding mpu to the nearest 48 bytes cell/entry, 385 * and then roundup to the next cell, calc the table entry one below, 386 * and compare. 387 */ 388 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab) 389 { 390 int low = roundup(r->mpu, 48); 391 int high = roundup(low+1, 48); 392 int cell_low = low >> r->cell_log; 393 int cell_high = (high >> r->cell_log) - 1; 394 395 /* rtab is too inaccurate at rates > 100Mbit/s */ 396 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) { 397 pr_debug("TC linklayer: Giving up ATM detection\n"); 398 return TC_LINKLAYER_ETHERNET; 399 } 400 401 if ((cell_high > cell_low) && (cell_high < 256) 402 && (rtab[cell_low] == rtab[cell_high])) { 403 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n", 404 cell_low, cell_high, rtab[cell_high]); 405 return TC_LINKLAYER_ATM; 406 } 407 return TC_LINKLAYER_ETHERNET; 408 } 409 410 static struct qdisc_rate_table *qdisc_rtab_list; 411 412 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, 413 struct nlattr *tab, 414 struct netlink_ext_ack *extack) 415 { 416 struct qdisc_rate_table *rtab; 417 418 if (tab == NULL || r->rate == 0 || 419 r->cell_log == 0 || r->cell_log >= 32 || 420 nla_len(tab) != TC_RTAB_SIZE) { 421 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching"); 422 return NULL; 423 } 424 425 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) { 426 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) && 427 !memcmp(&rtab->data, nla_data(tab), 1024)) { 428 rtab->refcnt++; 429 return rtab; 430 } 431 } 432 433 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL); 434 if (rtab) { 435 rtab->rate = *r; 436 rtab->refcnt = 1; 437 memcpy(rtab->data, nla_data(tab), 1024); 438 if (r->linklayer == TC_LINKLAYER_UNAWARE) 439 r->linklayer = __detect_linklayer(r, rtab->data); 440 rtab->next = qdisc_rtab_list; 441 qdisc_rtab_list = rtab; 442 } else { 443 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table"); 444 } 445 return rtab; 446 } 447 EXPORT_SYMBOL(qdisc_get_rtab); 448 449 void qdisc_put_rtab(struct qdisc_rate_table *tab) 450 { 451 struct qdisc_rate_table *rtab, **rtabp; 452 453 if (!tab || --tab->refcnt) 454 return; 455 456 for (rtabp = &qdisc_rtab_list; 457 (rtab = *rtabp) != NULL; 458 rtabp = &rtab->next) { 459 if (rtab == tab) { 460 *rtabp = rtab->next; 461 kfree(rtab); 462 return; 463 } 464 } 465 } 466 EXPORT_SYMBOL(qdisc_put_rtab); 467 468 static LIST_HEAD(qdisc_stab_list); 469 470 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = { 471 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) }, 472 [TCA_STAB_DATA] = { .type = NLA_BINARY }, 473 }; 474 475 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt, 476 struct netlink_ext_ack *extack) 477 { 478 struct nlattr *tb[TCA_STAB_MAX + 1]; 479 struct qdisc_size_table *stab; 480 struct tc_sizespec *s; 481 unsigned int tsize = 0; 482 u16 *tab = NULL; 483 int err; 484 485 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy, 486 extack); 487 if (err < 0) 488 return ERR_PTR(err); 489 if (!tb[TCA_STAB_BASE]) { 490 NL_SET_ERR_MSG(extack, "Size table base attribute is missing"); 491 return ERR_PTR(-EINVAL); 492 } 493 494 s = nla_data(tb[TCA_STAB_BASE]); 495 496 if (s->tsize > 0) { 497 if (!tb[TCA_STAB_DATA]) { 498 NL_SET_ERR_MSG(extack, "Size table data attribute is missing"); 499 return ERR_PTR(-EINVAL); 500 } 501 tab = nla_data(tb[TCA_STAB_DATA]); 502 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16); 503 } 504 505 if (tsize != s->tsize || (!tab && tsize > 0)) { 506 NL_SET_ERR_MSG(extack, "Invalid size of size table"); 507 return ERR_PTR(-EINVAL); 508 } 509 510 list_for_each_entry(stab, &qdisc_stab_list, list) { 511 if (memcmp(&stab->szopts, s, sizeof(*s))) 512 continue; 513 if (tsize > 0 && 514 memcmp(stab->data, tab, flex_array_size(stab, data, tsize))) 515 continue; 516 stab->refcnt++; 517 return stab; 518 } 519 520 if (s->size_log > STAB_SIZE_LOG_MAX || 521 s->cell_log > STAB_SIZE_LOG_MAX) { 522 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table"); 523 return ERR_PTR(-EINVAL); 524 } 525 526 stab = kmalloc(struct_size(stab, data, tsize), GFP_KERNEL); 527 if (!stab) 528 return ERR_PTR(-ENOMEM); 529 530 stab->refcnt = 1; 531 stab->szopts = *s; 532 if (tsize > 0) 533 memcpy(stab->data, tab, flex_array_size(stab, data, tsize)); 534 535 list_add_tail(&stab->list, &qdisc_stab_list); 536 537 return stab; 538 } 539 540 void qdisc_put_stab(struct qdisc_size_table *tab) 541 { 542 if (!tab) 543 return; 544 545 if (--tab->refcnt == 0) { 546 list_del(&tab->list); 547 kfree_rcu(tab, rcu); 548 } 549 } 550 EXPORT_SYMBOL(qdisc_put_stab); 551 552 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab) 553 { 554 struct nlattr *nest; 555 556 nest = nla_nest_start_noflag(skb, TCA_STAB); 557 if (nest == NULL) 558 goto nla_put_failure; 559 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts)) 560 goto nla_put_failure; 561 nla_nest_end(skb, nest); 562 563 return skb->len; 564 565 nla_put_failure: 566 return -1; 567 } 568 569 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 570 const struct qdisc_size_table *stab) 571 { 572 int pkt_len, slot; 573 574 pkt_len = skb->len + stab->szopts.overhead; 575 if (unlikely(!stab->szopts.tsize)) 576 goto out; 577 578 slot = pkt_len + stab->szopts.cell_align; 579 if (unlikely(slot < 0)) 580 slot = 0; 581 582 slot >>= stab->szopts.cell_log; 583 if (likely(slot < stab->szopts.tsize)) 584 pkt_len = stab->data[slot]; 585 else 586 pkt_len = stab->data[stab->szopts.tsize - 1] * 587 (slot / stab->szopts.tsize) + 588 stab->data[slot % stab->szopts.tsize]; 589 590 pkt_len <<= stab->szopts.size_log; 591 out: 592 if (unlikely(pkt_len < 1)) 593 pkt_len = 1; 594 qdisc_skb_cb(skb)->pkt_len = pkt_len; 595 } 596 EXPORT_SYMBOL(__qdisc_calculate_pkt_len); 597 598 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc) 599 { 600 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { 601 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", 602 txt, qdisc->ops->id, qdisc->handle >> 16); 603 qdisc->flags |= TCQ_F_WARN_NONWC; 604 } 605 } 606 EXPORT_SYMBOL(qdisc_warn_nonwc); 607 608 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) 609 { 610 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, 611 timer); 612 613 rcu_read_lock(); 614 __netif_schedule(qdisc_root(wd->qdisc)); 615 rcu_read_unlock(); 616 617 return HRTIMER_NORESTART; 618 } 619 620 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc, 621 clockid_t clockid) 622 { 623 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED); 624 wd->timer.function = qdisc_watchdog; 625 wd->qdisc = qdisc; 626 } 627 EXPORT_SYMBOL(qdisc_watchdog_init_clockid); 628 629 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc) 630 { 631 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC); 632 } 633 EXPORT_SYMBOL(qdisc_watchdog_init); 634 635 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires, 636 u64 delta_ns) 637 { 638 bool deactivated; 639 640 rcu_read_lock(); 641 deactivated = test_bit(__QDISC_STATE_DEACTIVATED, 642 &qdisc_root_sleeping(wd->qdisc)->state); 643 rcu_read_unlock(); 644 if (deactivated) 645 return; 646 647 if (hrtimer_is_queued(&wd->timer)) { 648 u64 softexpires; 649 650 softexpires = ktime_to_ns(hrtimer_get_softexpires(&wd->timer)); 651 /* If timer is already set in [expires, expires + delta_ns], 652 * do not reprogram it. 653 */ 654 if (softexpires - expires <= delta_ns) 655 return; 656 } 657 658 hrtimer_start_range_ns(&wd->timer, 659 ns_to_ktime(expires), 660 delta_ns, 661 HRTIMER_MODE_ABS_PINNED); 662 } 663 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns); 664 665 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd) 666 { 667 hrtimer_cancel(&wd->timer); 668 } 669 EXPORT_SYMBOL(qdisc_watchdog_cancel); 670 671 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n) 672 { 673 struct hlist_head *h; 674 unsigned int i; 675 676 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL); 677 678 if (h != NULL) { 679 for (i = 0; i < n; i++) 680 INIT_HLIST_HEAD(&h[i]); 681 } 682 return h; 683 } 684 685 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash) 686 { 687 struct Qdisc_class_common *cl; 688 struct hlist_node *next; 689 struct hlist_head *nhash, *ohash; 690 unsigned int nsize, nmask, osize; 691 unsigned int i, h; 692 693 /* Rehash when load factor exceeds 0.75 */ 694 if (clhash->hashelems * 4 <= clhash->hashsize * 3) 695 return; 696 nsize = clhash->hashsize * 2; 697 nmask = nsize - 1; 698 nhash = qdisc_class_hash_alloc(nsize); 699 if (nhash == NULL) 700 return; 701 702 ohash = clhash->hash; 703 osize = clhash->hashsize; 704 705 sch_tree_lock(sch); 706 for (i = 0; i < osize; i++) { 707 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) { 708 h = qdisc_class_hash(cl->classid, nmask); 709 hlist_add_head(&cl->hnode, &nhash[h]); 710 } 711 } 712 clhash->hash = nhash; 713 clhash->hashsize = nsize; 714 clhash->hashmask = nmask; 715 sch_tree_unlock(sch); 716 717 kvfree(ohash); 718 } 719 EXPORT_SYMBOL(qdisc_class_hash_grow); 720 721 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash) 722 { 723 unsigned int size = 4; 724 725 clhash->hash = qdisc_class_hash_alloc(size); 726 if (!clhash->hash) 727 return -ENOMEM; 728 clhash->hashsize = size; 729 clhash->hashmask = size - 1; 730 clhash->hashelems = 0; 731 return 0; 732 } 733 EXPORT_SYMBOL(qdisc_class_hash_init); 734 735 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash) 736 { 737 kvfree(clhash->hash); 738 } 739 EXPORT_SYMBOL(qdisc_class_hash_destroy); 740 741 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash, 742 struct Qdisc_class_common *cl) 743 { 744 unsigned int h; 745 746 INIT_HLIST_NODE(&cl->hnode); 747 h = qdisc_class_hash(cl->classid, clhash->hashmask); 748 hlist_add_head(&cl->hnode, &clhash->hash[h]); 749 clhash->hashelems++; 750 } 751 EXPORT_SYMBOL(qdisc_class_hash_insert); 752 753 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash, 754 struct Qdisc_class_common *cl) 755 { 756 hlist_del(&cl->hnode); 757 clhash->hashelems--; 758 } 759 EXPORT_SYMBOL(qdisc_class_hash_remove); 760 761 /* Allocate an unique handle from space managed by kernel 762 * Possible range is [8000-FFFF]:0000 (0x8000 values) 763 */ 764 static u32 qdisc_alloc_handle(struct net_device *dev) 765 { 766 int i = 0x8000; 767 static u32 autohandle = TC_H_MAKE(0x80000000U, 0); 768 769 do { 770 autohandle += TC_H_MAKE(0x10000U, 0); 771 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0)) 772 autohandle = TC_H_MAKE(0x80000000U, 0); 773 if (!qdisc_lookup(dev, autohandle)) 774 return autohandle; 775 cond_resched(); 776 } while (--i > 0); 777 778 return 0; 779 } 780 781 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) 782 { 783 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED; 784 const struct Qdisc_class_ops *cops; 785 unsigned long cl; 786 u32 parentid; 787 bool notify; 788 int drops; 789 790 if (n == 0 && len == 0) 791 return; 792 drops = max_t(int, n, 0); 793 rcu_read_lock(); 794 while ((parentid = sch->parent)) { 795 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS)) 796 break; 797 798 if (sch->flags & TCQ_F_NOPARENT) 799 break; 800 /* Notify parent qdisc only if child qdisc becomes empty. 801 * 802 * If child was empty even before update then backlog 803 * counter is screwed and we skip notification because 804 * parent class is already passive. 805 * 806 * If the original child was offloaded then it is allowed 807 * to be seem as empty, so the parent is notified anyway. 808 */ 809 notify = !sch->q.qlen && !WARN_ON_ONCE(!n && 810 !qdisc_is_offloaded); 811 /* TODO: perform the search on a per txq basis */ 812 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid)); 813 if (sch == NULL) { 814 WARN_ON_ONCE(parentid != TC_H_ROOT); 815 break; 816 } 817 cops = sch->ops->cl_ops; 818 if (notify && cops->qlen_notify) { 819 cl = cops->find(sch, parentid); 820 cops->qlen_notify(sch, cl); 821 } 822 sch->q.qlen -= n; 823 sch->qstats.backlog -= len; 824 __qdisc_qstats_drop(sch, drops); 825 } 826 rcu_read_unlock(); 827 } 828 EXPORT_SYMBOL(qdisc_tree_reduce_backlog); 829 830 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type, 831 void *type_data) 832 { 833 struct net_device *dev = qdisc_dev(sch); 834 int err; 835 836 sch->flags &= ~TCQ_F_OFFLOADED; 837 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 838 return 0; 839 840 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); 841 if (err == -EOPNOTSUPP) 842 return 0; 843 844 if (!err) 845 sch->flags |= TCQ_F_OFFLOADED; 846 847 return err; 848 } 849 EXPORT_SYMBOL(qdisc_offload_dump_helper); 850 851 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 852 struct Qdisc *new, struct Qdisc *old, 853 enum tc_setup_type type, void *type_data, 854 struct netlink_ext_ack *extack) 855 { 856 bool any_qdisc_is_offloaded; 857 int err; 858 859 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 860 return; 861 862 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); 863 864 /* Don't report error if the graft is part of destroy operation. */ 865 if (!err || !new || new == &noop_qdisc) 866 return; 867 868 /* Don't report error if the parent, the old child and the new 869 * one are not offloaded. 870 */ 871 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED; 872 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED; 873 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED; 874 875 if (any_qdisc_is_offloaded) 876 NL_SET_ERR_MSG(extack, "Offloading graft operation failed."); 877 } 878 EXPORT_SYMBOL(qdisc_offload_graft_helper); 879 880 void qdisc_offload_query_caps(struct net_device *dev, 881 enum tc_setup_type type, 882 void *caps, size_t caps_len) 883 { 884 const struct net_device_ops *ops = dev->netdev_ops; 885 struct tc_query_caps_base base = { 886 .type = type, 887 .caps = caps, 888 }; 889 890 memset(caps, 0, caps_len); 891 892 if (ops->ndo_setup_tc) 893 ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base); 894 } 895 EXPORT_SYMBOL(qdisc_offload_query_caps); 896 897 static void qdisc_offload_graft_root(struct net_device *dev, 898 struct Qdisc *new, struct Qdisc *old, 899 struct netlink_ext_ack *extack) 900 { 901 struct tc_root_qopt_offload graft_offload = { 902 .command = TC_ROOT_GRAFT, 903 .handle = new ? new->handle : 0, 904 .ingress = (new && new->flags & TCQ_F_INGRESS) || 905 (old && old->flags & TCQ_F_INGRESS), 906 }; 907 908 qdisc_offload_graft_helper(dev, NULL, new, old, 909 TC_SETUP_ROOT_QDISC, &graft_offload, extack); 910 } 911 912 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, 913 u32 portid, u32 seq, u16 flags, int event, 914 struct netlink_ext_ack *extack) 915 { 916 struct gnet_stats_basic_sync __percpu *cpu_bstats = NULL; 917 struct gnet_stats_queue __percpu *cpu_qstats = NULL; 918 struct tcmsg *tcm; 919 struct nlmsghdr *nlh; 920 unsigned char *b = skb_tail_pointer(skb); 921 struct gnet_dump d; 922 struct qdisc_size_table *stab; 923 u32 block_index; 924 __u32 qlen; 925 926 cond_resched(); 927 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 928 if (!nlh) 929 goto out_nlmsg_trim; 930 tcm = nlmsg_data(nlh); 931 tcm->tcm_family = AF_UNSPEC; 932 tcm->tcm__pad1 = 0; 933 tcm->tcm__pad2 = 0; 934 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 935 tcm->tcm_parent = clid; 936 tcm->tcm_handle = q->handle; 937 tcm->tcm_info = refcount_read(&q->refcnt); 938 if (nla_put_string(skb, TCA_KIND, q->ops->id)) 939 goto nla_put_failure; 940 if (q->ops->ingress_block_get) { 941 block_index = q->ops->ingress_block_get(q); 942 if (block_index && 943 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index)) 944 goto nla_put_failure; 945 } 946 if (q->ops->egress_block_get) { 947 block_index = q->ops->egress_block_get(q); 948 if (block_index && 949 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index)) 950 goto nla_put_failure; 951 } 952 if (q->ops->dump && q->ops->dump(q, skb) < 0) 953 goto nla_put_failure; 954 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED))) 955 goto nla_put_failure; 956 qlen = qdisc_qlen_sum(q); 957 958 stab = rtnl_dereference(q->stab); 959 if (stab && qdisc_dump_stab(skb, stab) < 0) 960 goto nla_put_failure; 961 962 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 963 NULL, &d, TCA_PAD) < 0) 964 goto nla_put_failure; 965 966 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0) 967 goto nla_put_failure; 968 969 if (qdisc_is_percpu_stats(q)) { 970 cpu_bstats = q->cpu_bstats; 971 cpu_qstats = q->cpu_qstats; 972 } 973 974 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 || 975 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 || 976 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0) 977 goto nla_put_failure; 978 979 if (gnet_stats_finish_copy(&d) < 0) 980 goto nla_put_failure; 981 982 if (extack && extack->_msg && 983 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 984 goto out_nlmsg_trim; 985 986 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 987 988 return skb->len; 989 990 out_nlmsg_trim: 991 nla_put_failure: 992 nlmsg_trim(skb, b); 993 return -1; 994 } 995 996 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible) 997 { 998 if (q->flags & TCQ_F_BUILTIN) 999 return true; 1000 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible) 1001 return true; 1002 1003 return false; 1004 } 1005 1006 static int qdisc_notify(struct net *net, struct sk_buff *oskb, 1007 struct nlmsghdr *n, u32 clid, 1008 struct Qdisc *old, struct Qdisc *new, 1009 struct netlink_ext_ack *extack) 1010 { 1011 struct sk_buff *skb; 1012 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1013 1014 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1015 if (!skb) 1016 return -ENOBUFS; 1017 1018 if (old && !tc_qdisc_dump_ignore(old, false)) { 1019 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq, 1020 0, RTM_DELQDISC, extack) < 0) 1021 goto err_out; 1022 } 1023 if (new && !tc_qdisc_dump_ignore(new, false)) { 1024 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq, 1025 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC, extack) < 0) 1026 goto err_out; 1027 } 1028 1029 if (skb->len) 1030 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1031 n->nlmsg_flags & NLM_F_ECHO); 1032 1033 err_out: 1034 kfree_skb(skb); 1035 return -EINVAL; 1036 } 1037 1038 static void notify_and_destroy(struct net *net, struct sk_buff *skb, 1039 struct nlmsghdr *n, u32 clid, 1040 struct Qdisc *old, struct Qdisc *new, 1041 struct netlink_ext_ack *extack) 1042 { 1043 if (new || old) 1044 qdisc_notify(net, skb, n, clid, old, new, extack); 1045 1046 if (old) 1047 qdisc_put(old); 1048 } 1049 1050 static void qdisc_clear_nolock(struct Qdisc *sch) 1051 { 1052 sch->flags &= ~TCQ_F_NOLOCK; 1053 if (!(sch->flags & TCQ_F_CPUSTATS)) 1054 return; 1055 1056 free_percpu(sch->cpu_bstats); 1057 free_percpu(sch->cpu_qstats); 1058 sch->cpu_bstats = NULL; 1059 sch->cpu_qstats = NULL; 1060 sch->flags &= ~TCQ_F_CPUSTATS; 1061 } 1062 1063 /* Graft qdisc "new" to class "classid" of qdisc "parent" or 1064 * to device "dev". 1065 * 1066 * When appropriate send a netlink notification using 'skb' 1067 * and "n". 1068 * 1069 * On success, destroy old qdisc. 1070 */ 1071 1072 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, 1073 struct sk_buff *skb, struct nlmsghdr *n, u32 classid, 1074 struct Qdisc *new, struct Qdisc *old, 1075 struct netlink_ext_ack *extack) 1076 { 1077 struct Qdisc *q = old; 1078 struct net *net = dev_net(dev); 1079 1080 if (parent == NULL) { 1081 unsigned int i, num_q, ingress; 1082 struct netdev_queue *dev_queue; 1083 1084 ingress = 0; 1085 num_q = dev->num_tx_queues; 1086 if ((q && q->flags & TCQ_F_INGRESS) || 1087 (new && new->flags & TCQ_F_INGRESS)) { 1088 ingress = 1; 1089 dev_queue = dev_ingress_queue(dev); 1090 if (!dev_queue) { 1091 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue"); 1092 return -ENOENT; 1093 } 1094 1095 q = rtnl_dereference(dev_queue->qdisc_sleeping); 1096 1097 /* This is the counterpart of that qdisc_refcount_inc_nz() call in 1098 * __tcf_qdisc_find() for filter requests. 1099 */ 1100 if (!qdisc_refcount_dec_if_one(q)) { 1101 NL_SET_ERR_MSG(extack, 1102 "Current ingress or clsact Qdisc has ongoing filter requests"); 1103 return -EBUSY; 1104 } 1105 } 1106 1107 if (dev->flags & IFF_UP) 1108 dev_deactivate(dev); 1109 1110 qdisc_offload_graft_root(dev, new, old, extack); 1111 1112 if (new && new->ops->attach && !ingress) 1113 goto skip; 1114 1115 if (!ingress) { 1116 for (i = 0; i < num_q; i++) { 1117 dev_queue = netdev_get_tx_queue(dev, i); 1118 old = dev_graft_qdisc(dev_queue, new); 1119 1120 if (new && i > 0) 1121 qdisc_refcount_inc(new); 1122 qdisc_put(old); 1123 } 1124 } else { 1125 old = dev_graft_qdisc(dev_queue, NULL); 1126 1127 /* {ingress,clsact}_destroy() @old before grafting @new to avoid 1128 * unprotected concurrent accesses to net_device::miniq_{in,e}gress 1129 * pointer(s) in mini_qdisc_pair_swap(). 1130 */ 1131 qdisc_notify(net, skb, n, classid, old, new, extack); 1132 qdisc_destroy(old); 1133 1134 dev_graft_qdisc(dev_queue, new); 1135 } 1136 1137 skip: 1138 if (!ingress) { 1139 old = rtnl_dereference(dev->qdisc); 1140 if (new && !new->ops->attach) 1141 qdisc_refcount_inc(new); 1142 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc); 1143 1144 notify_and_destroy(net, skb, n, classid, old, new, extack); 1145 1146 if (new && new->ops->attach) 1147 new->ops->attach(new); 1148 } 1149 1150 if (dev->flags & IFF_UP) 1151 dev_activate(dev); 1152 } else { 1153 const struct Qdisc_class_ops *cops = parent->ops->cl_ops; 1154 unsigned long cl; 1155 int err; 1156 1157 /* Only support running class lockless if parent is lockless */ 1158 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK)) 1159 qdisc_clear_nolock(new); 1160 1161 if (!cops || !cops->graft) 1162 return -EOPNOTSUPP; 1163 1164 cl = cops->find(parent, classid); 1165 if (!cl) { 1166 NL_SET_ERR_MSG(extack, "Specified class not found"); 1167 return -ENOENT; 1168 } 1169 1170 if (new && new->ops == &noqueue_qdisc_ops) { 1171 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class"); 1172 return -EINVAL; 1173 } 1174 1175 err = cops->graft(parent, cl, new, &old, extack); 1176 if (err) 1177 return err; 1178 notify_and_destroy(net, skb, n, classid, old, new, extack); 1179 } 1180 return 0; 1181 } 1182 1183 static int qdisc_block_add_dev(struct Qdisc *sch, struct net_device *dev, 1184 struct netlink_ext_ack *extack) 1185 { 1186 const struct Qdisc_class_ops *cl_ops = sch->ops->cl_ops; 1187 struct tcf_block *block; 1188 int err; 1189 1190 block = cl_ops->tcf_block(sch, TC_H_MIN_INGRESS, NULL); 1191 if (block) { 1192 err = xa_insert(&block->ports, dev->ifindex, dev, GFP_KERNEL); 1193 if (err) { 1194 NL_SET_ERR_MSG(extack, 1195 "ingress block dev insert failed"); 1196 return err; 1197 } 1198 } 1199 1200 block = cl_ops->tcf_block(sch, TC_H_MIN_EGRESS, NULL); 1201 if (block) { 1202 err = xa_insert(&block->ports, dev->ifindex, dev, GFP_KERNEL); 1203 if (err) { 1204 NL_SET_ERR_MSG(extack, 1205 "Egress block dev insert failed"); 1206 goto err_out; 1207 } 1208 } 1209 1210 return 0; 1211 1212 err_out: 1213 block = cl_ops->tcf_block(sch, TC_H_MIN_INGRESS, NULL); 1214 if (block) 1215 xa_erase(&block->ports, dev->ifindex); 1216 1217 return err; 1218 } 1219 1220 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca, 1221 struct netlink_ext_ack *extack) 1222 { 1223 u32 block_index; 1224 1225 if (tca[TCA_INGRESS_BLOCK]) { 1226 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]); 1227 1228 if (!block_index) { 1229 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0"); 1230 return -EINVAL; 1231 } 1232 if (!sch->ops->ingress_block_set) { 1233 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported"); 1234 return -EOPNOTSUPP; 1235 } 1236 sch->ops->ingress_block_set(sch, block_index); 1237 } 1238 if (tca[TCA_EGRESS_BLOCK]) { 1239 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]); 1240 1241 if (!block_index) { 1242 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0"); 1243 return -EINVAL; 1244 } 1245 if (!sch->ops->egress_block_set) { 1246 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported"); 1247 return -EOPNOTSUPP; 1248 } 1249 sch->ops->egress_block_set(sch, block_index); 1250 } 1251 return 0; 1252 } 1253 1254 /* 1255 Allocate and initialize new qdisc. 1256 1257 Parameters are passed via opt. 1258 */ 1259 1260 static struct Qdisc *qdisc_create(struct net_device *dev, 1261 struct netdev_queue *dev_queue, 1262 u32 parent, u32 handle, 1263 struct nlattr **tca, int *errp, 1264 struct netlink_ext_ack *extack) 1265 { 1266 int err; 1267 struct nlattr *kind = tca[TCA_KIND]; 1268 struct Qdisc *sch; 1269 struct Qdisc_ops *ops; 1270 struct qdisc_size_table *stab; 1271 1272 ops = qdisc_lookup_ops(kind); 1273 #ifdef CONFIG_MODULES 1274 if (ops == NULL && kind != NULL) { 1275 char name[IFNAMSIZ]; 1276 if (nla_strscpy(name, kind, IFNAMSIZ) >= 0) { 1277 /* We dropped the RTNL semaphore in order to 1278 * perform the module load. So, even if we 1279 * succeeded in loading the module we have to 1280 * tell the caller to replay the request. We 1281 * indicate this using -EAGAIN. 1282 * We replay the request because the device may 1283 * go away in the mean time. 1284 */ 1285 rtnl_unlock(); 1286 request_module("sch_%s", name); 1287 rtnl_lock(); 1288 ops = qdisc_lookup_ops(kind); 1289 if (ops != NULL) { 1290 /* We will try again qdisc_lookup_ops, 1291 * so don't keep a reference. 1292 */ 1293 module_put(ops->owner); 1294 err = -EAGAIN; 1295 goto err_out; 1296 } 1297 } 1298 } 1299 #endif 1300 1301 err = -ENOENT; 1302 if (!ops) { 1303 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown"); 1304 goto err_out; 1305 } 1306 1307 sch = qdisc_alloc(dev_queue, ops, extack); 1308 if (IS_ERR(sch)) { 1309 err = PTR_ERR(sch); 1310 goto err_out2; 1311 } 1312 1313 sch->parent = parent; 1314 1315 if (handle == TC_H_INGRESS) { 1316 if (!(sch->flags & TCQ_F_INGRESS)) { 1317 NL_SET_ERR_MSG(extack, 1318 "Specified parent ID is reserved for ingress and clsact Qdiscs"); 1319 err = -EINVAL; 1320 goto err_out3; 1321 } 1322 handle = TC_H_MAKE(TC_H_INGRESS, 0); 1323 } else { 1324 if (handle == 0) { 1325 handle = qdisc_alloc_handle(dev); 1326 if (handle == 0) { 1327 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded"); 1328 err = -ENOSPC; 1329 goto err_out3; 1330 } 1331 } 1332 if (!netif_is_multiqueue(dev)) 1333 sch->flags |= TCQ_F_ONETXQUEUE; 1334 } 1335 1336 sch->handle = handle; 1337 1338 /* This exist to keep backward compatible with a userspace 1339 * loophole, what allowed userspace to get IFF_NO_QUEUE 1340 * facility on older kernels by setting tx_queue_len=0 (prior 1341 * to qdisc init), and then forgot to reinit tx_queue_len 1342 * before again attaching a qdisc. 1343 */ 1344 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) { 1345 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 1346 netdev_info(dev, "Caught tx_queue_len zero misconfig\n"); 1347 } 1348 1349 err = qdisc_block_indexes_set(sch, tca, extack); 1350 if (err) 1351 goto err_out3; 1352 1353 if (tca[TCA_STAB]) { 1354 stab = qdisc_get_stab(tca[TCA_STAB], extack); 1355 if (IS_ERR(stab)) { 1356 err = PTR_ERR(stab); 1357 goto err_out3; 1358 } 1359 rcu_assign_pointer(sch->stab, stab); 1360 } 1361 1362 if (ops->init) { 1363 err = ops->init(sch, tca[TCA_OPTIONS], extack); 1364 if (err != 0) 1365 goto err_out4; 1366 } 1367 1368 if (tca[TCA_RATE]) { 1369 err = -EOPNOTSUPP; 1370 if (sch->flags & TCQ_F_MQROOT) { 1371 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc"); 1372 goto err_out4; 1373 } 1374 1375 err = gen_new_estimator(&sch->bstats, 1376 sch->cpu_bstats, 1377 &sch->rate_est, 1378 NULL, 1379 true, 1380 tca[TCA_RATE]); 1381 if (err) { 1382 NL_SET_ERR_MSG(extack, "Failed to generate new estimator"); 1383 goto err_out4; 1384 } 1385 } 1386 1387 qdisc_hash_add(sch, false); 1388 trace_qdisc_create(ops, dev, parent); 1389 1390 err = qdisc_block_add_dev(sch, dev, extack); 1391 if (err) 1392 goto err_out4; 1393 1394 return sch; 1395 1396 err_out4: 1397 /* Even if ops->init() failed, we call ops->destroy() 1398 * like qdisc_create_dflt(). 1399 */ 1400 if (ops->destroy) 1401 ops->destroy(sch); 1402 qdisc_put_stab(rtnl_dereference(sch->stab)); 1403 err_out3: 1404 netdev_put(dev, &sch->dev_tracker); 1405 qdisc_free(sch); 1406 err_out2: 1407 module_put(ops->owner); 1408 err_out: 1409 *errp = err; 1410 return NULL; 1411 } 1412 1413 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca, 1414 struct netlink_ext_ack *extack) 1415 { 1416 struct qdisc_size_table *ostab, *stab = NULL; 1417 int err = 0; 1418 1419 if (tca[TCA_OPTIONS]) { 1420 if (!sch->ops->change) { 1421 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc"); 1422 return -EINVAL; 1423 } 1424 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) { 1425 NL_SET_ERR_MSG(extack, "Change of blocks is not supported"); 1426 return -EOPNOTSUPP; 1427 } 1428 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack); 1429 if (err) 1430 return err; 1431 } 1432 1433 if (tca[TCA_STAB]) { 1434 stab = qdisc_get_stab(tca[TCA_STAB], extack); 1435 if (IS_ERR(stab)) 1436 return PTR_ERR(stab); 1437 } 1438 1439 ostab = rtnl_dereference(sch->stab); 1440 rcu_assign_pointer(sch->stab, stab); 1441 qdisc_put_stab(ostab); 1442 1443 if (tca[TCA_RATE]) { 1444 /* NB: ignores errors from replace_estimator 1445 because change can't be undone. */ 1446 if (sch->flags & TCQ_F_MQROOT) 1447 goto out; 1448 gen_replace_estimator(&sch->bstats, 1449 sch->cpu_bstats, 1450 &sch->rate_est, 1451 NULL, 1452 true, 1453 tca[TCA_RATE]); 1454 } 1455 out: 1456 return 0; 1457 } 1458 1459 struct check_loop_arg { 1460 struct qdisc_walker w; 1461 struct Qdisc *p; 1462 int depth; 1463 }; 1464 1465 static int check_loop_fn(struct Qdisc *q, unsigned long cl, 1466 struct qdisc_walker *w); 1467 1468 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth) 1469 { 1470 struct check_loop_arg arg; 1471 1472 if (q->ops->cl_ops == NULL) 1473 return 0; 1474 1475 arg.w.stop = arg.w.skip = arg.w.count = 0; 1476 arg.w.fn = check_loop_fn; 1477 arg.depth = depth; 1478 arg.p = p; 1479 q->ops->cl_ops->walk(q, &arg.w); 1480 return arg.w.stop ? -ELOOP : 0; 1481 } 1482 1483 static int 1484 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w) 1485 { 1486 struct Qdisc *leaf; 1487 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1488 struct check_loop_arg *arg = (struct check_loop_arg *)w; 1489 1490 leaf = cops->leaf(q, cl); 1491 if (leaf) { 1492 if (leaf == arg->p || arg->depth > 7) 1493 return -ELOOP; 1494 return check_loop(leaf, arg->p, arg->depth + 1); 1495 } 1496 return 0; 1497 } 1498 1499 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = { 1500 [TCA_KIND] = { .type = NLA_STRING }, 1501 [TCA_RATE] = { .type = NLA_BINARY, 1502 .len = sizeof(struct tc_estimator) }, 1503 [TCA_STAB] = { .type = NLA_NESTED }, 1504 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG }, 1505 [TCA_CHAIN] = { .type = NLA_U32 }, 1506 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 }, 1507 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 }, 1508 }; 1509 1510 /* 1511 * Delete/get qdisc. 1512 */ 1513 1514 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1515 struct netlink_ext_ack *extack) 1516 { 1517 struct net *net = sock_net(skb->sk); 1518 struct tcmsg *tcm = nlmsg_data(n); 1519 struct nlattr *tca[TCA_MAX + 1]; 1520 struct net_device *dev; 1521 u32 clid; 1522 struct Qdisc *q = NULL; 1523 struct Qdisc *p = NULL; 1524 int err; 1525 1526 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 1527 rtm_tca_policy, extack); 1528 if (err < 0) 1529 return err; 1530 1531 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1532 if (!dev) 1533 return -ENODEV; 1534 1535 clid = tcm->tcm_parent; 1536 if (clid) { 1537 if (clid != TC_H_ROOT) { 1538 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) { 1539 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1540 if (!p) { 1541 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid"); 1542 return -ENOENT; 1543 } 1544 q = qdisc_leaf(p, clid); 1545 } else if (dev_ingress_queue(dev)) { 1546 q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping); 1547 } 1548 } else { 1549 q = rtnl_dereference(dev->qdisc); 1550 } 1551 if (!q) { 1552 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device"); 1553 return -ENOENT; 1554 } 1555 1556 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) { 1557 NL_SET_ERR_MSG(extack, "Invalid handle"); 1558 return -EINVAL; 1559 } 1560 } else { 1561 q = qdisc_lookup(dev, tcm->tcm_handle); 1562 if (!q) { 1563 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle"); 1564 return -ENOENT; 1565 } 1566 } 1567 1568 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1569 NL_SET_ERR_MSG(extack, "Invalid qdisc name"); 1570 return -EINVAL; 1571 } 1572 1573 if (n->nlmsg_type == RTM_DELQDISC) { 1574 if (!clid) { 1575 NL_SET_ERR_MSG(extack, "Classid cannot be zero"); 1576 return -EINVAL; 1577 } 1578 if (q->handle == 0) { 1579 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero"); 1580 return -ENOENT; 1581 } 1582 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack); 1583 if (err != 0) 1584 return err; 1585 } else { 1586 qdisc_notify(net, skb, n, clid, NULL, q, NULL); 1587 } 1588 return 0; 1589 } 1590 1591 static bool req_create_or_replace(struct nlmsghdr *n) 1592 { 1593 return (n->nlmsg_flags & NLM_F_CREATE && 1594 n->nlmsg_flags & NLM_F_REPLACE); 1595 } 1596 1597 static bool req_create_exclusive(struct nlmsghdr *n) 1598 { 1599 return (n->nlmsg_flags & NLM_F_CREATE && 1600 n->nlmsg_flags & NLM_F_EXCL); 1601 } 1602 1603 static bool req_change(struct nlmsghdr *n) 1604 { 1605 return (!(n->nlmsg_flags & NLM_F_CREATE) && 1606 !(n->nlmsg_flags & NLM_F_REPLACE) && 1607 !(n->nlmsg_flags & NLM_F_EXCL)); 1608 } 1609 1610 /* 1611 * Create/change qdisc. 1612 */ 1613 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1614 struct netlink_ext_ack *extack) 1615 { 1616 struct net *net = sock_net(skb->sk); 1617 struct tcmsg *tcm; 1618 struct nlattr *tca[TCA_MAX + 1]; 1619 struct net_device *dev; 1620 u32 clid; 1621 struct Qdisc *q, *p; 1622 int err; 1623 1624 replay: 1625 /* Reinit, just in case something touches this. */ 1626 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 1627 rtm_tca_policy, extack); 1628 if (err < 0) 1629 return err; 1630 1631 tcm = nlmsg_data(n); 1632 clid = tcm->tcm_parent; 1633 q = p = NULL; 1634 1635 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1636 if (!dev) 1637 return -ENODEV; 1638 1639 1640 if (clid) { 1641 if (clid != TC_H_ROOT) { 1642 if (clid != TC_H_INGRESS) { 1643 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1644 if (!p) { 1645 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc"); 1646 return -ENOENT; 1647 } 1648 q = qdisc_leaf(p, clid); 1649 } else if (dev_ingress_queue_create(dev)) { 1650 q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping); 1651 } 1652 } else { 1653 q = rtnl_dereference(dev->qdisc); 1654 } 1655 1656 /* It may be default qdisc, ignore it */ 1657 if (q && q->handle == 0) 1658 q = NULL; 1659 1660 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) { 1661 if (tcm->tcm_handle) { 1662 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) { 1663 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override"); 1664 return -EEXIST; 1665 } 1666 if (TC_H_MIN(tcm->tcm_handle)) { 1667 NL_SET_ERR_MSG(extack, "Invalid minor handle"); 1668 return -EINVAL; 1669 } 1670 q = qdisc_lookup(dev, tcm->tcm_handle); 1671 if (!q) 1672 goto create_n_graft; 1673 if (n->nlmsg_flags & NLM_F_EXCL) { 1674 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override"); 1675 return -EEXIST; 1676 } 1677 if (tca[TCA_KIND] && 1678 nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1679 NL_SET_ERR_MSG(extack, "Invalid qdisc name"); 1680 return -EINVAL; 1681 } 1682 if (q->flags & TCQ_F_INGRESS) { 1683 NL_SET_ERR_MSG(extack, 1684 "Cannot regraft ingress or clsact Qdiscs"); 1685 return -EINVAL; 1686 } 1687 if (q == p || 1688 (p && check_loop(q, p, 0))) { 1689 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected"); 1690 return -ELOOP; 1691 } 1692 if (clid == TC_H_INGRESS) { 1693 NL_SET_ERR_MSG(extack, "Ingress cannot graft directly"); 1694 return -EINVAL; 1695 } 1696 qdisc_refcount_inc(q); 1697 goto graft; 1698 } else { 1699 if (!q) 1700 goto create_n_graft; 1701 1702 /* This magic test requires explanation. 1703 * 1704 * We know, that some child q is already 1705 * attached to this parent and have choice: 1706 * 1) change it or 2) create/graft new one. 1707 * If the requested qdisc kind is different 1708 * than the existing one, then we choose graft. 1709 * If they are the same then this is "change" 1710 * operation - just let it fallthrough.. 1711 * 1712 * 1. We are allowed to create/graft only 1713 * if the request is explicitly stating 1714 * "please create if it doesn't exist". 1715 * 1716 * 2. If the request is to exclusive create 1717 * then the qdisc tcm_handle is not expected 1718 * to exist, so that we choose create/graft too. 1719 * 1720 * 3. The last case is when no flags are set. 1721 * This will happen when for example tc 1722 * utility issues a "change" command. 1723 * Alas, it is sort of hole in API, we 1724 * cannot decide what to do unambiguously. 1725 * For now we select create/graft. 1726 */ 1727 if (tca[TCA_KIND] && 1728 nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1729 if (req_create_or_replace(n) || 1730 req_create_exclusive(n)) 1731 goto create_n_graft; 1732 else if (req_change(n)) 1733 goto create_n_graft2; 1734 } 1735 } 1736 } 1737 } else { 1738 if (!tcm->tcm_handle) { 1739 NL_SET_ERR_MSG(extack, "Handle cannot be zero"); 1740 return -EINVAL; 1741 } 1742 q = qdisc_lookup(dev, tcm->tcm_handle); 1743 } 1744 1745 /* Change qdisc parameters */ 1746 if (!q) { 1747 NL_SET_ERR_MSG(extack, "Specified qdisc not found"); 1748 return -ENOENT; 1749 } 1750 if (n->nlmsg_flags & NLM_F_EXCL) { 1751 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify"); 1752 return -EEXIST; 1753 } 1754 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1755 NL_SET_ERR_MSG(extack, "Invalid qdisc name"); 1756 return -EINVAL; 1757 } 1758 err = qdisc_change(q, tca, extack); 1759 if (err == 0) 1760 qdisc_notify(net, skb, n, clid, NULL, q, extack); 1761 return err; 1762 1763 create_n_graft: 1764 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 1765 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag"); 1766 return -ENOENT; 1767 } 1768 create_n_graft2: 1769 if (clid == TC_H_INGRESS) { 1770 if (dev_ingress_queue(dev)) { 1771 q = qdisc_create(dev, dev_ingress_queue(dev), 1772 tcm->tcm_parent, tcm->tcm_parent, 1773 tca, &err, extack); 1774 } else { 1775 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device"); 1776 err = -ENOENT; 1777 } 1778 } else { 1779 struct netdev_queue *dev_queue; 1780 1781 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue) 1782 dev_queue = p->ops->cl_ops->select_queue(p, tcm); 1783 else if (p) 1784 dev_queue = p->dev_queue; 1785 else 1786 dev_queue = netdev_get_tx_queue(dev, 0); 1787 1788 q = qdisc_create(dev, dev_queue, 1789 tcm->tcm_parent, tcm->tcm_handle, 1790 tca, &err, extack); 1791 } 1792 if (q == NULL) { 1793 if (err == -EAGAIN) 1794 goto replay; 1795 return err; 1796 } 1797 1798 graft: 1799 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack); 1800 if (err) { 1801 if (q) 1802 qdisc_put(q); 1803 return err; 1804 } 1805 1806 return 0; 1807 } 1808 1809 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, 1810 struct netlink_callback *cb, 1811 int *q_idx_p, int s_q_idx, bool recur, 1812 bool dump_invisible) 1813 { 1814 int ret = 0, q_idx = *q_idx_p; 1815 struct Qdisc *q; 1816 int b; 1817 1818 if (!root) 1819 return 0; 1820 1821 q = root; 1822 if (q_idx < s_q_idx) { 1823 q_idx++; 1824 } else { 1825 if (!tc_qdisc_dump_ignore(q, dump_invisible) && 1826 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid, 1827 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1828 RTM_NEWQDISC, NULL) <= 0) 1829 goto done; 1830 q_idx++; 1831 } 1832 1833 /* If dumping singletons, there is no qdisc_dev(root) and the singleton 1834 * itself has already been dumped. 1835 * 1836 * If we've already dumped the top-level (ingress) qdisc above and the global 1837 * qdisc hashtable, we don't want to hit it again 1838 */ 1839 if (!qdisc_dev(root) || !recur) 1840 goto out; 1841 1842 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) { 1843 if (q_idx < s_q_idx) { 1844 q_idx++; 1845 continue; 1846 } 1847 if (!tc_qdisc_dump_ignore(q, dump_invisible) && 1848 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid, 1849 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1850 RTM_NEWQDISC, NULL) <= 0) 1851 goto done; 1852 q_idx++; 1853 } 1854 1855 out: 1856 *q_idx_p = q_idx; 1857 return ret; 1858 done: 1859 ret = -1; 1860 goto out; 1861 } 1862 1863 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) 1864 { 1865 struct net *net = sock_net(skb->sk); 1866 int idx, q_idx; 1867 int s_idx, s_q_idx; 1868 struct net_device *dev; 1869 const struct nlmsghdr *nlh = cb->nlh; 1870 struct nlattr *tca[TCA_MAX + 1]; 1871 int err; 1872 1873 s_idx = cb->args[0]; 1874 s_q_idx = q_idx = cb->args[1]; 1875 1876 idx = 0; 1877 ASSERT_RTNL(); 1878 1879 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX, 1880 rtm_tca_policy, cb->extack); 1881 if (err < 0) 1882 return err; 1883 1884 for_each_netdev(net, dev) { 1885 struct netdev_queue *dev_queue; 1886 1887 if (idx < s_idx) 1888 goto cont; 1889 if (idx > s_idx) 1890 s_q_idx = 0; 1891 q_idx = 0; 1892 1893 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc), 1894 skb, cb, &q_idx, s_q_idx, 1895 true, tca[TCA_DUMP_INVISIBLE]) < 0) 1896 goto done; 1897 1898 dev_queue = dev_ingress_queue(dev); 1899 if (dev_queue && 1900 tc_dump_qdisc_root(rtnl_dereference(dev_queue->qdisc_sleeping), 1901 skb, cb, &q_idx, s_q_idx, false, 1902 tca[TCA_DUMP_INVISIBLE]) < 0) 1903 goto done; 1904 1905 cont: 1906 idx++; 1907 } 1908 1909 done: 1910 cb->args[0] = idx; 1911 cb->args[1] = q_idx; 1912 1913 return skb->len; 1914 } 1915 1916 1917 1918 /************************************************ 1919 * Traffic classes manipulation. * 1920 ************************************************/ 1921 1922 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q, 1923 unsigned long cl, u32 portid, u32 seq, u16 flags, 1924 int event, struct netlink_ext_ack *extack) 1925 { 1926 struct tcmsg *tcm; 1927 struct nlmsghdr *nlh; 1928 unsigned char *b = skb_tail_pointer(skb); 1929 struct gnet_dump d; 1930 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops; 1931 1932 cond_resched(); 1933 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 1934 if (!nlh) 1935 goto out_nlmsg_trim; 1936 tcm = nlmsg_data(nlh); 1937 tcm->tcm_family = AF_UNSPEC; 1938 tcm->tcm__pad1 = 0; 1939 tcm->tcm__pad2 = 0; 1940 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1941 tcm->tcm_parent = q->handle; 1942 tcm->tcm_handle = q->handle; 1943 tcm->tcm_info = 0; 1944 if (nla_put_string(skb, TCA_KIND, q->ops->id)) 1945 goto nla_put_failure; 1946 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0) 1947 goto nla_put_failure; 1948 1949 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 1950 NULL, &d, TCA_PAD) < 0) 1951 goto nla_put_failure; 1952 1953 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0) 1954 goto nla_put_failure; 1955 1956 if (gnet_stats_finish_copy(&d) < 0) 1957 goto nla_put_failure; 1958 1959 if (extack && extack->_msg && 1960 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 1961 goto out_nlmsg_trim; 1962 1963 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1964 1965 return skb->len; 1966 1967 out_nlmsg_trim: 1968 nla_put_failure: 1969 nlmsg_trim(skb, b); 1970 return -1; 1971 } 1972 1973 static int tclass_notify(struct net *net, struct sk_buff *oskb, 1974 struct nlmsghdr *n, struct Qdisc *q, 1975 unsigned long cl, int event, struct netlink_ext_ack *extack) 1976 { 1977 struct sk_buff *skb; 1978 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1979 1980 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1981 if (!skb) 1982 return -ENOBUFS; 1983 1984 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event, extack) < 0) { 1985 kfree_skb(skb); 1986 return -EINVAL; 1987 } 1988 1989 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1990 n->nlmsg_flags & NLM_F_ECHO); 1991 } 1992 1993 static int tclass_del_notify(struct net *net, 1994 const struct Qdisc_class_ops *cops, 1995 struct sk_buff *oskb, struct nlmsghdr *n, 1996 struct Qdisc *q, unsigned long cl, 1997 struct netlink_ext_ack *extack) 1998 { 1999 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2000 struct sk_buff *skb; 2001 int err = 0; 2002 2003 if (!cops->delete) 2004 return -EOPNOTSUPP; 2005 2006 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2007 if (!skb) 2008 return -ENOBUFS; 2009 2010 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, 2011 RTM_DELTCLASS, extack) < 0) { 2012 kfree_skb(skb); 2013 return -EINVAL; 2014 } 2015 2016 err = cops->delete(q, cl, extack); 2017 if (err) { 2018 kfree_skb(skb); 2019 return err; 2020 } 2021 2022 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2023 n->nlmsg_flags & NLM_F_ECHO); 2024 return err; 2025 } 2026 2027 #ifdef CONFIG_NET_CLS 2028 2029 struct tcf_bind_args { 2030 struct tcf_walker w; 2031 unsigned long base; 2032 unsigned long cl; 2033 u32 classid; 2034 }; 2035 2036 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 2037 { 2038 struct tcf_bind_args *a = (void *)arg; 2039 2040 if (n && tp->ops->bind_class) { 2041 struct Qdisc *q = tcf_block_q(tp->chain->block); 2042 2043 sch_tree_lock(q); 2044 tp->ops->bind_class(n, a->classid, a->cl, q, a->base); 2045 sch_tree_unlock(q); 2046 } 2047 return 0; 2048 } 2049 2050 struct tc_bind_class_args { 2051 struct qdisc_walker w; 2052 unsigned long new_cl; 2053 u32 portid; 2054 u32 clid; 2055 }; 2056 2057 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl, 2058 struct qdisc_walker *w) 2059 { 2060 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w; 2061 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 2062 struct tcf_block *block; 2063 struct tcf_chain *chain; 2064 2065 block = cops->tcf_block(q, cl, NULL); 2066 if (!block) 2067 return 0; 2068 for (chain = tcf_get_next_chain(block, NULL); 2069 chain; 2070 chain = tcf_get_next_chain(block, chain)) { 2071 struct tcf_proto *tp; 2072 2073 for (tp = tcf_get_next_proto(chain, NULL); 2074 tp; tp = tcf_get_next_proto(chain, tp)) { 2075 struct tcf_bind_args arg = {}; 2076 2077 arg.w.fn = tcf_node_bind; 2078 arg.classid = a->clid; 2079 arg.base = cl; 2080 arg.cl = a->new_cl; 2081 tp->ops->walk(tp, &arg.w, true); 2082 } 2083 } 2084 2085 return 0; 2086 } 2087 2088 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid, 2089 unsigned long new_cl) 2090 { 2091 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 2092 struct tc_bind_class_args args = {}; 2093 2094 if (!cops->tcf_block) 2095 return; 2096 args.portid = portid; 2097 args.clid = clid; 2098 args.new_cl = new_cl; 2099 args.w.fn = tc_bind_class_walker; 2100 q->ops->cl_ops->walk(q, &args.w); 2101 } 2102 2103 #else 2104 2105 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid, 2106 unsigned long new_cl) 2107 { 2108 } 2109 2110 #endif 2111 2112 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, 2113 struct netlink_ext_ack *extack) 2114 { 2115 struct net *net = sock_net(skb->sk); 2116 struct tcmsg *tcm = nlmsg_data(n); 2117 struct nlattr *tca[TCA_MAX + 1]; 2118 struct net_device *dev; 2119 struct Qdisc *q = NULL; 2120 const struct Qdisc_class_ops *cops; 2121 unsigned long cl = 0; 2122 unsigned long new_cl; 2123 u32 portid; 2124 u32 clid; 2125 u32 qid; 2126 int err; 2127 2128 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 2129 rtm_tca_policy, extack); 2130 if (err < 0) 2131 return err; 2132 2133 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2134 if (!dev) 2135 return -ENODEV; 2136 2137 /* 2138 parent == TC_H_UNSPEC - unspecified parent. 2139 parent == TC_H_ROOT - class is root, which has no parent. 2140 parent == X:0 - parent is root class. 2141 parent == X:Y - parent is a node in hierarchy. 2142 parent == 0:Y - parent is X:Y, where X:0 is qdisc. 2143 2144 handle == 0:0 - generate handle from kernel pool. 2145 handle == 0:Y - class is X:Y, where X:0 is qdisc. 2146 handle == X:Y - clear. 2147 handle == X:0 - root class. 2148 */ 2149 2150 /* Step 1. Determine qdisc handle X:0 */ 2151 2152 portid = tcm->tcm_parent; 2153 clid = tcm->tcm_handle; 2154 qid = TC_H_MAJ(clid); 2155 2156 if (portid != TC_H_ROOT) { 2157 u32 qid1 = TC_H_MAJ(portid); 2158 2159 if (qid && qid1) { 2160 /* If both majors are known, they must be identical. */ 2161 if (qid != qid1) 2162 return -EINVAL; 2163 } else if (qid1) { 2164 qid = qid1; 2165 } else if (qid == 0) 2166 qid = rtnl_dereference(dev->qdisc)->handle; 2167 2168 /* Now qid is genuine qdisc handle consistent 2169 * both with parent and child. 2170 * 2171 * TC_H_MAJ(portid) still may be unspecified, complete it now. 2172 */ 2173 if (portid) 2174 portid = TC_H_MAKE(qid, portid); 2175 } else { 2176 if (qid == 0) 2177 qid = rtnl_dereference(dev->qdisc)->handle; 2178 } 2179 2180 /* OK. Locate qdisc */ 2181 q = qdisc_lookup(dev, qid); 2182 if (!q) 2183 return -ENOENT; 2184 2185 /* An check that it supports classes */ 2186 cops = q->ops->cl_ops; 2187 if (cops == NULL) 2188 return -EINVAL; 2189 2190 /* Now try to get class */ 2191 if (clid == 0) { 2192 if (portid == TC_H_ROOT) 2193 clid = qid; 2194 } else 2195 clid = TC_H_MAKE(qid, clid); 2196 2197 if (clid) 2198 cl = cops->find(q, clid); 2199 2200 if (cl == 0) { 2201 err = -ENOENT; 2202 if (n->nlmsg_type != RTM_NEWTCLASS || 2203 !(n->nlmsg_flags & NLM_F_CREATE)) 2204 goto out; 2205 } else { 2206 switch (n->nlmsg_type) { 2207 case RTM_NEWTCLASS: 2208 err = -EEXIST; 2209 if (n->nlmsg_flags & NLM_F_EXCL) 2210 goto out; 2211 break; 2212 case RTM_DELTCLASS: 2213 err = tclass_del_notify(net, cops, skb, n, q, cl, extack); 2214 /* Unbind the class with flilters with 0 */ 2215 tc_bind_tclass(q, portid, clid, 0); 2216 goto out; 2217 case RTM_GETTCLASS: 2218 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS, extack); 2219 goto out; 2220 default: 2221 err = -EINVAL; 2222 goto out; 2223 } 2224 } 2225 2226 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) { 2227 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes"); 2228 return -EOPNOTSUPP; 2229 } 2230 2231 new_cl = cl; 2232 err = -EOPNOTSUPP; 2233 if (cops->change) 2234 err = cops->change(q, clid, portid, tca, &new_cl, extack); 2235 if (err == 0) { 2236 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS, extack); 2237 /* We just create a new class, need to do reverse binding. */ 2238 if (cl != new_cl) 2239 tc_bind_tclass(q, portid, clid, new_cl); 2240 } 2241 out: 2242 return err; 2243 } 2244 2245 struct qdisc_dump_args { 2246 struct qdisc_walker w; 2247 struct sk_buff *skb; 2248 struct netlink_callback *cb; 2249 }; 2250 2251 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, 2252 struct qdisc_walker *arg) 2253 { 2254 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg; 2255 2256 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid, 2257 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2258 RTM_NEWTCLASS, NULL); 2259 } 2260 2261 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb, 2262 struct tcmsg *tcm, struct netlink_callback *cb, 2263 int *t_p, int s_t) 2264 { 2265 struct qdisc_dump_args arg; 2266 2267 if (tc_qdisc_dump_ignore(q, false) || 2268 *t_p < s_t || !q->ops->cl_ops || 2269 (tcm->tcm_parent && 2270 TC_H_MAJ(tcm->tcm_parent) != q->handle)) { 2271 (*t_p)++; 2272 return 0; 2273 } 2274 if (*t_p > s_t) 2275 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0])); 2276 arg.w.fn = qdisc_class_dump; 2277 arg.skb = skb; 2278 arg.cb = cb; 2279 arg.w.stop = 0; 2280 arg.w.skip = cb->args[1]; 2281 arg.w.count = 0; 2282 q->ops->cl_ops->walk(q, &arg.w); 2283 cb->args[1] = arg.w.count; 2284 if (arg.w.stop) 2285 return -1; 2286 (*t_p)++; 2287 return 0; 2288 } 2289 2290 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, 2291 struct tcmsg *tcm, struct netlink_callback *cb, 2292 int *t_p, int s_t, bool recur) 2293 { 2294 struct Qdisc *q; 2295 int b; 2296 2297 if (!root) 2298 return 0; 2299 2300 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0) 2301 return -1; 2302 2303 if (!qdisc_dev(root) || !recur) 2304 return 0; 2305 2306 if (tcm->tcm_parent) { 2307 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent)); 2308 if (q && q != root && 2309 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 2310 return -1; 2311 return 0; 2312 } 2313 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) { 2314 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 2315 return -1; 2316 } 2317 2318 return 0; 2319 } 2320 2321 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) 2322 { 2323 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2324 struct net *net = sock_net(skb->sk); 2325 struct netdev_queue *dev_queue; 2326 struct net_device *dev; 2327 int t, s_t; 2328 2329 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2330 return 0; 2331 dev = dev_get_by_index(net, tcm->tcm_ifindex); 2332 if (!dev) 2333 return 0; 2334 2335 s_t = cb->args[0]; 2336 t = 0; 2337 2338 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc), 2339 skb, tcm, cb, &t, s_t, true) < 0) 2340 goto done; 2341 2342 dev_queue = dev_ingress_queue(dev); 2343 if (dev_queue && 2344 tc_dump_tclass_root(rtnl_dereference(dev_queue->qdisc_sleeping), 2345 skb, tcm, cb, &t, s_t, false) < 0) 2346 goto done; 2347 2348 done: 2349 cb->args[0] = t; 2350 2351 dev_put(dev); 2352 return skb->len; 2353 } 2354 2355 #ifdef CONFIG_PROC_FS 2356 static int psched_show(struct seq_file *seq, void *v) 2357 { 2358 seq_printf(seq, "%08x %08x %08x %08x\n", 2359 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1), 2360 1000000, 2361 (u32)NSEC_PER_SEC / hrtimer_resolution); 2362 2363 return 0; 2364 } 2365 2366 static int __net_init psched_net_init(struct net *net) 2367 { 2368 struct proc_dir_entry *e; 2369 2370 e = proc_create_single("psched", 0, net->proc_net, psched_show); 2371 if (e == NULL) 2372 return -ENOMEM; 2373 2374 return 0; 2375 } 2376 2377 static void __net_exit psched_net_exit(struct net *net) 2378 { 2379 remove_proc_entry("psched", net->proc_net); 2380 } 2381 #else 2382 static int __net_init psched_net_init(struct net *net) 2383 { 2384 return 0; 2385 } 2386 2387 static void __net_exit psched_net_exit(struct net *net) 2388 { 2389 } 2390 #endif 2391 2392 static struct pernet_operations psched_net_ops = { 2393 .init = psched_net_init, 2394 .exit = psched_net_exit, 2395 }; 2396 2397 #if IS_ENABLED(CONFIG_RETPOLINE) 2398 DEFINE_STATIC_KEY_FALSE(tc_skip_wrapper); 2399 #endif 2400 2401 static int __init pktsched_init(void) 2402 { 2403 int err; 2404 2405 err = register_pernet_subsys(&psched_net_ops); 2406 if (err) { 2407 pr_err("pktsched_init: " 2408 "cannot initialize per netns operations\n"); 2409 return err; 2410 } 2411 2412 register_qdisc(&pfifo_fast_ops); 2413 register_qdisc(&pfifo_qdisc_ops); 2414 register_qdisc(&bfifo_qdisc_ops); 2415 register_qdisc(&pfifo_head_drop_qdisc_ops); 2416 register_qdisc(&mq_qdisc_ops); 2417 register_qdisc(&noqueue_qdisc_ops); 2418 2419 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0); 2420 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0); 2421 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc, 2422 0); 2423 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0); 2424 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0); 2425 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass, 2426 0); 2427 2428 tc_wrapper_init(); 2429 2430 return 0; 2431 } 2432 2433 subsys_initcall(pktsched_init); 2434