1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/sch_api.c Packet scheduler API. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 * Fixes: 8 * 9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired. 10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support 12 */ 13 14 #include <linux/module.h> 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/string.h> 18 #include <linux/errno.h> 19 #include <linux/skbuff.h> 20 #include <linux/init.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/kmod.h> 24 #include <linux/list.h> 25 #include <linux/hrtimer.h> 26 #include <linux/slab.h> 27 #include <linux/hashtable.h> 28 29 #include <net/net_namespace.h> 30 #include <net/sock.h> 31 #include <net/netlink.h> 32 #include <net/pkt_sched.h> 33 #include <net/pkt_cls.h> 34 #include <net/tc_wrapper.h> 35 36 #include <trace/events/qdisc.h> 37 38 /* 39 40 Short review. 41 ------------- 42 43 This file consists of two interrelated parts: 44 45 1. queueing disciplines manager frontend. 46 2. traffic classes manager frontend. 47 48 Generally, queueing discipline ("qdisc") is a black box, 49 which is able to enqueue packets and to dequeue them (when 50 device is ready to send something) in order and at times 51 determined by algorithm hidden in it. 52 53 qdisc's are divided to two categories: 54 - "queues", which have no internal structure visible from outside. 55 - "schedulers", which split all the packets to "traffic classes", 56 using "packet classifiers" (look at cls_api.c) 57 58 In turn, classes may have child qdiscs (as rule, queues) 59 attached to them etc. etc. etc. 60 61 The goal of the routines in this file is to translate 62 information supplied by user in the form of handles 63 to more intelligible for kernel form, to make some sanity 64 checks and part of work, which is common to all qdiscs 65 and to provide rtnetlink notifications. 66 67 All real intelligent work is done inside qdisc modules. 68 69 70 71 Every discipline has two major routines: enqueue and dequeue. 72 73 ---dequeue 74 75 dequeue usually returns a skb to send. It is allowed to return NULL, 76 but it does not mean that queue is empty, it just means that 77 discipline does not want to send anything this time. 78 Queue is really empty if q->q.qlen == 0. 79 For complicated disciplines with multiple queues q->q is not 80 real packet queue, but however q->q.qlen must be valid. 81 82 ---enqueue 83 84 enqueue returns 0, if packet was enqueued successfully. 85 If packet (this one or another one) was dropped, it returns 86 not zero error code. 87 NET_XMIT_DROP - this packet dropped 88 Expected action: do not backoff, but wait until queue will clear. 89 NET_XMIT_CN - probably this packet enqueued, but another one dropped. 90 Expected action: backoff or ignore 91 92 Auxiliary routines: 93 94 ---peek 95 96 like dequeue but without removing a packet from the queue 97 98 ---reset 99 100 returns qdisc to initial state: purge all buffers, clear all 101 timers, counters (except for statistics) etc. 102 103 ---init 104 105 initializes newly created qdisc. 106 107 ---destroy 108 109 destroys resources allocated by init and during lifetime of qdisc. 110 111 ---change 112 113 changes qdisc parameters. 114 */ 115 116 /* Protects list of registered TC modules. It is pure SMP lock. */ 117 static DEFINE_RWLOCK(qdisc_mod_lock); 118 119 120 /************************************************ 121 * Queueing disciplines manipulation. * 122 ************************************************/ 123 124 125 /* The list of all installed queueing disciplines. */ 126 127 static struct Qdisc_ops *qdisc_base; 128 129 /* Register/unregister queueing discipline */ 130 131 int register_qdisc(struct Qdisc_ops *qops) 132 { 133 struct Qdisc_ops *q, **qp; 134 int rc = -EEXIST; 135 136 write_lock(&qdisc_mod_lock); 137 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 138 if (!strcmp(qops->id, q->id)) 139 goto out; 140 141 if (qops->enqueue == NULL) 142 qops->enqueue = noop_qdisc_ops.enqueue; 143 if (qops->peek == NULL) { 144 if (qops->dequeue == NULL) 145 qops->peek = noop_qdisc_ops.peek; 146 else 147 goto out_einval; 148 } 149 if (qops->dequeue == NULL) 150 qops->dequeue = noop_qdisc_ops.dequeue; 151 152 if (qops->cl_ops) { 153 const struct Qdisc_class_ops *cops = qops->cl_ops; 154 155 if (!(cops->find && cops->walk && cops->leaf)) 156 goto out_einval; 157 158 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf)) 159 goto out_einval; 160 } 161 162 qops->next = NULL; 163 *qp = qops; 164 rc = 0; 165 out: 166 write_unlock(&qdisc_mod_lock); 167 return rc; 168 169 out_einval: 170 rc = -EINVAL; 171 goto out; 172 } 173 EXPORT_SYMBOL(register_qdisc); 174 175 void unregister_qdisc(struct Qdisc_ops *qops) 176 { 177 struct Qdisc_ops *q, **qp; 178 int err = -ENOENT; 179 180 write_lock(&qdisc_mod_lock); 181 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 182 if (q == qops) 183 break; 184 if (q) { 185 *qp = q->next; 186 q->next = NULL; 187 err = 0; 188 } 189 write_unlock(&qdisc_mod_lock); 190 191 WARN(err, "unregister qdisc(%s) failed\n", qops->id); 192 } 193 EXPORT_SYMBOL(unregister_qdisc); 194 195 /* Get default qdisc if not otherwise specified */ 196 void qdisc_get_default(char *name, size_t len) 197 { 198 read_lock(&qdisc_mod_lock); 199 strscpy(name, default_qdisc_ops->id, len); 200 read_unlock(&qdisc_mod_lock); 201 } 202 203 static struct Qdisc_ops *qdisc_lookup_default(const char *name) 204 { 205 struct Qdisc_ops *q = NULL; 206 207 for (q = qdisc_base; q; q = q->next) { 208 if (!strcmp(name, q->id)) { 209 if (!try_module_get(q->owner)) 210 q = NULL; 211 break; 212 } 213 } 214 215 return q; 216 } 217 218 /* Set new default qdisc to use */ 219 int qdisc_set_default(const char *name) 220 { 221 const struct Qdisc_ops *ops; 222 223 if (!capable(CAP_NET_ADMIN)) 224 return -EPERM; 225 226 write_lock(&qdisc_mod_lock); 227 ops = qdisc_lookup_default(name); 228 if (!ops) { 229 /* Not found, drop lock and try to load module */ 230 write_unlock(&qdisc_mod_lock); 231 request_module(NET_SCH_ALIAS_PREFIX "%s", name); 232 write_lock(&qdisc_mod_lock); 233 234 ops = qdisc_lookup_default(name); 235 } 236 237 if (ops) { 238 /* Set new default */ 239 module_put(default_qdisc_ops->owner); 240 default_qdisc_ops = ops; 241 } 242 write_unlock(&qdisc_mod_lock); 243 244 return ops ? 0 : -ENOENT; 245 } 246 247 #ifdef CONFIG_NET_SCH_DEFAULT 248 /* Set default value from kernel config */ 249 static int __init sch_default_qdisc(void) 250 { 251 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH); 252 } 253 late_initcall(sch_default_qdisc); 254 #endif 255 256 /* We know handle. Find qdisc among all qdisc's attached to device 257 * (root qdisc, all its children, children of children etc.) 258 * Note: caller either uses rtnl or rcu_read_lock() 259 */ 260 261 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle) 262 { 263 struct Qdisc *q; 264 265 if (!qdisc_dev(root)) 266 return (root->handle == handle ? root : NULL); 267 268 if (!(root->flags & TCQ_F_BUILTIN) && 269 root->handle == handle) 270 return root; 271 272 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle, 273 lockdep_rtnl_is_held()) { 274 if (q->handle == handle) 275 return q; 276 } 277 return NULL; 278 } 279 280 void qdisc_hash_add(struct Qdisc *q, bool invisible) 281 { 282 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { 283 ASSERT_RTNL(); 284 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle); 285 if (invisible) 286 q->flags |= TCQ_F_INVISIBLE; 287 } 288 } 289 EXPORT_SYMBOL(qdisc_hash_add); 290 291 void qdisc_hash_del(struct Qdisc *q) 292 { 293 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { 294 ASSERT_RTNL(); 295 hash_del_rcu(&q->hash); 296 } 297 } 298 EXPORT_SYMBOL(qdisc_hash_del); 299 300 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) 301 { 302 struct Qdisc *q; 303 304 if (!handle) 305 return NULL; 306 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle); 307 if (q) 308 goto out; 309 310 if (dev_ingress_queue(dev)) 311 q = qdisc_match_from_root( 312 rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping), 313 handle); 314 out: 315 return q; 316 } 317 318 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle) 319 { 320 struct netdev_queue *nq; 321 struct Qdisc *q; 322 323 if (!handle) 324 return NULL; 325 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle); 326 if (q) 327 goto out; 328 329 nq = dev_ingress_queue_rcu(dev); 330 if (nq) 331 q = qdisc_match_from_root(rcu_dereference(nq->qdisc_sleeping), 332 handle); 333 out: 334 return q; 335 } 336 337 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) 338 { 339 unsigned long cl; 340 const struct Qdisc_class_ops *cops = p->ops->cl_ops; 341 342 if (cops == NULL) 343 return NULL; 344 cl = cops->find(p, classid); 345 346 if (cl == 0) 347 return NULL; 348 return cops->leaf(p, cl); 349 } 350 351 /* Find queueing discipline by name */ 352 353 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind) 354 { 355 struct Qdisc_ops *q = NULL; 356 357 if (kind) { 358 read_lock(&qdisc_mod_lock); 359 for (q = qdisc_base; q; q = q->next) { 360 if (nla_strcmp(kind, q->id) == 0) { 361 if (!try_module_get(q->owner)) 362 q = NULL; 363 break; 364 } 365 } 366 read_unlock(&qdisc_mod_lock); 367 } 368 return q; 369 } 370 371 /* The linklayer setting were not transferred from iproute2, in older 372 * versions, and the rate tables lookup systems have been dropped in 373 * the kernel. To keep backward compatible with older iproute2 tc 374 * utils, we detect the linklayer setting by detecting if the rate 375 * table were modified. 376 * 377 * For linklayer ATM table entries, the rate table will be aligned to 378 * 48 bytes, thus some table entries will contain the same value. The 379 * mpu (min packet unit) is also encoded into the old rate table, thus 380 * starting from the mpu, we find low and high table entries for 381 * mapping this cell. If these entries contain the same value, when 382 * the rate tables have been modified for linklayer ATM. 383 * 384 * This is done by rounding mpu to the nearest 48 bytes cell/entry, 385 * and then roundup to the next cell, calc the table entry one below, 386 * and compare. 387 */ 388 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab) 389 { 390 int low = roundup(r->mpu, 48); 391 int high = roundup(low+1, 48); 392 int cell_low = low >> r->cell_log; 393 int cell_high = (high >> r->cell_log) - 1; 394 395 /* rtab is too inaccurate at rates > 100Mbit/s */ 396 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) { 397 pr_debug("TC linklayer: Giving up ATM detection\n"); 398 return TC_LINKLAYER_ETHERNET; 399 } 400 401 if ((cell_high > cell_low) && (cell_high < 256) 402 && (rtab[cell_low] == rtab[cell_high])) { 403 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n", 404 cell_low, cell_high, rtab[cell_high]); 405 return TC_LINKLAYER_ATM; 406 } 407 return TC_LINKLAYER_ETHERNET; 408 } 409 410 static struct qdisc_rate_table *qdisc_rtab_list; 411 412 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, 413 struct nlattr *tab, 414 struct netlink_ext_ack *extack) 415 { 416 struct qdisc_rate_table *rtab; 417 418 if (tab == NULL || r->rate == 0 || 419 r->cell_log == 0 || r->cell_log >= 32 || 420 nla_len(tab) != TC_RTAB_SIZE) { 421 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching"); 422 return NULL; 423 } 424 425 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) { 426 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) && 427 !memcmp(&rtab->data, nla_data(tab), 1024)) { 428 rtab->refcnt++; 429 return rtab; 430 } 431 } 432 433 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL); 434 if (rtab) { 435 rtab->rate = *r; 436 rtab->refcnt = 1; 437 memcpy(rtab->data, nla_data(tab), 1024); 438 if (r->linklayer == TC_LINKLAYER_UNAWARE) 439 r->linklayer = __detect_linklayer(r, rtab->data); 440 rtab->next = qdisc_rtab_list; 441 qdisc_rtab_list = rtab; 442 } else { 443 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table"); 444 } 445 return rtab; 446 } 447 EXPORT_SYMBOL(qdisc_get_rtab); 448 449 void qdisc_put_rtab(struct qdisc_rate_table *tab) 450 { 451 struct qdisc_rate_table *rtab, **rtabp; 452 453 if (!tab || --tab->refcnt) 454 return; 455 456 for (rtabp = &qdisc_rtab_list; 457 (rtab = *rtabp) != NULL; 458 rtabp = &rtab->next) { 459 if (rtab == tab) { 460 *rtabp = rtab->next; 461 kfree(rtab); 462 return; 463 } 464 } 465 } 466 EXPORT_SYMBOL(qdisc_put_rtab); 467 468 static LIST_HEAD(qdisc_stab_list); 469 470 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = { 471 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) }, 472 [TCA_STAB_DATA] = { .type = NLA_BINARY }, 473 }; 474 475 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt, 476 struct netlink_ext_ack *extack) 477 { 478 struct nlattr *tb[TCA_STAB_MAX + 1]; 479 struct qdisc_size_table *stab; 480 struct tc_sizespec *s; 481 unsigned int tsize = 0; 482 u16 *tab = NULL; 483 int err; 484 485 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy, 486 extack); 487 if (err < 0) 488 return ERR_PTR(err); 489 if (!tb[TCA_STAB_BASE]) { 490 NL_SET_ERR_MSG(extack, "Size table base attribute is missing"); 491 return ERR_PTR(-EINVAL); 492 } 493 494 s = nla_data(tb[TCA_STAB_BASE]); 495 496 if (s->tsize > 0) { 497 if (!tb[TCA_STAB_DATA]) { 498 NL_SET_ERR_MSG(extack, "Size table data attribute is missing"); 499 return ERR_PTR(-EINVAL); 500 } 501 tab = nla_data(tb[TCA_STAB_DATA]); 502 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16); 503 } 504 505 if (tsize != s->tsize || (!tab && tsize > 0)) { 506 NL_SET_ERR_MSG(extack, "Invalid size of size table"); 507 return ERR_PTR(-EINVAL); 508 } 509 510 list_for_each_entry(stab, &qdisc_stab_list, list) { 511 if (memcmp(&stab->szopts, s, sizeof(*s))) 512 continue; 513 if (tsize > 0 && 514 memcmp(stab->data, tab, flex_array_size(stab, data, tsize))) 515 continue; 516 stab->refcnt++; 517 return stab; 518 } 519 520 if (s->size_log > STAB_SIZE_LOG_MAX || 521 s->cell_log > STAB_SIZE_LOG_MAX) { 522 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table"); 523 return ERR_PTR(-EINVAL); 524 } 525 526 stab = kmalloc(struct_size(stab, data, tsize), GFP_KERNEL); 527 if (!stab) 528 return ERR_PTR(-ENOMEM); 529 530 stab->refcnt = 1; 531 stab->szopts = *s; 532 if (tsize > 0) 533 memcpy(stab->data, tab, flex_array_size(stab, data, tsize)); 534 535 list_add_tail(&stab->list, &qdisc_stab_list); 536 537 return stab; 538 } 539 540 void qdisc_put_stab(struct qdisc_size_table *tab) 541 { 542 if (!tab) 543 return; 544 545 if (--tab->refcnt == 0) { 546 list_del(&tab->list); 547 kfree_rcu(tab, rcu); 548 } 549 } 550 EXPORT_SYMBOL(qdisc_put_stab); 551 552 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab) 553 { 554 struct nlattr *nest; 555 556 nest = nla_nest_start_noflag(skb, TCA_STAB); 557 if (nest == NULL) 558 goto nla_put_failure; 559 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts)) 560 goto nla_put_failure; 561 nla_nest_end(skb, nest); 562 563 return skb->len; 564 565 nla_put_failure: 566 return -1; 567 } 568 569 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 570 const struct qdisc_size_table *stab) 571 { 572 int pkt_len, slot; 573 574 pkt_len = skb->len + stab->szopts.overhead; 575 if (unlikely(!stab->szopts.tsize)) 576 goto out; 577 578 slot = pkt_len + stab->szopts.cell_align; 579 if (unlikely(slot < 0)) 580 slot = 0; 581 582 slot >>= stab->szopts.cell_log; 583 if (likely(slot < stab->szopts.tsize)) 584 pkt_len = stab->data[slot]; 585 else 586 pkt_len = stab->data[stab->szopts.tsize - 1] * 587 (slot / stab->szopts.tsize) + 588 stab->data[slot % stab->szopts.tsize]; 589 590 pkt_len <<= stab->szopts.size_log; 591 out: 592 if (unlikely(pkt_len < 1)) 593 pkt_len = 1; 594 qdisc_skb_cb(skb)->pkt_len = pkt_len; 595 } 596 597 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc) 598 { 599 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { 600 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", 601 txt, qdisc->ops->id, qdisc->handle >> 16); 602 qdisc->flags |= TCQ_F_WARN_NONWC; 603 } 604 } 605 EXPORT_SYMBOL(qdisc_warn_nonwc); 606 607 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) 608 { 609 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, 610 timer); 611 612 rcu_read_lock(); 613 __netif_schedule(qdisc_root(wd->qdisc)); 614 rcu_read_unlock(); 615 616 return HRTIMER_NORESTART; 617 } 618 619 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc, 620 clockid_t clockid) 621 { 622 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED); 623 wd->timer.function = qdisc_watchdog; 624 wd->qdisc = qdisc; 625 } 626 EXPORT_SYMBOL(qdisc_watchdog_init_clockid); 627 628 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc) 629 { 630 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC); 631 } 632 EXPORT_SYMBOL(qdisc_watchdog_init); 633 634 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires, 635 u64 delta_ns) 636 { 637 bool deactivated; 638 639 rcu_read_lock(); 640 deactivated = test_bit(__QDISC_STATE_DEACTIVATED, 641 &qdisc_root_sleeping(wd->qdisc)->state); 642 rcu_read_unlock(); 643 if (deactivated) 644 return; 645 646 if (hrtimer_is_queued(&wd->timer)) { 647 u64 softexpires; 648 649 softexpires = ktime_to_ns(hrtimer_get_softexpires(&wd->timer)); 650 /* If timer is already set in [expires, expires + delta_ns], 651 * do not reprogram it. 652 */ 653 if (softexpires - expires <= delta_ns) 654 return; 655 } 656 657 hrtimer_start_range_ns(&wd->timer, 658 ns_to_ktime(expires), 659 delta_ns, 660 HRTIMER_MODE_ABS_PINNED); 661 } 662 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns); 663 664 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd) 665 { 666 hrtimer_cancel(&wd->timer); 667 } 668 EXPORT_SYMBOL(qdisc_watchdog_cancel); 669 670 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n) 671 { 672 struct hlist_head *h; 673 unsigned int i; 674 675 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL); 676 677 if (h != NULL) { 678 for (i = 0; i < n; i++) 679 INIT_HLIST_HEAD(&h[i]); 680 } 681 return h; 682 } 683 684 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash) 685 { 686 struct Qdisc_class_common *cl; 687 struct hlist_node *next; 688 struct hlist_head *nhash, *ohash; 689 unsigned int nsize, nmask, osize; 690 unsigned int i, h; 691 692 /* Rehash when load factor exceeds 0.75 */ 693 if (clhash->hashelems * 4 <= clhash->hashsize * 3) 694 return; 695 nsize = clhash->hashsize * 2; 696 nmask = nsize - 1; 697 nhash = qdisc_class_hash_alloc(nsize); 698 if (nhash == NULL) 699 return; 700 701 ohash = clhash->hash; 702 osize = clhash->hashsize; 703 704 sch_tree_lock(sch); 705 for (i = 0; i < osize; i++) { 706 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) { 707 h = qdisc_class_hash(cl->classid, nmask); 708 hlist_add_head(&cl->hnode, &nhash[h]); 709 } 710 } 711 clhash->hash = nhash; 712 clhash->hashsize = nsize; 713 clhash->hashmask = nmask; 714 sch_tree_unlock(sch); 715 716 kvfree(ohash); 717 } 718 EXPORT_SYMBOL(qdisc_class_hash_grow); 719 720 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash) 721 { 722 unsigned int size = 4; 723 724 clhash->hash = qdisc_class_hash_alloc(size); 725 if (!clhash->hash) 726 return -ENOMEM; 727 clhash->hashsize = size; 728 clhash->hashmask = size - 1; 729 clhash->hashelems = 0; 730 return 0; 731 } 732 EXPORT_SYMBOL(qdisc_class_hash_init); 733 734 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash) 735 { 736 kvfree(clhash->hash); 737 } 738 EXPORT_SYMBOL(qdisc_class_hash_destroy); 739 740 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash, 741 struct Qdisc_class_common *cl) 742 { 743 unsigned int h; 744 745 INIT_HLIST_NODE(&cl->hnode); 746 h = qdisc_class_hash(cl->classid, clhash->hashmask); 747 hlist_add_head(&cl->hnode, &clhash->hash[h]); 748 clhash->hashelems++; 749 } 750 EXPORT_SYMBOL(qdisc_class_hash_insert); 751 752 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash, 753 struct Qdisc_class_common *cl) 754 { 755 hlist_del(&cl->hnode); 756 clhash->hashelems--; 757 } 758 EXPORT_SYMBOL(qdisc_class_hash_remove); 759 760 /* Allocate an unique handle from space managed by kernel 761 * Possible range is [8000-FFFF]:0000 (0x8000 values) 762 */ 763 static u32 qdisc_alloc_handle(struct net_device *dev) 764 { 765 int i = 0x8000; 766 static u32 autohandle = TC_H_MAKE(0x80000000U, 0); 767 768 do { 769 autohandle += TC_H_MAKE(0x10000U, 0); 770 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0)) 771 autohandle = TC_H_MAKE(0x80000000U, 0); 772 if (!qdisc_lookup(dev, autohandle)) 773 return autohandle; 774 cond_resched(); 775 } while (--i > 0); 776 777 return 0; 778 } 779 780 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) 781 { 782 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED; 783 const struct Qdisc_class_ops *cops; 784 unsigned long cl; 785 u32 parentid; 786 bool notify; 787 int drops; 788 789 if (n == 0 && len == 0) 790 return; 791 drops = max_t(int, n, 0); 792 rcu_read_lock(); 793 while ((parentid = sch->parent)) { 794 if (parentid == TC_H_ROOT) 795 break; 796 797 if (sch->flags & TCQ_F_NOPARENT) 798 break; 799 /* Notify parent qdisc only if child qdisc becomes empty. 800 * 801 * If child was empty even before update then backlog 802 * counter is screwed and we skip notification because 803 * parent class is already passive. 804 * 805 * If the original child was offloaded then it is allowed 806 * to be seem as empty, so the parent is notified anyway. 807 */ 808 notify = !sch->q.qlen && !WARN_ON_ONCE(!n && 809 !qdisc_is_offloaded); 810 /* TODO: perform the search on a per txq basis */ 811 sch = qdisc_lookup_rcu(qdisc_dev(sch), TC_H_MAJ(parentid)); 812 if (sch == NULL) { 813 WARN_ON_ONCE(parentid != TC_H_ROOT); 814 break; 815 } 816 cops = sch->ops->cl_ops; 817 if (notify && cops->qlen_notify) { 818 cl = cops->find(sch, parentid); 819 cops->qlen_notify(sch, cl); 820 } 821 sch->q.qlen -= n; 822 sch->qstats.backlog -= len; 823 __qdisc_qstats_drop(sch, drops); 824 } 825 rcu_read_unlock(); 826 } 827 EXPORT_SYMBOL(qdisc_tree_reduce_backlog); 828 829 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type, 830 void *type_data) 831 { 832 struct net_device *dev = qdisc_dev(sch); 833 int err; 834 835 sch->flags &= ~TCQ_F_OFFLOADED; 836 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 837 return 0; 838 839 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); 840 if (err == -EOPNOTSUPP) 841 return 0; 842 843 if (!err) 844 sch->flags |= TCQ_F_OFFLOADED; 845 846 return err; 847 } 848 EXPORT_SYMBOL(qdisc_offload_dump_helper); 849 850 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 851 struct Qdisc *new, struct Qdisc *old, 852 enum tc_setup_type type, void *type_data, 853 struct netlink_ext_ack *extack) 854 { 855 bool any_qdisc_is_offloaded; 856 int err; 857 858 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 859 return; 860 861 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); 862 863 /* Don't report error if the graft is part of destroy operation. */ 864 if (!err || !new || new == &noop_qdisc) 865 return; 866 867 /* Don't report error if the parent, the old child and the new 868 * one are not offloaded. 869 */ 870 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED; 871 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED; 872 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED; 873 874 if (any_qdisc_is_offloaded) 875 NL_SET_ERR_MSG(extack, "Offloading graft operation failed."); 876 } 877 EXPORT_SYMBOL(qdisc_offload_graft_helper); 878 879 void qdisc_offload_query_caps(struct net_device *dev, 880 enum tc_setup_type type, 881 void *caps, size_t caps_len) 882 { 883 const struct net_device_ops *ops = dev->netdev_ops; 884 struct tc_query_caps_base base = { 885 .type = type, 886 .caps = caps, 887 }; 888 889 memset(caps, 0, caps_len); 890 891 if (ops->ndo_setup_tc) 892 ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base); 893 } 894 EXPORT_SYMBOL(qdisc_offload_query_caps); 895 896 static void qdisc_offload_graft_root(struct net_device *dev, 897 struct Qdisc *new, struct Qdisc *old, 898 struct netlink_ext_ack *extack) 899 { 900 struct tc_root_qopt_offload graft_offload = { 901 .command = TC_ROOT_GRAFT, 902 .handle = new ? new->handle : 0, 903 .ingress = (new && new->flags & TCQ_F_INGRESS) || 904 (old && old->flags & TCQ_F_INGRESS), 905 }; 906 907 qdisc_offload_graft_helper(dev, NULL, new, old, 908 TC_SETUP_ROOT_QDISC, &graft_offload, extack); 909 } 910 911 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, 912 u32 portid, u32 seq, u16 flags, int event, 913 struct netlink_ext_ack *extack) 914 { 915 struct gnet_stats_basic_sync __percpu *cpu_bstats = NULL; 916 struct gnet_stats_queue __percpu *cpu_qstats = NULL; 917 struct tcmsg *tcm; 918 struct nlmsghdr *nlh; 919 unsigned char *b = skb_tail_pointer(skb); 920 struct gnet_dump d; 921 struct qdisc_size_table *stab; 922 u32 block_index; 923 __u32 qlen; 924 925 cond_resched(); 926 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 927 if (!nlh) 928 goto out_nlmsg_trim; 929 tcm = nlmsg_data(nlh); 930 tcm->tcm_family = AF_UNSPEC; 931 tcm->tcm__pad1 = 0; 932 tcm->tcm__pad2 = 0; 933 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 934 tcm->tcm_parent = clid; 935 tcm->tcm_handle = q->handle; 936 tcm->tcm_info = refcount_read(&q->refcnt); 937 if (nla_put_string(skb, TCA_KIND, q->ops->id)) 938 goto nla_put_failure; 939 if (q->ops->ingress_block_get) { 940 block_index = q->ops->ingress_block_get(q); 941 if (block_index && 942 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index)) 943 goto nla_put_failure; 944 } 945 if (q->ops->egress_block_get) { 946 block_index = q->ops->egress_block_get(q); 947 if (block_index && 948 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index)) 949 goto nla_put_failure; 950 } 951 if (q->ops->dump && q->ops->dump(q, skb) < 0) 952 goto nla_put_failure; 953 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED))) 954 goto nla_put_failure; 955 qlen = qdisc_qlen_sum(q); 956 957 stab = rtnl_dereference(q->stab); 958 if (stab && qdisc_dump_stab(skb, stab) < 0) 959 goto nla_put_failure; 960 961 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 962 NULL, &d, TCA_PAD) < 0) 963 goto nla_put_failure; 964 965 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0) 966 goto nla_put_failure; 967 968 if (qdisc_is_percpu_stats(q)) { 969 cpu_bstats = q->cpu_bstats; 970 cpu_qstats = q->cpu_qstats; 971 } 972 973 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 || 974 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 || 975 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0) 976 goto nla_put_failure; 977 978 if (gnet_stats_finish_copy(&d) < 0) 979 goto nla_put_failure; 980 981 if (extack && extack->_msg && 982 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 983 goto out_nlmsg_trim; 984 985 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 986 987 return skb->len; 988 989 out_nlmsg_trim: 990 nla_put_failure: 991 nlmsg_trim(skb, b); 992 return -1; 993 } 994 995 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible) 996 { 997 if (q->flags & TCQ_F_BUILTIN) 998 return true; 999 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible) 1000 return true; 1001 1002 return false; 1003 } 1004 1005 static int qdisc_get_notify(struct net *net, struct sk_buff *oskb, 1006 struct nlmsghdr *n, u32 clid, struct Qdisc *q, 1007 struct netlink_ext_ack *extack) 1008 { 1009 struct sk_buff *skb; 1010 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1011 1012 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1013 if (!skb) 1014 return -ENOBUFS; 1015 1016 if (!tc_qdisc_dump_ignore(q, false)) { 1017 if (tc_fill_qdisc(skb, q, clid, portid, n->nlmsg_seq, 0, 1018 RTM_NEWQDISC, extack) < 0) 1019 goto err_out; 1020 } 1021 1022 if (skb->len) 1023 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1024 n->nlmsg_flags & NLM_F_ECHO); 1025 1026 err_out: 1027 kfree_skb(skb); 1028 return -EINVAL; 1029 } 1030 1031 static int qdisc_notify(struct net *net, struct sk_buff *oskb, 1032 struct nlmsghdr *n, u32 clid, 1033 struct Qdisc *old, struct Qdisc *new, 1034 struct netlink_ext_ack *extack) 1035 { 1036 struct sk_buff *skb; 1037 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1038 1039 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) 1040 return 0; 1041 1042 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1043 if (!skb) 1044 return -ENOBUFS; 1045 1046 if (old && !tc_qdisc_dump_ignore(old, false)) { 1047 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq, 1048 0, RTM_DELQDISC, extack) < 0) 1049 goto err_out; 1050 } 1051 if (new && !tc_qdisc_dump_ignore(new, false)) { 1052 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq, 1053 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC, extack) < 0) 1054 goto err_out; 1055 } 1056 1057 if (skb->len) 1058 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1059 n->nlmsg_flags & NLM_F_ECHO); 1060 1061 err_out: 1062 kfree_skb(skb); 1063 return -EINVAL; 1064 } 1065 1066 static void notify_and_destroy(struct net *net, struct sk_buff *skb, 1067 struct nlmsghdr *n, u32 clid, 1068 struct Qdisc *old, struct Qdisc *new, 1069 struct netlink_ext_ack *extack) 1070 { 1071 if (new || old) 1072 qdisc_notify(net, skb, n, clid, old, new, extack); 1073 1074 if (old) 1075 qdisc_put(old); 1076 } 1077 1078 static void qdisc_clear_nolock(struct Qdisc *sch) 1079 { 1080 sch->flags &= ~TCQ_F_NOLOCK; 1081 if (!(sch->flags & TCQ_F_CPUSTATS)) 1082 return; 1083 1084 free_percpu(sch->cpu_bstats); 1085 free_percpu(sch->cpu_qstats); 1086 sch->cpu_bstats = NULL; 1087 sch->cpu_qstats = NULL; 1088 sch->flags &= ~TCQ_F_CPUSTATS; 1089 } 1090 1091 /* Graft qdisc "new" to class "classid" of qdisc "parent" or 1092 * to device "dev". 1093 * 1094 * When appropriate send a netlink notification using 'skb' 1095 * and "n". 1096 * 1097 * On success, destroy old qdisc. 1098 */ 1099 1100 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, 1101 struct sk_buff *skb, struct nlmsghdr *n, u32 classid, 1102 struct Qdisc *new, struct Qdisc *old, 1103 struct netlink_ext_ack *extack) 1104 { 1105 struct Qdisc *q = old; 1106 struct net *net = dev_net(dev); 1107 1108 if (parent == NULL) { 1109 unsigned int i, num_q, ingress; 1110 struct netdev_queue *dev_queue; 1111 1112 ingress = 0; 1113 num_q = dev->num_tx_queues; 1114 if ((q && q->flags & TCQ_F_INGRESS) || 1115 (new && new->flags & TCQ_F_INGRESS)) { 1116 ingress = 1; 1117 dev_queue = dev_ingress_queue(dev); 1118 if (!dev_queue) { 1119 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue"); 1120 return -ENOENT; 1121 } 1122 1123 q = rtnl_dereference(dev_queue->qdisc_sleeping); 1124 1125 /* This is the counterpart of that qdisc_refcount_inc_nz() call in 1126 * __tcf_qdisc_find() for filter requests. 1127 */ 1128 if (!qdisc_refcount_dec_if_one(q)) { 1129 NL_SET_ERR_MSG(extack, 1130 "Current ingress or clsact Qdisc has ongoing filter requests"); 1131 return -EBUSY; 1132 } 1133 } 1134 1135 if (dev->flags & IFF_UP) 1136 dev_deactivate(dev); 1137 1138 qdisc_offload_graft_root(dev, new, old, extack); 1139 1140 if (new && new->ops->attach && !ingress) 1141 goto skip; 1142 1143 if (!ingress) { 1144 for (i = 0; i < num_q; i++) { 1145 dev_queue = netdev_get_tx_queue(dev, i); 1146 old = dev_graft_qdisc(dev_queue, new); 1147 1148 if (new && i > 0) 1149 qdisc_refcount_inc(new); 1150 qdisc_put(old); 1151 } 1152 } else { 1153 old = dev_graft_qdisc(dev_queue, NULL); 1154 1155 /* {ingress,clsact}_destroy() @old before grafting @new to avoid 1156 * unprotected concurrent accesses to net_device::miniq_{in,e}gress 1157 * pointer(s) in mini_qdisc_pair_swap(). 1158 */ 1159 qdisc_notify(net, skb, n, classid, old, new, extack); 1160 qdisc_destroy(old); 1161 1162 dev_graft_qdisc(dev_queue, new); 1163 } 1164 1165 skip: 1166 if (!ingress) { 1167 old = rtnl_dereference(dev->qdisc); 1168 if (new && !new->ops->attach) 1169 qdisc_refcount_inc(new); 1170 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc); 1171 1172 notify_and_destroy(net, skb, n, classid, old, new, extack); 1173 1174 if (new && new->ops->attach) 1175 new->ops->attach(new); 1176 } 1177 1178 if (dev->flags & IFF_UP) 1179 dev_activate(dev); 1180 } else { 1181 const struct Qdisc_class_ops *cops = parent->ops->cl_ops; 1182 unsigned long cl; 1183 int err; 1184 1185 /* Only support running class lockless if parent is lockless */ 1186 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK)) 1187 qdisc_clear_nolock(new); 1188 1189 if (!cops || !cops->graft) 1190 return -EOPNOTSUPP; 1191 1192 cl = cops->find(parent, classid); 1193 if (!cl) { 1194 NL_SET_ERR_MSG(extack, "Specified class not found"); 1195 return -ENOENT; 1196 } 1197 1198 if (new && new->ops == &noqueue_qdisc_ops) { 1199 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class"); 1200 return -EINVAL; 1201 } 1202 1203 if (new && 1204 !(parent->flags & TCQ_F_MQROOT) && 1205 rcu_access_pointer(new->stab)) { 1206 NL_SET_ERR_MSG(extack, "STAB not supported on a non root"); 1207 return -EINVAL; 1208 } 1209 err = cops->graft(parent, cl, new, &old, extack); 1210 if (err) 1211 return err; 1212 notify_and_destroy(net, skb, n, classid, old, new, extack); 1213 } 1214 return 0; 1215 } 1216 1217 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca, 1218 struct netlink_ext_ack *extack) 1219 { 1220 u32 block_index; 1221 1222 if (tca[TCA_INGRESS_BLOCK]) { 1223 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]); 1224 1225 if (!block_index) { 1226 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0"); 1227 return -EINVAL; 1228 } 1229 if (!sch->ops->ingress_block_set) { 1230 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported"); 1231 return -EOPNOTSUPP; 1232 } 1233 sch->ops->ingress_block_set(sch, block_index); 1234 } 1235 if (tca[TCA_EGRESS_BLOCK]) { 1236 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]); 1237 1238 if (!block_index) { 1239 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0"); 1240 return -EINVAL; 1241 } 1242 if (!sch->ops->egress_block_set) { 1243 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported"); 1244 return -EOPNOTSUPP; 1245 } 1246 sch->ops->egress_block_set(sch, block_index); 1247 } 1248 return 0; 1249 } 1250 1251 /* 1252 Allocate and initialize new qdisc. 1253 1254 Parameters are passed via opt. 1255 */ 1256 1257 static struct Qdisc *qdisc_create(struct net_device *dev, 1258 struct netdev_queue *dev_queue, 1259 u32 parent, u32 handle, 1260 struct nlattr **tca, int *errp, 1261 struct netlink_ext_ack *extack) 1262 { 1263 int err; 1264 struct nlattr *kind = tca[TCA_KIND]; 1265 struct Qdisc *sch; 1266 struct Qdisc_ops *ops; 1267 struct qdisc_size_table *stab; 1268 1269 ops = qdisc_lookup_ops(kind); 1270 #ifdef CONFIG_MODULES 1271 if (ops == NULL && kind != NULL) { 1272 char name[IFNAMSIZ]; 1273 if (nla_strscpy(name, kind, IFNAMSIZ) >= 0) { 1274 /* We dropped the RTNL semaphore in order to 1275 * perform the module load. So, even if we 1276 * succeeded in loading the module we have to 1277 * tell the caller to replay the request. We 1278 * indicate this using -EAGAIN. 1279 * We replay the request because the device may 1280 * go away in the mean time. 1281 */ 1282 netdev_unlock_ops(dev); 1283 rtnl_unlock(); 1284 request_module(NET_SCH_ALIAS_PREFIX "%s", name); 1285 rtnl_lock(); 1286 netdev_lock_ops(dev); 1287 ops = qdisc_lookup_ops(kind); 1288 if (ops != NULL) { 1289 /* We will try again qdisc_lookup_ops, 1290 * so don't keep a reference. 1291 */ 1292 module_put(ops->owner); 1293 err = -EAGAIN; 1294 goto err_out; 1295 } 1296 } 1297 } 1298 #endif 1299 1300 err = -ENOENT; 1301 if (!ops) { 1302 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown"); 1303 goto err_out; 1304 } 1305 1306 sch = qdisc_alloc(dev_queue, ops, extack); 1307 if (IS_ERR(sch)) { 1308 err = PTR_ERR(sch); 1309 goto err_out2; 1310 } 1311 1312 sch->parent = parent; 1313 1314 if (handle == TC_H_INGRESS) { 1315 if (!(sch->flags & TCQ_F_INGRESS)) { 1316 NL_SET_ERR_MSG(extack, 1317 "Specified parent ID is reserved for ingress and clsact Qdiscs"); 1318 err = -EINVAL; 1319 goto err_out3; 1320 } 1321 handle = TC_H_MAKE(TC_H_INGRESS, 0); 1322 } else { 1323 if (handle == 0) { 1324 handle = qdisc_alloc_handle(dev); 1325 if (handle == 0) { 1326 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded"); 1327 err = -ENOSPC; 1328 goto err_out3; 1329 } 1330 } 1331 if (!netif_is_multiqueue(dev)) 1332 sch->flags |= TCQ_F_ONETXQUEUE; 1333 } 1334 1335 sch->handle = handle; 1336 1337 /* This exist to keep backward compatible with a userspace 1338 * loophole, what allowed userspace to get IFF_NO_QUEUE 1339 * facility on older kernels by setting tx_queue_len=0 (prior 1340 * to qdisc init), and then forgot to reinit tx_queue_len 1341 * before again attaching a qdisc. 1342 */ 1343 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) { 1344 WRITE_ONCE(dev->tx_queue_len, DEFAULT_TX_QUEUE_LEN); 1345 netdev_info(dev, "Caught tx_queue_len zero misconfig\n"); 1346 } 1347 1348 err = qdisc_block_indexes_set(sch, tca, extack); 1349 if (err) 1350 goto err_out3; 1351 1352 if (tca[TCA_STAB]) { 1353 stab = qdisc_get_stab(tca[TCA_STAB], extack); 1354 if (IS_ERR(stab)) { 1355 err = PTR_ERR(stab); 1356 goto err_out3; 1357 } 1358 rcu_assign_pointer(sch->stab, stab); 1359 } 1360 1361 if (ops->init) { 1362 err = ops->init(sch, tca[TCA_OPTIONS], extack); 1363 if (err != 0) 1364 goto err_out4; 1365 } 1366 1367 if (tca[TCA_RATE]) { 1368 err = -EOPNOTSUPP; 1369 if (sch->flags & TCQ_F_MQROOT) { 1370 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc"); 1371 goto err_out4; 1372 } 1373 1374 err = gen_new_estimator(&sch->bstats, 1375 sch->cpu_bstats, 1376 &sch->rate_est, 1377 NULL, 1378 true, 1379 tca[TCA_RATE]); 1380 if (err) { 1381 NL_SET_ERR_MSG(extack, "Failed to generate new estimator"); 1382 goto err_out4; 1383 } 1384 } 1385 1386 qdisc_hash_add(sch, false); 1387 trace_qdisc_create(ops, dev, parent); 1388 1389 return sch; 1390 1391 err_out4: 1392 /* Even if ops->init() failed, we call ops->destroy() 1393 * like qdisc_create_dflt(). 1394 */ 1395 if (ops->destroy) 1396 ops->destroy(sch); 1397 qdisc_put_stab(rtnl_dereference(sch->stab)); 1398 err_out3: 1399 lockdep_unregister_key(&sch->root_lock_key); 1400 netdev_put(dev, &sch->dev_tracker); 1401 qdisc_free(sch); 1402 err_out2: 1403 module_put(ops->owner); 1404 err_out: 1405 *errp = err; 1406 return NULL; 1407 } 1408 1409 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca, 1410 struct netlink_ext_ack *extack) 1411 { 1412 struct qdisc_size_table *ostab, *stab = NULL; 1413 int err = 0; 1414 1415 if (tca[TCA_OPTIONS]) { 1416 if (!sch->ops->change) { 1417 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc"); 1418 return -EINVAL; 1419 } 1420 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) { 1421 NL_SET_ERR_MSG(extack, "Change of blocks is not supported"); 1422 return -EOPNOTSUPP; 1423 } 1424 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack); 1425 if (err) 1426 return err; 1427 } 1428 1429 if (tca[TCA_STAB]) { 1430 stab = qdisc_get_stab(tca[TCA_STAB], extack); 1431 if (IS_ERR(stab)) 1432 return PTR_ERR(stab); 1433 } 1434 1435 ostab = rtnl_dereference(sch->stab); 1436 rcu_assign_pointer(sch->stab, stab); 1437 qdisc_put_stab(ostab); 1438 1439 if (tca[TCA_RATE]) { 1440 /* NB: ignores errors from replace_estimator 1441 because change can't be undone. */ 1442 if (sch->flags & TCQ_F_MQROOT) 1443 goto out; 1444 gen_replace_estimator(&sch->bstats, 1445 sch->cpu_bstats, 1446 &sch->rate_est, 1447 NULL, 1448 true, 1449 tca[TCA_RATE]); 1450 } 1451 out: 1452 return 0; 1453 } 1454 1455 struct check_loop_arg { 1456 struct qdisc_walker w; 1457 struct Qdisc *p; 1458 int depth; 1459 }; 1460 1461 static int check_loop_fn(struct Qdisc *q, unsigned long cl, 1462 struct qdisc_walker *w); 1463 1464 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth) 1465 { 1466 struct check_loop_arg arg; 1467 1468 if (q->ops->cl_ops == NULL) 1469 return 0; 1470 1471 arg.w.stop = arg.w.skip = arg.w.count = 0; 1472 arg.w.fn = check_loop_fn; 1473 arg.depth = depth; 1474 arg.p = p; 1475 q->ops->cl_ops->walk(q, &arg.w); 1476 return arg.w.stop ? -ELOOP : 0; 1477 } 1478 1479 static int 1480 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w) 1481 { 1482 struct Qdisc *leaf; 1483 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1484 struct check_loop_arg *arg = (struct check_loop_arg *)w; 1485 1486 leaf = cops->leaf(q, cl); 1487 if (leaf) { 1488 if (leaf == arg->p || arg->depth > 7) 1489 return -ELOOP; 1490 return check_loop(leaf, arg->p, arg->depth + 1); 1491 } 1492 return 0; 1493 } 1494 1495 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = { 1496 [TCA_KIND] = { .type = NLA_STRING }, 1497 [TCA_RATE] = { .type = NLA_BINARY, 1498 .len = sizeof(struct tc_estimator) }, 1499 [TCA_STAB] = { .type = NLA_NESTED }, 1500 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG }, 1501 [TCA_CHAIN] = { .type = NLA_U32 }, 1502 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 }, 1503 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 }, 1504 }; 1505 1506 /* 1507 * Delete/get qdisc. 1508 */ 1509 1510 static int __tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1511 struct netlink_ext_ack *extack, 1512 struct net_device *dev, 1513 struct nlattr *tca[TCA_MAX + 1], 1514 struct tcmsg *tcm) 1515 { 1516 struct net *net = sock_net(skb->sk); 1517 struct Qdisc *q = NULL; 1518 struct Qdisc *p = NULL; 1519 u32 clid; 1520 int err; 1521 1522 clid = tcm->tcm_parent; 1523 if (clid) { 1524 if (clid != TC_H_ROOT) { 1525 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) { 1526 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1527 if (!p) { 1528 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid"); 1529 return -ENOENT; 1530 } 1531 q = qdisc_leaf(p, clid); 1532 } else if (dev_ingress_queue(dev)) { 1533 q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping); 1534 } 1535 } else { 1536 q = rtnl_dereference(dev->qdisc); 1537 } 1538 if (!q) { 1539 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device"); 1540 return -ENOENT; 1541 } 1542 1543 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) { 1544 NL_SET_ERR_MSG(extack, "Invalid handle"); 1545 return -EINVAL; 1546 } 1547 } else { 1548 q = qdisc_lookup(dev, tcm->tcm_handle); 1549 if (!q) { 1550 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle"); 1551 return -ENOENT; 1552 } 1553 } 1554 1555 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1556 NL_SET_ERR_MSG(extack, "Invalid qdisc name: must match existing qdisc"); 1557 return -EINVAL; 1558 } 1559 1560 if (n->nlmsg_type == RTM_DELQDISC) { 1561 if (!clid) { 1562 NL_SET_ERR_MSG(extack, "Classid cannot be zero"); 1563 return -EINVAL; 1564 } 1565 if (q->handle == 0) { 1566 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero"); 1567 return -ENOENT; 1568 } 1569 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack); 1570 if (err != 0) 1571 return err; 1572 } else { 1573 qdisc_get_notify(net, skb, n, clid, q, NULL); 1574 } 1575 return 0; 1576 } 1577 1578 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1579 struct netlink_ext_ack *extack) 1580 { 1581 struct net *net = sock_net(skb->sk); 1582 struct tcmsg *tcm = nlmsg_data(n); 1583 struct nlattr *tca[TCA_MAX + 1]; 1584 struct net_device *dev; 1585 int err; 1586 1587 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 1588 rtm_tca_policy, extack); 1589 if (err < 0) 1590 return err; 1591 1592 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1593 if (!dev) 1594 return -ENODEV; 1595 1596 netdev_lock_ops(dev); 1597 err = __tc_get_qdisc(skb, n, extack, dev, tca, tcm); 1598 netdev_unlock_ops(dev); 1599 1600 return err; 1601 } 1602 1603 static bool req_create_or_replace(struct nlmsghdr *n) 1604 { 1605 return (n->nlmsg_flags & NLM_F_CREATE && 1606 n->nlmsg_flags & NLM_F_REPLACE); 1607 } 1608 1609 static bool req_create_exclusive(struct nlmsghdr *n) 1610 { 1611 return (n->nlmsg_flags & NLM_F_CREATE && 1612 n->nlmsg_flags & NLM_F_EXCL); 1613 } 1614 1615 static bool req_change(struct nlmsghdr *n) 1616 { 1617 return (!(n->nlmsg_flags & NLM_F_CREATE) && 1618 !(n->nlmsg_flags & NLM_F_REPLACE) && 1619 !(n->nlmsg_flags & NLM_F_EXCL)); 1620 } 1621 1622 static int __tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1623 struct netlink_ext_ack *extack, 1624 struct net_device *dev, 1625 struct nlattr *tca[TCA_MAX + 1], 1626 struct tcmsg *tcm, 1627 bool *replay) 1628 { 1629 struct Qdisc *q = NULL; 1630 struct Qdisc *p = NULL; 1631 u32 clid; 1632 int err; 1633 1634 clid = tcm->tcm_parent; 1635 1636 if (clid) { 1637 if (clid != TC_H_ROOT) { 1638 if (clid != TC_H_INGRESS) { 1639 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1640 if (!p) { 1641 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc"); 1642 return -ENOENT; 1643 } 1644 q = qdisc_leaf(p, clid); 1645 } else if (dev_ingress_queue_create(dev)) { 1646 q = rtnl_dereference(dev_ingress_queue(dev)->qdisc_sleeping); 1647 } 1648 } else { 1649 q = rtnl_dereference(dev->qdisc); 1650 } 1651 1652 /* It may be default qdisc, ignore it */ 1653 if (q && q->handle == 0) 1654 q = NULL; 1655 1656 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) { 1657 if (tcm->tcm_handle) { 1658 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) { 1659 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override"); 1660 return -EEXIST; 1661 } 1662 if (TC_H_MIN(tcm->tcm_handle)) { 1663 NL_SET_ERR_MSG(extack, "Invalid minor handle"); 1664 return -EINVAL; 1665 } 1666 q = qdisc_lookup(dev, tcm->tcm_handle); 1667 if (!q) 1668 goto create_n_graft; 1669 if (q->parent != tcm->tcm_parent) { 1670 NL_SET_ERR_MSG(extack, "Cannot move an existing qdisc to a different parent"); 1671 return -EINVAL; 1672 } 1673 if (n->nlmsg_flags & NLM_F_EXCL) { 1674 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override"); 1675 return -EEXIST; 1676 } 1677 if (tca[TCA_KIND] && 1678 nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1679 NL_SET_ERR_MSG(extack, "Invalid qdisc name: must match existing qdisc"); 1680 return -EINVAL; 1681 } 1682 if (q->flags & TCQ_F_INGRESS) { 1683 NL_SET_ERR_MSG(extack, 1684 "Cannot regraft ingress or clsact Qdiscs"); 1685 return -EINVAL; 1686 } 1687 if (q == p || 1688 (p && check_loop(q, p, 0))) { 1689 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected"); 1690 return -ELOOP; 1691 } 1692 if (clid == TC_H_INGRESS) { 1693 NL_SET_ERR_MSG(extack, "Ingress cannot graft directly"); 1694 return -EINVAL; 1695 } 1696 qdisc_refcount_inc(q); 1697 goto graft; 1698 } else { 1699 if (!q) 1700 goto create_n_graft; 1701 1702 /* This magic test requires explanation. 1703 * 1704 * We know, that some child q is already 1705 * attached to this parent and have choice: 1706 * 1) change it or 2) create/graft new one. 1707 * If the requested qdisc kind is different 1708 * than the existing one, then we choose graft. 1709 * If they are the same then this is "change" 1710 * operation - just let it fallthrough.. 1711 * 1712 * 1. We are allowed to create/graft only 1713 * if the request is explicitly stating 1714 * "please create if it doesn't exist". 1715 * 1716 * 2. If the request is to exclusive create 1717 * then the qdisc tcm_handle is not expected 1718 * to exist, so that we choose create/graft too. 1719 * 1720 * 3. The last case is when no flags are set. 1721 * This will happen when for example tc 1722 * utility issues a "change" command. 1723 * Alas, it is sort of hole in API, we 1724 * cannot decide what to do unambiguously. 1725 * For now we select create/graft. 1726 */ 1727 if (tca[TCA_KIND] && 1728 nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1729 if (req_create_or_replace(n) || 1730 req_create_exclusive(n)) 1731 goto create_n_graft; 1732 else if (req_change(n)) 1733 goto create_n_graft2; 1734 } 1735 } 1736 } 1737 } else { 1738 if (!tcm->tcm_handle) { 1739 NL_SET_ERR_MSG(extack, "Handle cannot be zero"); 1740 return -EINVAL; 1741 } 1742 q = qdisc_lookup(dev, tcm->tcm_handle); 1743 } 1744 1745 /* Change qdisc parameters */ 1746 if (!q) { 1747 NL_SET_ERR_MSG(extack, "Specified qdisc not found"); 1748 return -ENOENT; 1749 } 1750 if (n->nlmsg_flags & NLM_F_EXCL) { 1751 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify"); 1752 return -EEXIST; 1753 } 1754 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1755 NL_SET_ERR_MSG(extack, "Invalid qdisc name: must match existing qdisc"); 1756 return -EINVAL; 1757 } 1758 err = qdisc_change(q, tca, extack); 1759 if (err == 0) 1760 qdisc_notify(sock_net(skb->sk), skb, n, clid, NULL, q, extack); 1761 return err; 1762 1763 create_n_graft: 1764 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 1765 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag"); 1766 return -ENOENT; 1767 } 1768 create_n_graft2: 1769 if (clid == TC_H_INGRESS) { 1770 if (dev_ingress_queue(dev)) { 1771 q = qdisc_create(dev, dev_ingress_queue(dev), 1772 tcm->tcm_parent, tcm->tcm_parent, 1773 tca, &err, extack); 1774 } else { 1775 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device"); 1776 err = -ENOENT; 1777 } 1778 } else { 1779 struct netdev_queue *dev_queue; 1780 1781 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue) 1782 dev_queue = p->ops->cl_ops->select_queue(p, tcm); 1783 else if (p) 1784 dev_queue = p->dev_queue; 1785 else 1786 dev_queue = netdev_get_tx_queue(dev, 0); 1787 1788 q = qdisc_create(dev, dev_queue, 1789 tcm->tcm_parent, tcm->tcm_handle, 1790 tca, &err, extack); 1791 } 1792 if (q == NULL) { 1793 if (err == -EAGAIN) { 1794 *replay = true; 1795 return 0; 1796 } 1797 return err; 1798 } 1799 1800 graft: 1801 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack); 1802 if (err) { 1803 if (q) 1804 qdisc_put(q); 1805 return err; 1806 } 1807 1808 return 0; 1809 } 1810 1811 /* 1812 * Create/change qdisc. 1813 */ 1814 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1815 struct netlink_ext_ack *extack) 1816 { 1817 struct net *net = sock_net(skb->sk); 1818 struct nlattr *tca[TCA_MAX + 1]; 1819 struct net_device *dev; 1820 struct tcmsg *tcm; 1821 bool replay; 1822 int err; 1823 1824 replay: 1825 /* Reinit, just in case something touches this. */ 1826 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 1827 rtm_tca_policy, extack); 1828 if (err < 0) 1829 return err; 1830 1831 tcm = nlmsg_data(n); 1832 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1833 if (!dev) 1834 return -ENODEV; 1835 1836 replay = false; 1837 netdev_lock_ops(dev); 1838 err = __tc_modify_qdisc(skb, n, extack, dev, tca, tcm, &replay); 1839 netdev_unlock_ops(dev); 1840 if (replay) 1841 goto replay; 1842 1843 return err; 1844 } 1845 1846 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, 1847 struct netlink_callback *cb, 1848 int *q_idx_p, int s_q_idx, bool recur, 1849 bool dump_invisible) 1850 { 1851 int ret = 0, q_idx = *q_idx_p; 1852 struct Qdisc *q; 1853 int b; 1854 1855 if (!root) 1856 return 0; 1857 1858 q = root; 1859 if (q_idx < s_q_idx) { 1860 q_idx++; 1861 } else { 1862 if (!tc_qdisc_dump_ignore(q, dump_invisible) && 1863 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid, 1864 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1865 RTM_NEWQDISC, NULL) <= 0) 1866 goto done; 1867 q_idx++; 1868 } 1869 1870 /* If dumping singletons, there is no qdisc_dev(root) and the singleton 1871 * itself has already been dumped. 1872 * 1873 * If we've already dumped the top-level (ingress) qdisc above and the global 1874 * qdisc hashtable, we don't want to hit it again 1875 */ 1876 if (!qdisc_dev(root) || !recur) 1877 goto out; 1878 1879 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) { 1880 if (q_idx < s_q_idx) { 1881 q_idx++; 1882 continue; 1883 } 1884 if (!tc_qdisc_dump_ignore(q, dump_invisible) && 1885 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid, 1886 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1887 RTM_NEWQDISC, NULL) <= 0) 1888 goto done; 1889 q_idx++; 1890 } 1891 1892 out: 1893 *q_idx_p = q_idx; 1894 return ret; 1895 done: 1896 ret = -1; 1897 goto out; 1898 } 1899 1900 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) 1901 { 1902 struct net *net = sock_net(skb->sk); 1903 int idx, q_idx; 1904 int s_idx, s_q_idx; 1905 struct net_device *dev; 1906 const struct nlmsghdr *nlh = cb->nlh; 1907 struct nlattr *tca[TCA_MAX + 1]; 1908 int err; 1909 1910 s_idx = cb->args[0]; 1911 s_q_idx = q_idx = cb->args[1]; 1912 1913 idx = 0; 1914 ASSERT_RTNL(); 1915 1916 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX, 1917 rtm_tca_policy, cb->extack); 1918 if (err < 0) 1919 return err; 1920 1921 for_each_netdev(net, dev) { 1922 struct netdev_queue *dev_queue; 1923 1924 if (idx < s_idx) 1925 goto cont; 1926 if (idx > s_idx) 1927 s_q_idx = 0; 1928 q_idx = 0; 1929 1930 netdev_lock_ops(dev); 1931 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc), 1932 skb, cb, &q_idx, s_q_idx, 1933 true, tca[TCA_DUMP_INVISIBLE]) < 0) { 1934 netdev_unlock_ops(dev); 1935 goto done; 1936 } 1937 1938 dev_queue = dev_ingress_queue(dev); 1939 if (dev_queue && 1940 tc_dump_qdisc_root(rtnl_dereference(dev_queue->qdisc_sleeping), 1941 skb, cb, &q_idx, s_q_idx, false, 1942 tca[TCA_DUMP_INVISIBLE]) < 0) { 1943 netdev_unlock_ops(dev); 1944 goto done; 1945 } 1946 netdev_unlock_ops(dev); 1947 1948 cont: 1949 idx++; 1950 } 1951 1952 done: 1953 cb->args[0] = idx; 1954 cb->args[1] = q_idx; 1955 1956 return skb->len; 1957 } 1958 1959 1960 1961 /************************************************ 1962 * Traffic classes manipulation. * 1963 ************************************************/ 1964 1965 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q, 1966 unsigned long cl, u32 portid, u32 seq, u16 flags, 1967 int event, struct netlink_ext_ack *extack) 1968 { 1969 struct tcmsg *tcm; 1970 struct nlmsghdr *nlh; 1971 unsigned char *b = skb_tail_pointer(skb); 1972 struct gnet_dump d; 1973 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops; 1974 1975 cond_resched(); 1976 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 1977 if (!nlh) 1978 goto out_nlmsg_trim; 1979 tcm = nlmsg_data(nlh); 1980 tcm->tcm_family = AF_UNSPEC; 1981 tcm->tcm__pad1 = 0; 1982 tcm->tcm__pad2 = 0; 1983 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1984 tcm->tcm_parent = q->handle; 1985 tcm->tcm_handle = q->handle; 1986 tcm->tcm_info = 0; 1987 if (nla_put_string(skb, TCA_KIND, q->ops->id)) 1988 goto nla_put_failure; 1989 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0) 1990 goto nla_put_failure; 1991 1992 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 1993 NULL, &d, TCA_PAD) < 0) 1994 goto nla_put_failure; 1995 1996 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0) 1997 goto nla_put_failure; 1998 1999 if (gnet_stats_finish_copy(&d) < 0) 2000 goto nla_put_failure; 2001 2002 if (extack && extack->_msg && 2003 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 2004 goto out_nlmsg_trim; 2005 2006 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2007 2008 return skb->len; 2009 2010 out_nlmsg_trim: 2011 nla_put_failure: 2012 nlmsg_trim(skb, b); 2013 return -1; 2014 } 2015 2016 static int tclass_notify(struct net *net, struct sk_buff *oskb, 2017 struct nlmsghdr *n, struct Qdisc *q, 2018 unsigned long cl, int event, struct netlink_ext_ack *extack) 2019 { 2020 struct sk_buff *skb; 2021 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2022 2023 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) 2024 return 0; 2025 2026 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2027 if (!skb) 2028 return -ENOBUFS; 2029 2030 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event, extack) < 0) { 2031 kfree_skb(skb); 2032 return -EINVAL; 2033 } 2034 2035 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2036 n->nlmsg_flags & NLM_F_ECHO); 2037 } 2038 2039 static int tclass_get_notify(struct net *net, struct sk_buff *oskb, 2040 struct nlmsghdr *n, struct Qdisc *q, 2041 unsigned long cl, struct netlink_ext_ack *extack) 2042 { 2043 struct sk_buff *skb; 2044 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2045 2046 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2047 if (!skb) 2048 return -ENOBUFS; 2049 2050 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, RTM_NEWTCLASS, 2051 extack) < 0) { 2052 kfree_skb(skb); 2053 return -EINVAL; 2054 } 2055 2056 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2057 n->nlmsg_flags & NLM_F_ECHO); 2058 } 2059 2060 static int tclass_del_notify(struct net *net, 2061 const struct Qdisc_class_ops *cops, 2062 struct sk_buff *oskb, struct nlmsghdr *n, 2063 struct Qdisc *q, unsigned long cl, 2064 struct netlink_ext_ack *extack) 2065 { 2066 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2067 struct sk_buff *skb; 2068 int err = 0; 2069 2070 if (!cops->delete) 2071 return -EOPNOTSUPP; 2072 2073 if (rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 2074 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2075 if (!skb) 2076 return -ENOBUFS; 2077 2078 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, 2079 RTM_DELTCLASS, extack) < 0) { 2080 kfree_skb(skb); 2081 return -EINVAL; 2082 } 2083 } else { 2084 skb = NULL; 2085 } 2086 2087 err = cops->delete(q, cl, extack); 2088 if (err) { 2089 kfree_skb(skb); 2090 return err; 2091 } 2092 2093 err = rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2094 n->nlmsg_flags & NLM_F_ECHO); 2095 return err; 2096 } 2097 2098 #ifdef CONFIG_NET_CLS 2099 2100 struct tcf_bind_args { 2101 struct tcf_walker w; 2102 unsigned long base; 2103 unsigned long cl; 2104 u32 classid; 2105 }; 2106 2107 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 2108 { 2109 struct tcf_bind_args *a = (void *)arg; 2110 2111 if (n && tp->ops->bind_class) { 2112 struct Qdisc *q = tcf_block_q(tp->chain->block); 2113 2114 sch_tree_lock(q); 2115 tp->ops->bind_class(n, a->classid, a->cl, q, a->base); 2116 sch_tree_unlock(q); 2117 } 2118 return 0; 2119 } 2120 2121 struct tc_bind_class_args { 2122 struct qdisc_walker w; 2123 unsigned long new_cl; 2124 u32 portid; 2125 u32 clid; 2126 }; 2127 2128 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl, 2129 struct qdisc_walker *w) 2130 { 2131 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w; 2132 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 2133 struct tcf_block *block; 2134 struct tcf_chain *chain; 2135 2136 block = cops->tcf_block(q, cl, NULL); 2137 if (!block) 2138 return 0; 2139 for (chain = tcf_get_next_chain(block, NULL); 2140 chain; 2141 chain = tcf_get_next_chain(block, chain)) { 2142 struct tcf_proto *tp; 2143 2144 for (tp = tcf_get_next_proto(chain, NULL); 2145 tp; tp = tcf_get_next_proto(chain, tp)) { 2146 struct tcf_bind_args arg = {}; 2147 2148 arg.w.fn = tcf_node_bind; 2149 arg.classid = a->clid; 2150 arg.base = cl; 2151 arg.cl = a->new_cl; 2152 tp->ops->walk(tp, &arg.w, true); 2153 } 2154 } 2155 2156 return 0; 2157 } 2158 2159 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid, 2160 unsigned long new_cl) 2161 { 2162 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 2163 struct tc_bind_class_args args = {}; 2164 2165 if (!cops->tcf_block) 2166 return; 2167 args.portid = portid; 2168 args.clid = clid; 2169 args.new_cl = new_cl; 2170 args.w.fn = tc_bind_class_walker; 2171 q->ops->cl_ops->walk(q, &args.w); 2172 } 2173 2174 #else 2175 2176 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid, 2177 unsigned long new_cl) 2178 { 2179 } 2180 2181 #endif 2182 2183 static int __tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, 2184 struct netlink_ext_ack *extack, 2185 struct net_device *dev, 2186 struct nlattr *tca[TCA_MAX + 1], 2187 struct tcmsg *tcm) 2188 { 2189 struct net *net = sock_net(skb->sk); 2190 const struct Qdisc_class_ops *cops; 2191 struct Qdisc *q = NULL; 2192 unsigned long cl = 0; 2193 unsigned long new_cl; 2194 u32 portid; 2195 u32 clid; 2196 u32 qid; 2197 int err; 2198 2199 /* 2200 parent == TC_H_UNSPEC - unspecified parent. 2201 parent == TC_H_ROOT - class is root, which has no parent. 2202 parent == X:0 - parent is root class. 2203 parent == X:Y - parent is a node in hierarchy. 2204 parent == 0:Y - parent is X:Y, where X:0 is qdisc. 2205 2206 handle == 0:0 - generate handle from kernel pool. 2207 handle == 0:Y - class is X:Y, where X:0 is qdisc. 2208 handle == X:Y - clear. 2209 handle == X:0 - root class. 2210 */ 2211 2212 /* Step 1. Determine qdisc handle X:0 */ 2213 2214 portid = tcm->tcm_parent; 2215 clid = tcm->tcm_handle; 2216 qid = TC_H_MAJ(clid); 2217 2218 if (portid != TC_H_ROOT) { 2219 u32 qid1 = TC_H_MAJ(portid); 2220 2221 if (qid && qid1) { 2222 /* If both majors are known, they must be identical. */ 2223 if (qid != qid1) 2224 return -EINVAL; 2225 } else if (qid1) { 2226 qid = qid1; 2227 } else if (qid == 0) 2228 qid = rtnl_dereference(dev->qdisc)->handle; 2229 2230 /* Now qid is genuine qdisc handle consistent 2231 * both with parent and child. 2232 * 2233 * TC_H_MAJ(portid) still may be unspecified, complete it now. 2234 */ 2235 if (portid) 2236 portid = TC_H_MAKE(qid, portid); 2237 } else { 2238 if (qid == 0) 2239 qid = rtnl_dereference(dev->qdisc)->handle; 2240 } 2241 2242 /* OK. Locate qdisc */ 2243 q = qdisc_lookup(dev, qid); 2244 if (!q) 2245 return -ENOENT; 2246 2247 /* An check that it supports classes */ 2248 cops = q->ops->cl_ops; 2249 if (cops == NULL) 2250 return -EINVAL; 2251 2252 /* Now try to get class */ 2253 if (clid == 0) { 2254 if (portid == TC_H_ROOT) 2255 clid = qid; 2256 } else 2257 clid = TC_H_MAKE(qid, clid); 2258 2259 if (clid) 2260 cl = cops->find(q, clid); 2261 2262 if (cl == 0) { 2263 err = -ENOENT; 2264 if (n->nlmsg_type != RTM_NEWTCLASS || 2265 !(n->nlmsg_flags & NLM_F_CREATE)) 2266 goto out; 2267 } else { 2268 switch (n->nlmsg_type) { 2269 case RTM_NEWTCLASS: 2270 err = -EEXIST; 2271 if (n->nlmsg_flags & NLM_F_EXCL) 2272 goto out; 2273 break; 2274 case RTM_DELTCLASS: 2275 err = tclass_del_notify(net, cops, skb, n, q, cl, extack); 2276 /* Unbind the class with flilters with 0 */ 2277 tc_bind_tclass(q, portid, clid, 0); 2278 goto out; 2279 case RTM_GETTCLASS: 2280 err = tclass_get_notify(net, skb, n, q, cl, extack); 2281 goto out; 2282 default: 2283 err = -EINVAL; 2284 goto out; 2285 } 2286 } 2287 2288 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) { 2289 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes"); 2290 return -EOPNOTSUPP; 2291 } 2292 2293 new_cl = cl; 2294 err = -EOPNOTSUPP; 2295 if (cops->change) 2296 err = cops->change(q, clid, portid, tca, &new_cl, extack); 2297 if (err == 0) { 2298 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS, extack); 2299 /* We just create a new class, need to do reverse binding. */ 2300 if (cl != new_cl) 2301 tc_bind_tclass(q, portid, clid, new_cl); 2302 } 2303 out: 2304 return err; 2305 } 2306 2307 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, 2308 struct netlink_ext_ack *extack) 2309 { 2310 struct net *net = sock_net(skb->sk); 2311 struct tcmsg *tcm = nlmsg_data(n); 2312 struct nlattr *tca[TCA_MAX + 1]; 2313 struct net_device *dev; 2314 int err; 2315 2316 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 2317 rtm_tca_policy, extack); 2318 if (err < 0) 2319 return err; 2320 2321 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2322 if (!dev) 2323 return -ENODEV; 2324 2325 netdev_lock_ops(dev); 2326 err = __tc_ctl_tclass(skb, n, extack, dev, tca, tcm); 2327 netdev_unlock_ops(dev); 2328 2329 return err; 2330 } 2331 2332 struct qdisc_dump_args { 2333 struct qdisc_walker w; 2334 struct sk_buff *skb; 2335 struct netlink_callback *cb; 2336 }; 2337 2338 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, 2339 struct qdisc_walker *arg) 2340 { 2341 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg; 2342 2343 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid, 2344 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2345 RTM_NEWTCLASS, NULL); 2346 } 2347 2348 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb, 2349 struct tcmsg *tcm, struct netlink_callback *cb, 2350 int *t_p, int s_t) 2351 { 2352 struct qdisc_dump_args arg; 2353 2354 if (tc_qdisc_dump_ignore(q, false) || 2355 *t_p < s_t || !q->ops->cl_ops || 2356 (tcm->tcm_parent && 2357 TC_H_MAJ(tcm->tcm_parent) != q->handle)) { 2358 (*t_p)++; 2359 return 0; 2360 } 2361 if (*t_p > s_t) 2362 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0])); 2363 arg.w.fn = qdisc_class_dump; 2364 arg.skb = skb; 2365 arg.cb = cb; 2366 arg.w.stop = 0; 2367 arg.w.skip = cb->args[1]; 2368 arg.w.count = 0; 2369 q->ops->cl_ops->walk(q, &arg.w); 2370 cb->args[1] = arg.w.count; 2371 if (arg.w.stop) 2372 return -1; 2373 (*t_p)++; 2374 return 0; 2375 } 2376 2377 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, 2378 struct tcmsg *tcm, struct netlink_callback *cb, 2379 int *t_p, int s_t, bool recur) 2380 { 2381 struct Qdisc *q; 2382 int b; 2383 2384 if (!root) 2385 return 0; 2386 2387 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0) 2388 return -1; 2389 2390 if (!qdisc_dev(root) || !recur) 2391 return 0; 2392 2393 if (tcm->tcm_parent) { 2394 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent)); 2395 if (q && q != root && 2396 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 2397 return -1; 2398 return 0; 2399 } 2400 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) { 2401 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 2402 return -1; 2403 } 2404 2405 return 0; 2406 } 2407 2408 static int __tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb, 2409 struct tcmsg *tcm, struct net_device *dev) 2410 { 2411 struct netdev_queue *dev_queue; 2412 int t, s_t; 2413 2414 s_t = cb->args[0]; 2415 t = 0; 2416 2417 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc), 2418 skb, tcm, cb, &t, s_t, true) < 0) 2419 goto done; 2420 2421 dev_queue = dev_ingress_queue(dev); 2422 if (dev_queue && 2423 tc_dump_tclass_root(rtnl_dereference(dev_queue->qdisc_sleeping), 2424 skb, tcm, cb, &t, s_t, false) < 0) 2425 goto done; 2426 2427 done: 2428 cb->args[0] = t; 2429 2430 return skb->len; 2431 } 2432 2433 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) 2434 { 2435 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2436 struct net *net = sock_net(skb->sk); 2437 struct net_device *dev; 2438 int err; 2439 2440 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2441 return 0; 2442 2443 dev = dev_get_by_index(net, tcm->tcm_ifindex); 2444 if (!dev) 2445 return 0; 2446 2447 netdev_lock_ops(dev); 2448 err = __tc_dump_tclass(skb, cb, tcm, dev); 2449 netdev_unlock_ops(dev); 2450 2451 dev_put(dev); 2452 2453 return err; 2454 } 2455 2456 #ifdef CONFIG_PROC_FS 2457 static int psched_show(struct seq_file *seq, void *v) 2458 { 2459 seq_printf(seq, "%08x %08x %08x %08x\n", 2460 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1), 2461 1000000, 2462 (u32)NSEC_PER_SEC / hrtimer_resolution); 2463 2464 return 0; 2465 } 2466 2467 static int __net_init psched_net_init(struct net *net) 2468 { 2469 struct proc_dir_entry *e; 2470 2471 e = proc_create_single("psched", 0, net->proc_net, psched_show); 2472 if (e == NULL) 2473 return -ENOMEM; 2474 2475 return 0; 2476 } 2477 2478 static void __net_exit psched_net_exit(struct net *net) 2479 { 2480 remove_proc_entry("psched", net->proc_net); 2481 } 2482 #else 2483 static int __net_init psched_net_init(struct net *net) 2484 { 2485 return 0; 2486 } 2487 2488 static void __net_exit psched_net_exit(struct net *net) 2489 { 2490 } 2491 #endif 2492 2493 static struct pernet_operations psched_net_ops = { 2494 .init = psched_net_init, 2495 .exit = psched_net_exit, 2496 }; 2497 2498 #if IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) 2499 DEFINE_STATIC_KEY_FALSE(tc_skip_wrapper); 2500 #endif 2501 2502 static const struct rtnl_msg_handler psched_rtnl_msg_handlers[] __initconst = { 2503 {.msgtype = RTM_NEWQDISC, .doit = tc_modify_qdisc}, 2504 {.msgtype = RTM_DELQDISC, .doit = tc_get_qdisc}, 2505 {.msgtype = RTM_GETQDISC, .doit = tc_get_qdisc, 2506 .dumpit = tc_dump_qdisc}, 2507 {.msgtype = RTM_NEWTCLASS, .doit = tc_ctl_tclass}, 2508 {.msgtype = RTM_DELTCLASS, .doit = tc_ctl_tclass}, 2509 {.msgtype = RTM_GETTCLASS, .doit = tc_ctl_tclass, 2510 .dumpit = tc_dump_tclass}, 2511 }; 2512 2513 static int __init pktsched_init(void) 2514 { 2515 int err; 2516 2517 err = register_pernet_subsys(&psched_net_ops); 2518 if (err) { 2519 pr_err("pktsched_init: " 2520 "cannot initialize per netns operations\n"); 2521 return err; 2522 } 2523 2524 register_qdisc(&pfifo_fast_ops); 2525 register_qdisc(&pfifo_qdisc_ops); 2526 register_qdisc(&bfifo_qdisc_ops); 2527 register_qdisc(&pfifo_head_drop_qdisc_ops); 2528 register_qdisc(&mq_qdisc_ops); 2529 register_qdisc(&noqueue_qdisc_ops); 2530 2531 rtnl_register_many(psched_rtnl_msg_handlers); 2532 2533 tc_wrapper_init(); 2534 2535 return 0; 2536 } 2537 2538 subsys_initcall(pktsched_init); 2539