1 /* 2 * net/sched/sch_generic.c Generic packet scheduler routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 10 * Jamal Hadi Salim, <hadi@cyberus.ca> 990601 11 * - Ingress support 12 */ 13 14 #include <linux/bitops.h> 15 #include <linux/module.h> 16 #include <linux/types.h> 17 #include <linux/kernel.h> 18 #include <linux/sched.h> 19 #include <linux/string.h> 20 #include <linux/errno.h> 21 #include <linux/netdevice.h> 22 #include <linux/skbuff.h> 23 #include <linux/rtnetlink.h> 24 #include <linux/init.h> 25 #include <linux/rcupdate.h> 26 #include <linux/list.h> 27 #include <linux/slab.h> 28 #include <linux/if_vlan.h> 29 #include <linux/skb_array.h> 30 #include <linux/if_macvlan.h> 31 #include <net/sch_generic.h> 32 #include <net/pkt_sched.h> 33 #include <net/dst.h> 34 #include <trace/events/qdisc.h> 35 #include <net/xfrm.h> 36 37 /* Qdisc to use by default */ 38 const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops; 39 EXPORT_SYMBOL(default_qdisc_ops); 40 41 /* Main transmission queue. */ 42 43 /* Modifications to data participating in scheduling must be protected with 44 * qdisc_lock(qdisc) spinlock. 45 * 46 * The idea is the following: 47 * - enqueue, dequeue are serialized via qdisc root lock 48 * - ingress filtering is also serialized via qdisc root lock 49 * - updates to tree and tree walking are only done under the rtnl mutex. 50 */ 51 52 static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q) 53 { 54 const struct netdev_queue *txq = q->dev_queue; 55 spinlock_t *lock = NULL; 56 struct sk_buff *skb; 57 58 if (q->flags & TCQ_F_NOLOCK) { 59 lock = qdisc_lock(q); 60 spin_lock(lock); 61 } 62 63 skb = skb_peek(&q->skb_bad_txq); 64 if (skb) { 65 /* check the reason of requeuing without tx lock first */ 66 txq = skb_get_tx_queue(txq->dev, skb); 67 if (!netif_xmit_frozen_or_stopped(txq)) { 68 skb = __skb_dequeue(&q->skb_bad_txq); 69 if (qdisc_is_percpu_stats(q)) { 70 qdisc_qstats_cpu_backlog_dec(q, skb); 71 qdisc_qstats_cpu_qlen_dec(q); 72 } else { 73 qdisc_qstats_backlog_dec(q, skb); 74 q->q.qlen--; 75 } 76 } else { 77 skb = NULL; 78 } 79 } 80 81 if (lock) 82 spin_unlock(lock); 83 84 return skb; 85 } 86 87 static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q) 88 { 89 struct sk_buff *skb = skb_peek(&q->skb_bad_txq); 90 91 if (unlikely(skb)) 92 skb = __skb_dequeue_bad_txq(q); 93 94 return skb; 95 } 96 97 static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q, 98 struct sk_buff *skb) 99 { 100 spinlock_t *lock = NULL; 101 102 if (q->flags & TCQ_F_NOLOCK) { 103 lock = qdisc_lock(q); 104 spin_lock(lock); 105 } 106 107 __skb_queue_tail(&q->skb_bad_txq, skb); 108 109 if (qdisc_is_percpu_stats(q)) { 110 qdisc_qstats_cpu_backlog_inc(q, skb); 111 qdisc_qstats_cpu_qlen_inc(q); 112 } else { 113 qdisc_qstats_backlog_inc(q, skb); 114 q->q.qlen++; 115 } 116 117 if (lock) 118 spin_unlock(lock); 119 } 120 121 static inline int __dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) 122 { 123 while (skb) { 124 struct sk_buff *next = skb->next; 125 126 __skb_queue_tail(&q->gso_skb, skb); 127 q->qstats.requeues++; 128 qdisc_qstats_backlog_inc(q, skb); 129 q->q.qlen++; /* it's still part of the queue */ 130 131 skb = next; 132 } 133 __netif_schedule(q); 134 135 return 0; 136 } 137 138 static inline int dev_requeue_skb_locked(struct sk_buff *skb, struct Qdisc *q) 139 { 140 spinlock_t *lock = qdisc_lock(q); 141 142 spin_lock(lock); 143 while (skb) { 144 struct sk_buff *next = skb->next; 145 146 __skb_queue_tail(&q->gso_skb, skb); 147 148 qdisc_qstats_cpu_requeues_inc(q); 149 qdisc_qstats_cpu_backlog_inc(q, skb); 150 qdisc_qstats_cpu_qlen_inc(q); 151 152 skb = next; 153 } 154 spin_unlock(lock); 155 156 __netif_schedule(q); 157 158 return 0; 159 } 160 161 static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) 162 { 163 if (q->flags & TCQ_F_NOLOCK) 164 return dev_requeue_skb_locked(skb, q); 165 else 166 return __dev_requeue_skb(skb, q); 167 } 168 169 static void try_bulk_dequeue_skb(struct Qdisc *q, 170 struct sk_buff *skb, 171 const struct netdev_queue *txq, 172 int *packets) 173 { 174 int bytelimit = qdisc_avail_bulklimit(txq) - skb->len; 175 176 while (bytelimit > 0) { 177 struct sk_buff *nskb = q->dequeue(q); 178 179 if (!nskb) 180 break; 181 182 bytelimit -= nskb->len; /* covers GSO len */ 183 skb->next = nskb; 184 skb = nskb; 185 (*packets)++; /* GSO counts as one pkt */ 186 } 187 skb_mark_not_on_list(skb); 188 } 189 190 /* This variant of try_bulk_dequeue_skb() makes sure 191 * all skbs in the chain are for the same txq 192 */ 193 static void try_bulk_dequeue_skb_slow(struct Qdisc *q, 194 struct sk_buff *skb, 195 int *packets) 196 { 197 int mapping = skb_get_queue_mapping(skb); 198 struct sk_buff *nskb; 199 int cnt = 0; 200 201 do { 202 nskb = q->dequeue(q); 203 if (!nskb) 204 break; 205 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) { 206 qdisc_enqueue_skb_bad_txq(q, nskb); 207 break; 208 } 209 skb->next = nskb; 210 skb = nskb; 211 } while (++cnt < 8); 212 (*packets) += cnt; 213 skb_mark_not_on_list(skb); 214 } 215 216 /* Note that dequeue_skb can possibly return a SKB list (via skb->next). 217 * A requeued skb (via q->gso_skb) can also be a SKB list. 218 */ 219 static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, 220 int *packets) 221 { 222 const struct netdev_queue *txq = q->dev_queue; 223 struct sk_buff *skb = NULL; 224 225 *packets = 1; 226 if (unlikely(!skb_queue_empty(&q->gso_skb))) { 227 spinlock_t *lock = NULL; 228 229 if (q->flags & TCQ_F_NOLOCK) { 230 lock = qdisc_lock(q); 231 spin_lock(lock); 232 } 233 234 skb = skb_peek(&q->gso_skb); 235 236 /* skb may be null if another cpu pulls gso_skb off in between 237 * empty check and lock. 238 */ 239 if (!skb) { 240 if (lock) 241 spin_unlock(lock); 242 goto validate; 243 } 244 245 /* skb in gso_skb were already validated */ 246 *validate = false; 247 if (xfrm_offload(skb)) 248 *validate = true; 249 /* check the reason of requeuing without tx lock first */ 250 txq = skb_get_tx_queue(txq->dev, skb); 251 if (!netif_xmit_frozen_or_stopped(txq)) { 252 skb = __skb_dequeue(&q->gso_skb); 253 if (qdisc_is_percpu_stats(q)) { 254 qdisc_qstats_cpu_backlog_dec(q, skb); 255 qdisc_qstats_cpu_qlen_dec(q); 256 } else { 257 qdisc_qstats_backlog_dec(q, skb); 258 q->q.qlen--; 259 } 260 } else { 261 skb = NULL; 262 } 263 if (lock) 264 spin_unlock(lock); 265 goto trace; 266 } 267 validate: 268 *validate = true; 269 270 if ((q->flags & TCQ_F_ONETXQUEUE) && 271 netif_xmit_frozen_or_stopped(txq)) 272 return skb; 273 274 skb = qdisc_dequeue_skb_bad_txq(q); 275 if (unlikely(skb)) 276 goto bulk; 277 skb = q->dequeue(q); 278 if (skb) { 279 bulk: 280 if (qdisc_may_bulk(q)) 281 try_bulk_dequeue_skb(q, skb, txq, packets); 282 else 283 try_bulk_dequeue_skb_slow(q, skb, packets); 284 } 285 trace: 286 trace_qdisc_dequeue(q, txq, *packets, skb); 287 return skb; 288 } 289 290 /* 291 * Transmit possibly several skbs, and handle the return status as 292 * required. Owning running seqcount bit guarantees that 293 * only one CPU can execute this function. 294 * 295 * Returns to the caller: 296 * false - hardware queue frozen backoff 297 * true - feel free to send more pkts 298 */ 299 bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, 300 struct net_device *dev, struct netdev_queue *txq, 301 spinlock_t *root_lock, bool validate) 302 { 303 int ret = NETDEV_TX_BUSY; 304 bool again = false; 305 306 /* And release qdisc */ 307 if (root_lock) 308 spin_unlock(root_lock); 309 310 /* Note that we validate skb (GSO, checksum, ...) outside of locks */ 311 if (validate) 312 skb = validate_xmit_skb_list(skb, dev, &again); 313 314 #ifdef CONFIG_XFRM_OFFLOAD 315 if (unlikely(again)) { 316 if (root_lock) 317 spin_lock(root_lock); 318 319 dev_requeue_skb(skb, q); 320 return false; 321 } 322 #endif 323 324 if (likely(skb)) { 325 HARD_TX_LOCK(dev, txq, smp_processor_id()); 326 if (!netif_xmit_frozen_or_stopped(txq)) 327 skb = dev_hard_start_xmit(skb, dev, txq, &ret); 328 329 HARD_TX_UNLOCK(dev, txq); 330 } else { 331 if (root_lock) 332 spin_lock(root_lock); 333 return true; 334 } 335 336 if (root_lock) 337 spin_lock(root_lock); 338 339 if (!dev_xmit_complete(ret)) { 340 /* Driver returned NETDEV_TX_BUSY - requeue skb */ 341 if (unlikely(ret != NETDEV_TX_BUSY)) 342 net_warn_ratelimited("BUG %s code %d qlen %d\n", 343 dev->name, ret, q->q.qlen); 344 345 dev_requeue_skb(skb, q); 346 return false; 347 } 348 349 return true; 350 } 351 352 /* 353 * NOTE: Called under qdisc_lock(q) with locally disabled BH. 354 * 355 * running seqcount guarantees only one CPU can process 356 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for 357 * this queue. 358 * 359 * netif_tx_lock serializes accesses to device driver. 360 * 361 * qdisc_lock(q) and netif_tx_lock are mutually exclusive, 362 * if one is grabbed, another must be free. 363 * 364 * Note, that this procedure can be called by a watchdog timer 365 * 366 * Returns to the caller: 367 * 0 - queue is empty or throttled. 368 * >0 - queue is not empty. 369 * 370 */ 371 static inline bool qdisc_restart(struct Qdisc *q, int *packets) 372 { 373 spinlock_t *root_lock = NULL; 374 struct netdev_queue *txq; 375 struct net_device *dev; 376 struct sk_buff *skb; 377 bool validate; 378 379 /* Dequeue packet */ 380 skb = dequeue_skb(q, &validate, packets); 381 if (unlikely(!skb)) 382 return false; 383 384 if (!(q->flags & TCQ_F_NOLOCK)) 385 root_lock = qdisc_lock(q); 386 387 dev = qdisc_dev(q); 388 txq = skb_get_tx_queue(dev, skb); 389 390 return sch_direct_xmit(skb, q, dev, txq, root_lock, validate); 391 } 392 393 void __qdisc_run(struct Qdisc *q) 394 { 395 int quota = dev_tx_weight; 396 int packets; 397 398 while (qdisc_restart(q, &packets)) { 399 /* 400 * Ordered by possible occurrence: Postpone processing if 401 * 1. we've exceeded packet quota 402 * 2. another process needs the CPU; 403 */ 404 quota -= packets; 405 if (quota <= 0 || need_resched()) { 406 __netif_schedule(q); 407 break; 408 } 409 } 410 } 411 412 unsigned long dev_trans_start(struct net_device *dev) 413 { 414 unsigned long val, res; 415 unsigned int i; 416 417 if (is_vlan_dev(dev)) 418 dev = vlan_dev_real_dev(dev); 419 else if (netif_is_macvlan(dev)) 420 dev = macvlan_dev_real_dev(dev); 421 res = netdev_get_tx_queue(dev, 0)->trans_start; 422 for (i = 1; i < dev->num_tx_queues; i++) { 423 val = netdev_get_tx_queue(dev, i)->trans_start; 424 if (val && time_after(val, res)) 425 res = val; 426 } 427 428 return res; 429 } 430 EXPORT_SYMBOL(dev_trans_start); 431 432 static void dev_watchdog(struct timer_list *t) 433 { 434 struct net_device *dev = from_timer(dev, t, watchdog_timer); 435 436 netif_tx_lock(dev); 437 if (!qdisc_tx_is_noop(dev)) { 438 if (netif_device_present(dev) && 439 netif_running(dev) && 440 netif_carrier_ok(dev)) { 441 int some_queue_timedout = 0; 442 unsigned int i; 443 unsigned long trans_start; 444 445 for (i = 0; i < dev->num_tx_queues; i++) { 446 struct netdev_queue *txq; 447 448 txq = netdev_get_tx_queue(dev, i); 449 trans_start = txq->trans_start; 450 if (netif_xmit_stopped(txq) && 451 time_after(jiffies, (trans_start + 452 dev->watchdog_timeo))) { 453 some_queue_timedout = 1; 454 txq->trans_timeout++; 455 break; 456 } 457 } 458 459 if (some_queue_timedout) { 460 WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n", 461 dev->name, netdev_drivername(dev), i); 462 dev->netdev_ops->ndo_tx_timeout(dev); 463 } 464 if (!mod_timer(&dev->watchdog_timer, 465 round_jiffies(jiffies + 466 dev->watchdog_timeo))) 467 dev_hold(dev); 468 } 469 } 470 netif_tx_unlock(dev); 471 472 dev_put(dev); 473 } 474 475 void __netdev_watchdog_up(struct net_device *dev) 476 { 477 if (dev->netdev_ops->ndo_tx_timeout) { 478 if (dev->watchdog_timeo <= 0) 479 dev->watchdog_timeo = 5*HZ; 480 if (!mod_timer(&dev->watchdog_timer, 481 round_jiffies(jiffies + dev->watchdog_timeo))) 482 dev_hold(dev); 483 } 484 } 485 486 static void dev_watchdog_up(struct net_device *dev) 487 { 488 __netdev_watchdog_up(dev); 489 } 490 491 static void dev_watchdog_down(struct net_device *dev) 492 { 493 netif_tx_lock_bh(dev); 494 if (del_timer(&dev->watchdog_timer)) 495 dev_put(dev); 496 netif_tx_unlock_bh(dev); 497 } 498 499 /** 500 * netif_carrier_on - set carrier 501 * @dev: network device 502 * 503 * Device has detected that carrier. 504 */ 505 void netif_carrier_on(struct net_device *dev) 506 { 507 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) { 508 if (dev->reg_state == NETREG_UNINITIALIZED) 509 return; 510 atomic_inc(&dev->carrier_up_count); 511 linkwatch_fire_event(dev); 512 if (netif_running(dev)) 513 __netdev_watchdog_up(dev); 514 } 515 } 516 EXPORT_SYMBOL(netif_carrier_on); 517 518 /** 519 * netif_carrier_off - clear carrier 520 * @dev: network device 521 * 522 * Device has detected loss of carrier. 523 */ 524 void netif_carrier_off(struct net_device *dev) 525 { 526 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) { 527 if (dev->reg_state == NETREG_UNINITIALIZED) 528 return; 529 atomic_inc(&dev->carrier_down_count); 530 linkwatch_fire_event(dev); 531 } 532 } 533 EXPORT_SYMBOL(netif_carrier_off); 534 535 /* "NOOP" scheduler: the best scheduler, recommended for all interfaces 536 under all circumstances. It is difficult to invent anything faster or 537 cheaper. 538 */ 539 540 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, 541 struct sk_buff **to_free) 542 { 543 __qdisc_drop(skb, to_free); 544 return NET_XMIT_CN; 545 } 546 547 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc) 548 { 549 return NULL; 550 } 551 552 struct Qdisc_ops noop_qdisc_ops __read_mostly = { 553 .id = "noop", 554 .priv_size = 0, 555 .enqueue = noop_enqueue, 556 .dequeue = noop_dequeue, 557 .peek = noop_dequeue, 558 .owner = THIS_MODULE, 559 }; 560 561 static struct netdev_queue noop_netdev_queue = { 562 .qdisc = &noop_qdisc, 563 .qdisc_sleeping = &noop_qdisc, 564 }; 565 566 struct Qdisc noop_qdisc = { 567 .enqueue = noop_enqueue, 568 .dequeue = noop_dequeue, 569 .flags = TCQ_F_BUILTIN, 570 .ops = &noop_qdisc_ops, 571 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), 572 .dev_queue = &noop_netdev_queue, 573 .running = SEQCNT_ZERO(noop_qdisc.running), 574 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock), 575 }; 576 EXPORT_SYMBOL(noop_qdisc); 577 578 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt, 579 struct netlink_ext_ack *extack) 580 { 581 /* register_qdisc() assigns a default of noop_enqueue if unset, 582 * but __dev_queue_xmit() treats noqueue only as such 583 * if this is NULL - so clear it here. */ 584 qdisc->enqueue = NULL; 585 return 0; 586 } 587 588 struct Qdisc_ops noqueue_qdisc_ops __read_mostly = { 589 .id = "noqueue", 590 .priv_size = 0, 591 .init = noqueue_init, 592 .enqueue = noop_enqueue, 593 .dequeue = noop_dequeue, 594 .peek = noop_dequeue, 595 .owner = THIS_MODULE, 596 }; 597 598 static const u8 prio2band[TC_PRIO_MAX + 1] = { 599 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 600 }; 601 602 /* 3-band FIFO queue: old style, but should be a bit faster than 603 generic prio+fifo combination. 604 */ 605 606 #define PFIFO_FAST_BANDS 3 607 608 /* 609 * Private data for a pfifo_fast scheduler containing: 610 * - rings for priority bands 611 */ 612 struct pfifo_fast_priv { 613 struct skb_array q[PFIFO_FAST_BANDS]; 614 }; 615 616 static inline struct skb_array *band2list(struct pfifo_fast_priv *priv, 617 int band) 618 { 619 return &priv->q[band]; 620 } 621 622 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, 623 struct sk_buff **to_free) 624 { 625 int band = prio2band[skb->priority & TC_PRIO_MAX]; 626 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 627 struct skb_array *q = band2list(priv, band); 628 unsigned int pkt_len = qdisc_pkt_len(skb); 629 int err; 630 631 err = skb_array_produce(q, skb); 632 633 if (unlikely(err)) 634 return qdisc_drop_cpu(skb, qdisc, to_free); 635 636 qdisc_qstats_cpu_qlen_inc(qdisc); 637 /* Note: skb can not be used after skb_array_produce(), 638 * so we better not use qdisc_qstats_cpu_backlog_inc() 639 */ 640 this_cpu_add(qdisc->cpu_qstats->backlog, pkt_len); 641 return NET_XMIT_SUCCESS; 642 } 643 644 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc) 645 { 646 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 647 struct sk_buff *skb = NULL; 648 int band; 649 650 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) { 651 struct skb_array *q = band2list(priv, band); 652 653 if (__skb_array_empty(q)) 654 continue; 655 656 skb = __skb_array_consume(q); 657 } 658 if (likely(skb)) { 659 qdisc_qstats_cpu_backlog_dec(qdisc, skb); 660 qdisc_bstats_cpu_update(qdisc, skb); 661 qdisc_qstats_cpu_qlen_dec(qdisc); 662 } 663 664 return skb; 665 } 666 667 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc) 668 { 669 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 670 struct sk_buff *skb = NULL; 671 int band; 672 673 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) { 674 struct skb_array *q = band2list(priv, band); 675 676 skb = __skb_array_peek(q); 677 } 678 679 return skb; 680 } 681 682 static void pfifo_fast_reset(struct Qdisc *qdisc) 683 { 684 int i, band; 685 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 686 687 for (band = 0; band < PFIFO_FAST_BANDS; band++) { 688 struct skb_array *q = band2list(priv, band); 689 struct sk_buff *skb; 690 691 /* NULL ring is possible if destroy path is due to a failed 692 * skb_array_init() in pfifo_fast_init() case. 693 */ 694 if (!q->ring.queue) 695 continue; 696 697 while ((skb = __skb_array_consume(q)) != NULL) 698 kfree_skb(skb); 699 } 700 701 for_each_possible_cpu(i) { 702 struct gnet_stats_queue *q = per_cpu_ptr(qdisc->cpu_qstats, i); 703 704 q->backlog = 0; 705 q->qlen = 0; 706 } 707 } 708 709 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb) 710 { 711 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS }; 712 713 memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1); 714 if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt)) 715 goto nla_put_failure; 716 return skb->len; 717 718 nla_put_failure: 719 return -1; 720 } 721 722 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt, 723 struct netlink_ext_ack *extack) 724 { 725 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len; 726 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 727 int prio; 728 729 /* guard against zero length rings */ 730 if (!qlen) 731 return -EINVAL; 732 733 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { 734 struct skb_array *q = band2list(priv, prio); 735 int err; 736 737 err = skb_array_init(q, qlen, GFP_KERNEL); 738 if (err) 739 return -ENOMEM; 740 } 741 742 /* Can by-pass the queue discipline */ 743 qdisc->flags |= TCQ_F_CAN_BYPASS; 744 return 0; 745 } 746 747 static void pfifo_fast_destroy(struct Qdisc *sch) 748 { 749 struct pfifo_fast_priv *priv = qdisc_priv(sch); 750 int prio; 751 752 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { 753 struct skb_array *q = band2list(priv, prio); 754 755 /* NULL ring is possible if destroy path is due to a failed 756 * skb_array_init() in pfifo_fast_init() case. 757 */ 758 if (!q->ring.queue) 759 continue; 760 /* Destroy ring but no need to kfree_skb because a call to 761 * pfifo_fast_reset() has already done that work. 762 */ 763 ptr_ring_cleanup(&q->ring, NULL); 764 } 765 } 766 767 static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch, 768 unsigned int new_len) 769 { 770 struct pfifo_fast_priv *priv = qdisc_priv(sch); 771 struct skb_array *bands[PFIFO_FAST_BANDS]; 772 int prio; 773 774 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { 775 struct skb_array *q = band2list(priv, prio); 776 777 bands[prio] = q; 778 } 779 780 return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len, 781 GFP_KERNEL); 782 } 783 784 struct Qdisc_ops pfifo_fast_ops __read_mostly = { 785 .id = "pfifo_fast", 786 .priv_size = sizeof(struct pfifo_fast_priv), 787 .enqueue = pfifo_fast_enqueue, 788 .dequeue = pfifo_fast_dequeue, 789 .peek = pfifo_fast_peek, 790 .init = pfifo_fast_init, 791 .destroy = pfifo_fast_destroy, 792 .reset = pfifo_fast_reset, 793 .dump = pfifo_fast_dump, 794 .change_tx_queue_len = pfifo_fast_change_tx_queue_len, 795 .owner = THIS_MODULE, 796 .static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS, 797 }; 798 EXPORT_SYMBOL(pfifo_fast_ops); 799 800 static struct lock_class_key qdisc_tx_busylock; 801 static struct lock_class_key qdisc_running_key; 802 803 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 804 const struct Qdisc_ops *ops, 805 struct netlink_ext_ack *extack) 806 { 807 void *p; 808 struct Qdisc *sch; 809 unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size; 810 int err = -ENOBUFS; 811 struct net_device *dev; 812 813 if (!dev_queue) { 814 NL_SET_ERR_MSG(extack, "No device queue given"); 815 err = -EINVAL; 816 goto errout; 817 } 818 819 dev = dev_queue->dev; 820 p = kzalloc_node(size, GFP_KERNEL, 821 netdev_queue_numa_node_read(dev_queue)); 822 823 if (!p) 824 goto errout; 825 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); 826 /* if we got non aligned memory, ask more and do alignment ourself */ 827 if (sch != p) { 828 kfree(p); 829 p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL, 830 netdev_queue_numa_node_read(dev_queue)); 831 if (!p) 832 goto errout; 833 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); 834 sch->padded = (char *) sch - (char *) p; 835 } 836 __skb_queue_head_init(&sch->gso_skb); 837 __skb_queue_head_init(&sch->skb_bad_txq); 838 qdisc_skb_head_init(&sch->q); 839 spin_lock_init(&sch->q.lock); 840 841 if (ops->static_flags & TCQ_F_CPUSTATS) { 842 sch->cpu_bstats = 843 netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu); 844 if (!sch->cpu_bstats) 845 goto errout1; 846 847 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue); 848 if (!sch->cpu_qstats) { 849 free_percpu(sch->cpu_bstats); 850 goto errout1; 851 } 852 } 853 854 spin_lock_init(&sch->busylock); 855 lockdep_set_class(&sch->busylock, 856 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock); 857 858 /* seqlock has the same scope of busylock, for NOLOCK qdisc */ 859 spin_lock_init(&sch->seqlock); 860 lockdep_set_class(&sch->busylock, 861 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock); 862 863 seqcount_init(&sch->running); 864 lockdep_set_class(&sch->running, 865 dev->qdisc_running_key ?: &qdisc_running_key); 866 867 sch->ops = ops; 868 sch->flags = ops->static_flags; 869 sch->enqueue = ops->enqueue; 870 sch->dequeue = ops->dequeue; 871 sch->dev_queue = dev_queue; 872 dev_hold(dev); 873 refcount_set(&sch->refcnt, 1); 874 875 return sch; 876 errout1: 877 kfree(p); 878 errout: 879 return ERR_PTR(err); 880 } 881 882 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 883 const struct Qdisc_ops *ops, 884 unsigned int parentid, 885 struct netlink_ext_ack *extack) 886 { 887 struct Qdisc *sch; 888 889 if (!try_module_get(ops->owner)) { 890 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter"); 891 return NULL; 892 } 893 894 sch = qdisc_alloc(dev_queue, ops, extack); 895 if (IS_ERR(sch)) { 896 module_put(ops->owner); 897 return NULL; 898 } 899 sch->parent = parentid; 900 901 if (!ops->init || ops->init(sch, NULL, extack) == 0) 902 return sch; 903 904 qdisc_put(sch); 905 return NULL; 906 } 907 EXPORT_SYMBOL(qdisc_create_dflt); 908 909 /* Under qdisc_lock(qdisc) and BH! */ 910 911 void qdisc_reset(struct Qdisc *qdisc) 912 { 913 const struct Qdisc_ops *ops = qdisc->ops; 914 struct sk_buff *skb, *tmp; 915 916 if (ops->reset) 917 ops->reset(qdisc); 918 919 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) { 920 __skb_unlink(skb, &qdisc->gso_skb); 921 kfree_skb_list(skb); 922 } 923 924 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) { 925 __skb_unlink(skb, &qdisc->skb_bad_txq); 926 kfree_skb_list(skb); 927 } 928 929 qdisc->q.qlen = 0; 930 qdisc->qstats.backlog = 0; 931 } 932 EXPORT_SYMBOL(qdisc_reset); 933 934 void qdisc_free(struct Qdisc *qdisc) 935 { 936 if (qdisc_is_percpu_stats(qdisc)) { 937 free_percpu(qdisc->cpu_bstats); 938 free_percpu(qdisc->cpu_qstats); 939 } 940 941 kfree((char *) qdisc - qdisc->padded); 942 } 943 944 static void qdisc_free_cb(struct rcu_head *head) 945 { 946 struct Qdisc *q = container_of(head, struct Qdisc, rcu); 947 948 qdisc_free(q); 949 } 950 951 static void qdisc_destroy(struct Qdisc *qdisc) 952 { 953 const struct Qdisc_ops *ops = qdisc->ops; 954 struct sk_buff *skb, *tmp; 955 956 #ifdef CONFIG_NET_SCHED 957 qdisc_hash_del(qdisc); 958 959 qdisc_put_stab(rtnl_dereference(qdisc->stab)); 960 #endif 961 gen_kill_estimator(&qdisc->rate_est); 962 if (ops->reset) 963 ops->reset(qdisc); 964 if (ops->destroy) 965 ops->destroy(qdisc); 966 967 module_put(ops->owner); 968 dev_put(qdisc_dev(qdisc)); 969 970 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) { 971 __skb_unlink(skb, &qdisc->gso_skb); 972 kfree_skb_list(skb); 973 } 974 975 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) { 976 __skb_unlink(skb, &qdisc->skb_bad_txq); 977 kfree_skb_list(skb); 978 } 979 980 call_rcu(&qdisc->rcu, qdisc_free_cb); 981 } 982 983 void qdisc_put(struct Qdisc *qdisc) 984 { 985 if (qdisc->flags & TCQ_F_BUILTIN || 986 !refcount_dec_and_test(&qdisc->refcnt)) 987 return; 988 989 qdisc_destroy(qdisc); 990 } 991 EXPORT_SYMBOL(qdisc_put); 992 993 /* Version of qdisc_put() that is called with rtnl mutex unlocked. 994 * Intended to be used as optimization, this function only takes rtnl lock if 995 * qdisc reference counter reached zero. 996 */ 997 998 void qdisc_put_unlocked(struct Qdisc *qdisc) 999 { 1000 if (qdisc->flags & TCQ_F_BUILTIN || 1001 !refcount_dec_and_rtnl_lock(&qdisc->refcnt)) 1002 return; 1003 1004 qdisc_destroy(qdisc); 1005 rtnl_unlock(); 1006 } 1007 EXPORT_SYMBOL(qdisc_put_unlocked); 1008 1009 /* Attach toplevel qdisc to device queue. */ 1010 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 1011 struct Qdisc *qdisc) 1012 { 1013 struct Qdisc *oqdisc = dev_queue->qdisc_sleeping; 1014 spinlock_t *root_lock; 1015 1016 root_lock = qdisc_lock(oqdisc); 1017 spin_lock_bh(root_lock); 1018 1019 /* ... and graft new one */ 1020 if (qdisc == NULL) 1021 qdisc = &noop_qdisc; 1022 dev_queue->qdisc_sleeping = qdisc; 1023 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc); 1024 1025 spin_unlock_bh(root_lock); 1026 1027 return oqdisc; 1028 } 1029 EXPORT_SYMBOL(dev_graft_qdisc); 1030 1031 static void attach_one_default_qdisc(struct net_device *dev, 1032 struct netdev_queue *dev_queue, 1033 void *_unused) 1034 { 1035 struct Qdisc *qdisc; 1036 const struct Qdisc_ops *ops = default_qdisc_ops; 1037 1038 if (dev->priv_flags & IFF_NO_QUEUE) 1039 ops = &noqueue_qdisc_ops; 1040 1041 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL); 1042 if (!qdisc) { 1043 netdev_info(dev, "activation failed\n"); 1044 return; 1045 } 1046 if (!netif_is_multiqueue(dev)) 1047 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 1048 dev_queue->qdisc_sleeping = qdisc; 1049 } 1050 1051 static void attach_default_qdiscs(struct net_device *dev) 1052 { 1053 struct netdev_queue *txq; 1054 struct Qdisc *qdisc; 1055 1056 txq = netdev_get_tx_queue(dev, 0); 1057 1058 if (!netif_is_multiqueue(dev) || 1059 dev->priv_flags & IFF_NO_QUEUE) { 1060 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL); 1061 dev->qdisc = txq->qdisc_sleeping; 1062 qdisc_refcount_inc(dev->qdisc); 1063 } else { 1064 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL); 1065 if (qdisc) { 1066 dev->qdisc = qdisc; 1067 qdisc->ops->attach(qdisc); 1068 } 1069 } 1070 #ifdef CONFIG_NET_SCHED 1071 if (dev->qdisc != &noop_qdisc) 1072 qdisc_hash_add(dev->qdisc, false); 1073 #endif 1074 } 1075 1076 static void transition_one_qdisc(struct net_device *dev, 1077 struct netdev_queue *dev_queue, 1078 void *_need_watchdog) 1079 { 1080 struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping; 1081 int *need_watchdog_p = _need_watchdog; 1082 1083 if (!(new_qdisc->flags & TCQ_F_BUILTIN)) 1084 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state); 1085 1086 rcu_assign_pointer(dev_queue->qdisc, new_qdisc); 1087 if (need_watchdog_p) { 1088 dev_queue->trans_start = 0; 1089 *need_watchdog_p = 1; 1090 } 1091 } 1092 1093 void dev_activate(struct net_device *dev) 1094 { 1095 int need_watchdog; 1096 1097 /* No queueing discipline is attached to device; 1098 * create default one for devices, which need queueing 1099 * and noqueue_qdisc for virtual interfaces 1100 */ 1101 1102 if (dev->qdisc == &noop_qdisc) 1103 attach_default_qdiscs(dev); 1104 1105 if (!netif_carrier_ok(dev)) 1106 /* Delay activation until next carrier-on event */ 1107 return; 1108 1109 need_watchdog = 0; 1110 netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog); 1111 if (dev_ingress_queue(dev)) 1112 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL); 1113 1114 if (need_watchdog) { 1115 netif_trans_update(dev); 1116 dev_watchdog_up(dev); 1117 } 1118 } 1119 EXPORT_SYMBOL(dev_activate); 1120 1121 static void dev_deactivate_queue(struct net_device *dev, 1122 struct netdev_queue *dev_queue, 1123 void *_qdisc_default) 1124 { 1125 struct Qdisc *qdisc_default = _qdisc_default; 1126 struct Qdisc *qdisc; 1127 1128 qdisc = rtnl_dereference(dev_queue->qdisc); 1129 if (qdisc) { 1130 bool nolock = qdisc->flags & TCQ_F_NOLOCK; 1131 1132 if (nolock) 1133 spin_lock_bh(&qdisc->seqlock); 1134 spin_lock_bh(qdisc_lock(qdisc)); 1135 1136 if (!(qdisc->flags & TCQ_F_BUILTIN)) 1137 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state); 1138 1139 rcu_assign_pointer(dev_queue->qdisc, qdisc_default); 1140 qdisc_reset(qdisc); 1141 1142 spin_unlock_bh(qdisc_lock(qdisc)); 1143 if (nolock) 1144 spin_unlock_bh(&qdisc->seqlock); 1145 } 1146 } 1147 1148 static bool some_qdisc_is_busy(struct net_device *dev) 1149 { 1150 unsigned int i; 1151 1152 for (i = 0; i < dev->num_tx_queues; i++) { 1153 struct netdev_queue *dev_queue; 1154 spinlock_t *root_lock; 1155 struct Qdisc *q; 1156 int val; 1157 1158 dev_queue = netdev_get_tx_queue(dev, i); 1159 q = dev_queue->qdisc_sleeping; 1160 1161 root_lock = qdisc_lock(q); 1162 spin_lock_bh(root_lock); 1163 1164 val = (qdisc_is_running(q) || 1165 test_bit(__QDISC_STATE_SCHED, &q->state)); 1166 1167 spin_unlock_bh(root_lock); 1168 1169 if (val) 1170 return true; 1171 } 1172 return false; 1173 } 1174 1175 static void dev_qdisc_reset(struct net_device *dev, 1176 struct netdev_queue *dev_queue, 1177 void *none) 1178 { 1179 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 1180 1181 if (qdisc) 1182 qdisc_reset(qdisc); 1183 } 1184 1185 /** 1186 * dev_deactivate_many - deactivate transmissions on several devices 1187 * @head: list of devices to deactivate 1188 * 1189 * This function returns only when all outstanding transmissions 1190 * have completed, unless all devices are in dismantle phase. 1191 */ 1192 void dev_deactivate_many(struct list_head *head) 1193 { 1194 struct net_device *dev; 1195 1196 list_for_each_entry(dev, head, close_list) { 1197 netdev_for_each_tx_queue(dev, dev_deactivate_queue, 1198 &noop_qdisc); 1199 if (dev_ingress_queue(dev)) 1200 dev_deactivate_queue(dev, dev_ingress_queue(dev), 1201 &noop_qdisc); 1202 1203 dev_watchdog_down(dev); 1204 } 1205 1206 /* Wait for outstanding qdisc-less dev_queue_xmit calls. 1207 * This is avoided if all devices are in dismantle phase : 1208 * Caller will call synchronize_net() for us 1209 */ 1210 synchronize_net(); 1211 1212 /* Wait for outstanding qdisc_run calls. */ 1213 list_for_each_entry(dev, head, close_list) { 1214 while (some_qdisc_is_busy(dev)) 1215 yield(); 1216 /* The new qdisc is assigned at this point so we can safely 1217 * unwind stale skb lists and qdisc statistics 1218 */ 1219 netdev_for_each_tx_queue(dev, dev_qdisc_reset, NULL); 1220 if (dev_ingress_queue(dev)) 1221 dev_qdisc_reset(dev, dev_ingress_queue(dev), NULL); 1222 } 1223 } 1224 1225 void dev_deactivate(struct net_device *dev) 1226 { 1227 LIST_HEAD(single); 1228 1229 list_add(&dev->close_list, &single); 1230 dev_deactivate_many(&single); 1231 list_del(&single); 1232 } 1233 EXPORT_SYMBOL(dev_deactivate); 1234 1235 static int qdisc_change_tx_queue_len(struct net_device *dev, 1236 struct netdev_queue *dev_queue) 1237 { 1238 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 1239 const struct Qdisc_ops *ops = qdisc->ops; 1240 1241 if (ops->change_tx_queue_len) 1242 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len); 1243 return 0; 1244 } 1245 1246 int dev_qdisc_change_tx_queue_len(struct net_device *dev) 1247 { 1248 bool up = dev->flags & IFF_UP; 1249 unsigned int i; 1250 int ret = 0; 1251 1252 if (up) 1253 dev_deactivate(dev); 1254 1255 for (i = 0; i < dev->num_tx_queues; i++) { 1256 ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]); 1257 1258 /* TODO: revert changes on a partial failure */ 1259 if (ret) 1260 break; 1261 } 1262 1263 if (up) 1264 dev_activate(dev); 1265 return ret; 1266 } 1267 1268 static void dev_init_scheduler_queue(struct net_device *dev, 1269 struct netdev_queue *dev_queue, 1270 void *_qdisc) 1271 { 1272 struct Qdisc *qdisc = _qdisc; 1273 1274 rcu_assign_pointer(dev_queue->qdisc, qdisc); 1275 dev_queue->qdisc_sleeping = qdisc; 1276 __skb_queue_head_init(&qdisc->gso_skb); 1277 __skb_queue_head_init(&qdisc->skb_bad_txq); 1278 } 1279 1280 void dev_init_scheduler(struct net_device *dev) 1281 { 1282 dev->qdisc = &noop_qdisc; 1283 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc); 1284 if (dev_ingress_queue(dev)) 1285 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc); 1286 1287 timer_setup(&dev->watchdog_timer, dev_watchdog, 0); 1288 } 1289 1290 static void shutdown_scheduler_queue(struct net_device *dev, 1291 struct netdev_queue *dev_queue, 1292 void *_qdisc_default) 1293 { 1294 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 1295 struct Qdisc *qdisc_default = _qdisc_default; 1296 1297 if (qdisc) { 1298 rcu_assign_pointer(dev_queue->qdisc, qdisc_default); 1299 dev_queue->qdisc_sleeping = qdisc_default; 1300 1301 qdisc_put(qdisc); 1302 } 1303 } 1304 1305 void dev_shutdown(struct net_device *dev) 1306 { 1307 netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); 1308 if (dev_ingress_queue(dev)) 1309 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc); 1310 qdisc_put(dev->qdisc); 1311 dev->qdisc = &noop_qdisc; 1312 1313 WARN_ON(timer_pending(&dev->watchdog_timer)); 1314 } 1315 1316 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1317 const struct tc_ratespec *conf, 1318 u64 rate64) 1319 { 1320 memset(r, 0, sizeof(*r)); 1321 r->overhead = conf->overhead; 1322 r->rate_bytes_ps = max_t(u64, conf->rate, rate64); 1323 r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK); 1324 r->mult = 1; 1325 /* 1326 * The deal here is to replace a divide by a reciprocal one 1327 * in fast path (a reciprocal divide is a multiply and a shift) 1328 * 1329 * Normal formula would be : 1330 * time_in_ns = (NSEC_PER_SEC * len) / rate_bps 1331 * 1332 * We compute mult/shift to use instead : 1333 * time_in_ns = (len * mult) >> shift; 1334 * 1335 * We try to get the highest possible mult value for accuracy, 1336 * but have to make sure no overflows will ever happen. 1337 */ 1338 if (r->rate_bytes_ps > 0) { 1339 u64 factor = NSEC_PER_SEC; 1340 1341 for (;;) { 1342 r->mult = div64_u64(factor, r->rate_bytes_ps); 1343 if (r->mult & (1U << 31) || factor & (1ULL << 63)) 1344 break; 1345 factor <<= 1; 1346 r->shift++; 1347 } 1348 } 1349 } 1350 EXPORT_SYMBOL(psched_ratecfg_precompute); 1351 1352 static void mini_qdisc_rcu_func(struct rcu_head *head) 1353 { 1354 } 1355 1356 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1357 struct tcf_proto *tp_head) 1358 { 1359 struct mini_Qdisc *miniq_old = rtnl_dereference(*miniqp->p_miniq); 1360 struct mini_Qdisc *miniq; 1361 1362 if (!tp_head) { 1363 RCU_INIT_POINTER(*miniqp->p_miniq, NULL); 1364 /* Wait for flying RCU callback before it is freed. */ 1365 rcu_barrier_bh(); 1366 return; 1367 } 1368 1369 miniq = !miniq_old || miniq_old == &miniqp->miniq2 ? 1370 &miniqp->miniq1 : &miniqp->miniq2; 1371 1372 /* We need to make sure that readers won't see the miniq 1373 * we are about to modify. So wait until previous call_rcu_bh callback 1374 * is done. 1375 */ 1376 rcu_barrier_bh(); 1377 miniq->filter_list = tp_head; 1378 rcu_assign_pointer(*miniqp->p_miniq, miniq); 1379 1380 if (miniq_old) 1381 /* This is counterpart of the rcu barriers above. We need to 1382 * block potential new user of miniq_old until all readers 1383 * are not seeing it. 1384 */ 1385 call_rcu_bh(&miniq_old->rcu, mini_qdisc_rcu_func); 1386 } 1387 EXPORT_SYMBOL(mini_qdisc_pair_swap); 1388 1389 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1390 struct mini_Qdisc __rcu **p_miniq) 1391 { 1392 miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats; 1393 miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats; 1394 miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats; 1395 miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats; 1396 miniqp->p_miniq = p_miniq; 1397 } 1398 EXPORT_SYMBOL(mini_qdisc_pair_init); 1399