1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/sch_generic.c Generic packet scheduler routines. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * Jamal Hadi Salim, <hadi@cyberus.ca> 990601 7 * - Ingress support 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/module.h> 12 #include <linux/types.h> 13 #include <linux/kernel.h> 14 #include <linux/sched.h> 15 #include <linux/string.h> 16 #include <linux/errno.h> 17 #include <linux/netdevice.h> 18 #include <linux/skbuff.h> 19 #include <linux/rtnetlink.h> 20 #include <linux/init.h> 21 #include <linux/rcupdate.h> 22 #include <linux/list.h> 23 #include <linux/slab.h> 24 #include <linux/if_vlan.h> 25 #include <linux/skb_array.h> 26 #include <linux/if_macvlan.h> 27 #include <net/sch_generic.h> 28 #include <net/pkt_sched.h> 29 #include <net/dst.h> 30 #include <trace/events/qdisc.h> 31 #include <trace/events/net.h> 32 #include <net/xfrm.h> 33 34 /* Qdisc to use by default */ 35 const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops; 36 EXPORT_SYMBOL(default_qdisc_ops); 37 38 /* Main transmission queue. */ 39 40 /* Modifications to data participating in scheduling must be protected with 41 * qdisc_lock(qdisc) spinlock. 42 * 43 * The idea is the following: 44 * - enqueue, dequeue are serialized via qdisc root lock 45 * - ingress filtering is also serialized via qdisc root lock 46 * - updates to tree and tree walking are only done under the rtnl mutex. 47 */ 48 49 #define SKB_XOFF_MAGIC ((struct sk_buff *)1UL) 50 51 static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q) 52 { 53 const struct netdev_queue *txq = q->dev_queue; 54 spinlock_t *lock = NULL; 55 struct sk_buff *skb; 56 57 if (q->flags & TCQ_F_NOLOCK) { 58 lock = qdisc_lock(q); 59 spin_lock(lock); 60 } 61 62 skb = skb_peek(&q->skb_bad_txq); 63 if (skb) { 64 /* check the reason of requeuing without tx lock first */ 65 txq = skb_get_tx_queue(txq->dev, skb); 66 if (!netif_xmit_frozen_or_stopped(txq)) { 67 skb = __skb_dequeue(&q->skb_bad_txq); 68 if (qdisc_is_percpu_stats(q)) { 69 qdisc_qstats_cpu_backlog_dec(q, skb); 70 qdisc_qstats_cpu_qlen_dec(q); 71 } else { 72 qdisc_qstats_backlog_dec(q, skb); 73 q->q.qlen--; 74 } 75 } else { 76 skb = SKB_XOFF_MAGIC; 77 } 78 } 79 80 if (lock) 81 spin_unlock(lock); 82 83 return skb; 84 } 85 86 static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q) 87 { 88 struct sk_buff *skb = skb_peek(&q->skb_bad_txq); 89 90 if (unlikely(skb)) 91 skb = __skb_dequeue_bad_txq(q); 92 93 return skb; 94 } 95 96 static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q, 97 struct sk_buff *skb) 98 { 99 spinlock_t *lock = NULL; 100 101 if (q->flags & TCQ_F_NOLOCK) { 102 lock = qdisc_lock(q); 103 spin_lock(lock); 104 } 105 106 __skb_queue_tail(&q->skb_bad_txq, skb); 107 108 if (qdisc_is_percpu_stats(q)) { 109 qdisc_qstats_cpu_backlog_inc(q, skb); 110 qdisc_qstats_cpu_qlen_inc(q); 111 } else { 112 qdisc_qstats_backlog_inc(q, skb); 113 q->q.qlen++; 114 } 115 116 if (lock) 117 spin_unlock(lock); 118 } 119 120 static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) 121 { 122 spinlock_t *lock = NULL; 123 124 if (q->flags & TCQ_F_NOLOCK) { 125 lock = qdisc_lock(q); 126 spin_lock(lock); 127 } 128 129 while (skb) { 130 struct sk_buff *next = skb->next; 131 132 __skb_queue_tail(&q->gso_skb, skb); 133 134 /* it's still part of the queue */ 135 if (qdisc_is_percpu_stats(q)) { 136 qdisc_qstats_cpu_requeues_inc(q); 137 qdisc_qstats_cpu_backlog_inc(q, skb); 138 qdisc_qstats_cpu_qlen_inc(q); 139 } else { 140 q->qstats.requeues++; 141 qdisc_qstats_backlog_inc(q, skb); 142 q->q.qlen++; 143 } 144 145 skb = next; 146 } 147 if (lock) 148 spin_unlock(lock); 149 __netif_schedule(q); 150 } 151 152 static void try_bulk_dequeue_skb(struct Qdisc *q, 153 struct sk_buff *skb, 154 const struct netdev_queue *txq, 155 int *packets) 156 { 157 int bytelimit = qdisc_avail_bulklimit(txq) - skb->len; 158 159 while (bytelimit > 0) { 160 struct sk_buff *nskb = q->dequeue(q); 161 162 if (!nskb) 163 break; 164 165 bytelimit -= nskb->len; /* covers GSO len */ 166 skb->next = nskb; 167 skb = nskb; 168 (*packets)++; /* GSO counts as one pkt */ 169 } 170 skb_mark_not_on_list(skb); 171 } 172 173 /* This variant of try_bulk_dequeue_skb() makes sure 174 * all skbs in the chain are for the same txq 175 */ 176 static void try_bulk_dequeue_skb_slow(struct Qdisc *q, 177 struct sk_buff *skb, 178 int *packets) 179 { 180 int mapping = skb_get_queue_mapping(skb); 181 struct sk_buff *nskb; 182 int cnt = 0; 183 184 do { 185 nskb = q->dequeue(q); 186 if (!nskb) 187 break; 188 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) { 189 qdisc_enqueue_skb_bad_txq(q, nskb); 190 break; 191 } 192 skb->next = nskb; 193 skb = nskb; 194 } while (++cnt < 8); 195 (*packets) += cnt; 196 skb_mark_not_on_list(skb); 197 } 198 199 /* Note that dequeue_skb can possibly return a SKB list (via skb->next). 200 * A requeued skb (via q->gso_skb) can also be a SKB list. 201 */ 202 static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, 203 int *packets) 204 { 205 const struct netdev_queue *txq = q->dev_queue; 206 struct sk_buff *skb = NULL; 207 208 *packets = 1; 209 if (unlikely(!skb_queue_empty(&q->gso_skb))) { 210 spinlock_t *lock = NULL; 211 212 if (q->flags & TCQ_F_NOLOCK) { 213 lock = qdisc_lock(q); 214 spin_lock(lock); 215 } 216 217 skb = skb_peek(&q->gso_skb); 218 219 /* skb may be null if another cpu pulls gso_skb off in between 220 * empty check and lock. 221 */ 222 if (!skb) { 223 if (lock) 224 spin_unlock(lock); 225 goto validate; 226 } 227 228 /* skb in gso_skb were already validated */ 229 *validate = false; 230 if (xfrm_offload(skb)) 231 *validate = true; 232 /* check the reason of requeuing without tx lock first */ 233 txq = skb_get_tx_queue(txq->dev, skb); 234 if (!netif_xmit_frozen_or_stopped(txq)) { 235 skb = __skb_dequeue(&q->gso_skb); 236 if (qdisc_is_percpu_stats(q)) { 237 qdisc_qstats_cpu_backlog_dec(q, skb); 238 qdisc_qstats_cpu_qlen_dec(q); 239 } else { 240 qdisc_qstats_backlog_dec(q, skb); 241 q->q.qlen--; 242 } 243 } else { 244 skb = NULL; 245 } 246 if (lock) 247 spin_unlock(lock); 248 goto trace; 249 } 250 validate: 251 *validate = true; 252 253 if ((q->flags & TCQ_F_ONETXQUEUE) && 254 netif_xmit_frozen_or_stopped(txq)) 255 return skb; 256 257 skb = qdisc_dequeue_skb_bad_txq(q); 258 if (unlikely(skb)) { 259 if (skb == SKB_XOFF_MAGIC) 260 return NULL; 261 goto bulk; 262 } 263 skb = q->dequeue(q); 264 if (skb) { 265 bulk: 266 if (qdisc_may_bulk(q)) 267 try_bulk_dequeue_skb(q, skb, txq, packets); 268 else 269 try_bulk_dequeue_skb_slow(q, skb, packets); 270 } 271 trace: 272 trace_qdisc_dequeue(q, txq, *packets, skb); 273 return skb; 274 } 275 276 /* 277 * Transmit possibly several skbs, and handle the return status as 278 * required. Owning running seqcount bit guarantees that 279 * only one CPU can execute this function. 280 * 281 * Returns to the caller: 282 * false - hardware queue frozen backoff 283 * true - feel free to send more pkts 284 */ 285 bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, 286 struct net_device *dev, struct netdev_queue *txq, 287 spinlock_t *root_lock, bool validate) 288 { 289 int ret = NETDEV_TX_BUSY; 290 bool again = false; 291 292 /* And release qdisc */ 293 if (root_lock) 294 spin_unlock(root_lock); 295 296 /* Note that we validate skb (GSO, checksum, ...) outside of locks */ 297 if (validate) 298 skb = validate_xmit_skb_list(skb, dev, &again); 299 300 #ifdef CONFIG_XFRM_OFFLOAD 301 if (unlikely(again)) { 302 if (root_lock) 303 spin_lock(root_lock); 304 305 dev_requeue_skb(skb, q); 306 return false; 307 } 308 #endif 309 310 if (likely(skb)) { 311 HARD_TX_LOCK(dev, txq, smp_processor_id()); 312 if (!netif_xmit_frozen_or_stopped(txq)) 313 skb = dev_hard_start_xmit(skb, dev, txq, &ret); 314 315 HARD_TX_UNLOCK(dev, txq); 316 } else { 317 if (root_lock) 318 spin_lock(root_lock); 319 return true; 320 } 321 322 if (root_lock) 323 spin_lock(root_lock); 324 325 if (!dev_xmit_complete(ret)) { 326 /* Driver returned NETDEV_TX_BUSY - requeue skb */ 327 if (unlikely(ret != NETDEV_TX_BUSY)) 328 net_warn_ratelimited("BUG %s code %d qlen %d\n", 329 dev->name, ret, q->q.qlen); 330 331 dev_requeue_skb(skb, q); 332 return false; 333 } 334 335 return true; 336 } 337 338 /* 339 * NOTE: Called under qdisc_lock(q) with locally disabled BH. 340 * 341 * running seqcount guarantees only one CPU can process 342 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for 343 * this queue. 344 * 345 * netif_tx_lock serializes accesses to device driver. 346 * 347 * qdisc_lock(q) and netif_tx_lock are mutually exclusive, 348 * if one is grabbed, another must be free. 349 * 350 * Note, that this procedure can be called by a watchdog timer 351 * 352 * Returns to the caller: 353 * 0 - queue is empty or throttled. 354 * >0 - queue is not empty. 355 * 356 */ 357 static inline bool qdisc_restart(struct Qdisc *q, int *packets) 358 { 359 spinlock_t *root_lock = NULL; 360 struct netdev_queue *txq; 361 struct net_device *dev; 362 struct sk_buff *skb; 363 bool validate; 364 365 /* Dequeue packet */ 366 skb = dequeue_skb(q, &validate, packets); 367 if (unlikely(!skb)) 368 return false; 369 370 if (!(q->flags & TCQ_F_NOLOCK)) 371 root_lock = qdisc_lock(q); 372 373 dev = qdisc_dev(q); 374 txq = skb_get_tx_queue(dev, skb); 375 376 return sch_direct_xmit(skb, q, dev, txq, root_lock, validate); 377 } 378 379 void __qdisc_run(struct Qdisc *q) 380 { 381 int quota = dev_tx_weight; 382 int packets; 383 384 while (qdisc_restart(q, &packets)) { 385 /* 386 * Ordered by possible occurrence: Postpone processing if 387 * 1. we've exceeded packet quota 388 * 2. another process needs the CPU; 389 */ 390 quota -= packets; 391 if (quota <= 0 || need_resched()) { 392 __netif_schedule(q); 393 break; 394 } 395 } 396 } 397 398 unsigned long dev_trans_start(struct net_device *dev) 399 { 400 unsigned long val, res; 401 unsigned int i; 402 403 if (is_vlan_dev(dev)) 404 dev = vlan_dev_real_dev(dev); 405 else if (netif_is_macvlan(dev)) 406 dev = macvlan_dev_real_dev(dev); 407 res = netdev_get_tx_queue(dev, 0)->trans_start; 408 for (i = 1; i < dev->num_tx_queues; i++) { 409 val = netdev_get_tx_queue(dev, i)->trans_start; 410 if (val && time_after(val, res)) 411 res = val; 412 } 413 414 return res; 415 } 416 EXPORT_SYMBOL(dev_trans_start); 417 418 static void dev_watchdog(struct timer_list *t) 419 { 420 struct net_device *dev = from_timer(dev, t, watchdog_timer); 421 422 netif_tx_lock(dev); 423 if (!qdisc_tx_is_noop(dev)) { 424 if (netif_device_present(dev) && 425 netif_running(dev) && 426 netif_carrier_ok(dev)) { 427 int some_queue_timedout = 0; 428 unsigned int i; 429 unsigned long trans_start; 430 431 for (i = 0; i < dev->num_tx_queues; i++) { 432 struct netdev_queue *txq; 433 434 txq = netdev_get_tx_queue(dev, i); 435 trans_start = txq->trans_start; 436 if (netif_xmit_stopped(txq) && 437 time_after(jiffies, (trans_start + 438 dev->watchdog_timeo))) { 439 some_queue_timedout = 1; 440 txq->trans_timeout++; 441 break; 442 } 443 } 444 445 if (some_queue_timedout) { 446 trace_net_dev_xmit_timeout(dev, i); 447 WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n", 448 dev->name, netdev_drivername(dev), i); 449 dev->netdev_ops->ndo_tx_timeout(dev); 450 } 451 if (!mod_timer(&dev->watchdog_timer, 452 round_jiffies(jiffies + 453 dev->watchdog_timeo))) 454 dev_hold(dev); 455 } 456 } 457 netif_tx_unlock(dev); 458 459 dev_put(dev); 460 } 461 462 void __netdev_watchdog_up(struct net_device *dev) 463 { 464 if (dev->netdev_ops->ndo_tx_timeout) { 465 if (dev->watchdog_timeo <= 0) 466 dev->watchdog_timeo = 5*HZ; 467 if (!mod_timer(&dev->watchdog_timer, 468 round_jiffies(jiffies + dev->watchdog_timeo))) 469 dev_hold(dev); 470 } 471 } 472 473 static void dev_watchdog_up(struct net_device *dev) 474 { 475 __netdev_watchdog_up(dev); 476 } 477 478 static void dev_watchdog_down(struct net_device *dev) 479 { 480 netif_tx_lock_bh(dev); 481 if (del_timer(&dev->watchdog_timer)) 482 dev_put(dev); 483 netif_tx_unlock_bh(dev); 484 } 485 486 /** 487 * netif_carrier_on - set carrier 488 * @dev: network device 489 * 490 * Device has detected acquisition of carrier. 491 */ 492 void netif_carrier_on(struct net_device *dev) 493 { 494 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) { 495 if (dev->reg_state == NETREG_UNINITIALIZED) 496 return; 497 atomic_inc(&dev->carrier_up_count); 498 linkwatch_fire_event(dev); 499 if (netif_running(dev)) 500 __netdev_watchdog_up(dev); 501 } 502 } 503 EXPORT_SYMBOL(netif_carrier_on); 504 505 /** 506 * netif_carrier_off - clear carrier 507 * @dev: network device 508 * 509 * Device has detected loss of carrier. 510 */ 511 void netif_carrier_off(struct net_device *dev) 512 { 513 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) { 514 if (dev->reg_state == NETREG_UNINITIALIZED) 515 return; 516 atomic_inc(&dev->carrier_down_count); 517 linkwatch_fire_event(dev); 518 } 519 } 520 EXPORT_SYMBOL(netif_carrier_off); 521 522 /* "NOOP" scheduler: the best scheduler, recommended for all interfaces 523 under all circumstances. It is difficult to invent anything faster or 524 cheaper. 525 */ 526 527 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, 528 struct sk_buff **to_free) 529 { 530 __qdisc_drop(skb, to_free); 531 return NET_XMIT_CN; 532 } 533 534 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc) 535 { 536 return NULL; 537 } 538 539 struct Qdisc_ops noop_qdisc_ops __read_mostly = { 540 .id = "noop", 541 .priv_size = 0, 542 .enqueue = noop_enqueue, 543 .dequeue = noop_dequeue, 544 .peek = noop_dequeue, 545 .owner = THIS_MODULE, 546 }; 547 548 static struct netdev_queue noop_netdev_queue = { 549 RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc), 550 .qdisc_sleeping = &noop_qdisc, 551 }; 552 553 struct Qdisc noop_qdisc = { 554 .enqueue = noop_enqueue, 555 .dequeue = noop_dequeue, 556 .flags = TCQ_F_BUILTIN, 557 .ops = &noop_qdisc_ops, 558 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), 559 .dev_queue = &noop_netdev_queue, 560 .running = SEQCNT_ZERO(noop_qdisc.running), 561 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock), 562 .gso_skb = { 563 .next = (struct sk_buff *)&noop_qdisc.gso_skb, 564 .prev = (struct sk_buff *)&noop_qdisc.gso_skb, 565 .qlen = 0, 566 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock), 567 }, 568 .skb_bad_txq = { 569 .next = (struct sk_buff *)&noop_qdisc.skb_bad_txq, 570 .prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq, 571 .qlen = 0, 572 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock), 573 }, 574 }; 575 EXPORT_SYMBOL(noop_qdisc); 576 577 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt, 578 struct netlink_ext_ack *extack) 579 { 580 /* register_qdisc() assigns a default of noop_enqueue if unset, 581 * but __dev_queue_xmit() treats noqueue only as such 582 * if this is NULL - so clear it here. */ 583 qdisc->enqueue = NULL; 584 return 0; 585 } 586 587 struct Qdisc_ops noqueue_qdisc_ops __read_mostly = { 588 .id = "noqueue", 589 .priv_size = 0, 590 .init = noqueue_init, 591 .enqueue = noop_enqueue, 592 .dequeue = noop_dequeue, 593 .peek = noop_dequeue, 594 .owner = THIS_MODULE, 595 }; 596 597 static const u8 prio2band[TC_PRIO_MAX + 1] = { 598 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 599 }; 600 601 /* 3-band FIFO queue: old style, but should be a bit faster than 602 generic prio+fifo combination. 603 */ 604 605 #define PFIFO_FAST_BANDS 3 606 607 /* 608 * Private data for a pfifo_fast scheduler containing: 609 * - rings for priority bands 610 */ 611 struct pfifo_fast_priv { 612 struct skb_array q[PFIFO_FAST_BANDS]; 613 }; 614 615 static inline struct skb_array *band2list(struct pfifo_fast_priv *priv, 616 int band) 617 { 618 return &priv->q[band]; 619 } 620 621 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, 622 struct sk_buff **to_free) 623 { 624 int band = prio2band[skb->priority & TC_PRIO_MAX]; 625 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 626 struct skb_array *q = band2list(priv, band); 627 unsigned int pkt_len = qdisc_pkt_len(skb); 628 int err; 629 630 err = skb_array_produce(q, skb); 631 632 if (unlikely(err)) { 633 if (qdisc_is_percpu_stats(qdisc)) 634 return qdisc_drop_cpu(skb, qdisc, to_free); 635 else 636 return qdisc_drop(skb, qdisc, to_free); 637 } 638 639 qdisc_update_stats_at_enqueue(qdisc, pkt_len); 640 return NET_XMIT_SUCCESS; 641 } 642 643 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc) 644 { 645 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 646 struct sk_buff *skb = NULL; 647 int band; 648 649 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) { 650 struct skb_array *q = band2list(priv, band); 651 652 if (__skb_array_empty(q)) 653 continue; 654 655 skb = __skb_array_consume(q); 656 } 657 if (likely(skb)) { 658 qdisc_update_stats_at_dequeue(qdisc, skb); 659 } else { 660 qdisc->empty = true; 661 } 662 663 return skb; 664 } 665 666 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc) 667 { 668 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 669 struct sk_buff *skb = NULL; 670 int band; 671 672 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) { 673 struct skb_array *q = band2list(priv, band); 674 675 skb = __skb_array_peek(q); 676 } 677 678 return skb; 679 } 680 681 static void pfifo_fast_reset(struct Qdisc *qdisc) 682 { 683 int i, band; 684 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 685 686 for (band = 0; band < PFIFO_FAST_BANDS; band++) { 687 struct skb_array *q = band2list(priv, band); 688 struct sk_buff *skb; 689 690 /* NULL ring is possible if destroy path is due to a failed 691 * skb_array_init() in pfifo_fast_init() case. 692 */ 693 if (!q->ring.queue) 694 continue; 695 696 while ((skb = __skb_array_consume(q)) != NULL) 697 kfree_skb(skb); 698 } 699 700 if (qdisc_is_percpu_stats(qdisc)) { 701 for_each_possible_cpu(i) { 702 struct gnet_stats_queue *q; 703 704 q = per_cpu_ptr(qdisc->cpu_qstats, i); 705 q->backlog = 0; 706 q->qlen = 0; 707 } 708 } 709 } 710 711 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb) 712 { 713 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS }; 714 715 memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1); 716 if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt)) 717 goto nla_put_failure; 718 return skb->len; 719 720 nla_put_failure: 721 return -1; 722 } 723 724 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt, 725 struct netlink_ext_ack *extack) 726 { 727 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len; 728 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 729 int prio; 730 731 /* guard against zero length rings */ 732 if (!qlen) 733 return -EINVAL; 734 735 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { 736 struct skb_array *q = band2list(priv, prio); 737 int err; 738 739 err = skb_array_init(q, qlen, GFP_KERNEL); 740 if (err) 741 return -ENOMEM; 742 } 743 744 /* Can by-pass the queue discipline */ 745 qdisc->flags |= TCQ_F_CAN_BYPASS; 746 return 0; 747 } 748 749 static void pfifo_fast_destroy(struct Qdisc *sch) 750 { 751 struct pfifo_fast_priv *priv = qdisc_priv(sch); 752 int prio; 753 754 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { 755 struct skb_array *q = band2list(priv, prio); 756 757 /* NULL ring is possible if destroy path is due to a failed 758 * skb_array_init() in pfifo_fast_init() case. 759 */ 760 if (!q->ring.queue) 761 continue; 762 /* Destroy ring but no need to kfree_skb because a call to 763 * pfifo_fast_reset() has already done that work. 764 */ 765 ptr_ring_cleanup(&q->ring, NULL); 766 } 767 } 768 769 static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch, 770 unsigned int new_len) 771 { 772 struct pfifo_fast_priv *priv = qdisc_priv(sch); 773 struct skb_array *bands[PFIFO_FAST_BANDS]; 774 int prio; 775 776 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) { 777 struct skb_array *q = band2list(priv, prio); 778 779 bands[prio] = q; 780 } 781 782 return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len, 783 GFP_KERNEL); 784 } 785 786 struct Qdisc_ops pfifo_fast_ops __read_mostly = { 787 .id = "pfifo_fast", 788 .priv_size = sizeof(struct pfifo_fast_priv), 789 .enqueue = pfifo_fast_enqueue, 790 .dequeue = pfifo_fast_dequeue, 791 .peek = pfifo_fast_peek, 792 .init = pfifo_fast_init, 793 .destroy = pfifo_fast_destroy, 794 .reset = pfifo_fast_reset, 795 .dump = pfifo_fast_dump, 796 .change_tx_queue_len = pfifo_fast_change_tx_queue_len, 797 .owner = THIS_MODULE, 798 .static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS, 799 }; 800 EXPORT_SYMBOL(pfifo_fast_ops); 801 802 static struct lock_class_key qdisc_tx_busylock; 803 static struct lock_class_key qdisc_running_key; 804 805 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 806 const struct Qdisc_ops *ops, 807 struct netlink_ext_ack *extack) 808 { 809 void *p; 810 struct Qdisc *sch; 811 unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size; 812 int err = -ENOBUFS; 813 struct net_device *dev; 814 815 if (!dev_queue) { 816 NL_SET_ERR_MSG(extack, "No device queue given"); 817 err = -EINVAL; 818 goto errout; 819 } 820 821 dev = dev_queue->dev; 822 p = kzalloc_node(size, GFP_KERNEL, 823 netdev_queue_numa_node_read(dev_queue)); 824 825 if (!p) 826 goto errout; 827 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); 828 /* if we got non aligned memory, ask more and do alignment ourself */ 829 if (sch != p) { 830 kfree(p); 831 p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL, 832 netdev_queue_numa_node_read(dev_queue)); 833 if (!p) 834 goto errout; 835 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); 836 sch->padded = (char *) sch - (char *) p; 837 } 838 __skb_queue_head_init(&sch->gso_skb); 839 __skb_queue_head_init(&sch->skb_bad_txq); 840 qdisc_skb_head_init(&sch->q); 841 spin_lock_init(&sch->q.lock); 842 843 if (ops->static_flags & TCQ_F_CPUSTATS) { 844 sch->cpu_bstats = 845 netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu); 846 if (!sch->cpu_bstats) 847 goto errout1; 848 849 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue); 850 if (!sch->cpu_qstats) { 851 free_percpu(sch->cpu_bstats); 852 goto errout1; 853 } 854 } 855 856 spin_lock_init(&sch->busylock); 857 lockdep_set_class(&sch->busylock, 858 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock); 859 860 /* seqlock has the same scope of busylock, for NOLOCK qdisc */ 861 spin_lock_init(&sch->seqlock); 862 lockdep_set_class(&sch->busylock, 863 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock); 864 865 seqcount_init(&sch->running); 866 lockdep_set_class(&sch->running, 867 dev->qdisc_running_key ?: &qdisc_running_key); 868 869 sch->ops = ops; 870 sch->flags = ops->static_flags; 871 sch->enqueue = ops->enqueue; 872 sch->dequeue = ops->dequeue; 873 sch->dev_queue = dev_queue; 874 sch->empty = true; 875 dev_hold(dev); 876 refcount_set(&sch->refcnt, 1); 877 878 return sch; 879 errout1: 880 kfree(p); 881 errout: 882 return ERR_PTR(err); 883 } 884 885 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 886 const struct Qdisc_ops *ops, 887 unsigned int parentid, 888 struct netlink_ext_ack *extack) 889 { 890 struct Qdisc *sch; 891 892 if (!try_module_get(ops->owner)) { 893 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter"); 894 return NULL; 895 } 896 897 sch = qdisc_alloc(dev_queue, ops, extack); 898 if (IS_ERR(sch)) { 899 module_put(ops->owner); 900 return NULL; 901 } 902 sch->parent = parentid; 903 904 if (!ops->init || ops->init(sch, NULL, extack) == 0) 905 return sch; 906 907 qdisc_put(sch); 908 return NULL; 909 } 910 EXPORT_SYMBOL(qdisc_create_dflt); 911 912 /* Under qdisc_lock(qdisc) and BH! */ 913 914 void qdisc_reset(struct Qdisc *qdisc) 915 { 916 const struct Qdisc_ops *ops = qdisc->ops; 917 struct sk_buff *skb, *tmp; 918 919 if (ops->reset) 920 ops->reset(qdisc); 921 922 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) { 923 __skb_unlink(skb, &qdisc->gso_skb); 924 kfree_skb_list(skb); 925 } 926 927 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) { 928 __skb_unlink(skb, &qdisc->skb_bad_txq); 929 kfree_skb_list(skb); 930 } 931 932 qdisc->q.qlen = 0; 933 qdisc->qstats.backlog = 0; 934 } 935 EXPORT_SYMBOL(qdisc_reset); 936 937 void qdisc_free(struct Qdisc *qdisc) 938 { 939 if (qdisc_is_percpu_stats(qdisc)) { 940 free_percpu(qdisc->cpu_bstats); 941 free_percpu(qdisc->cpu_qstats); 942 } 943 944 kfree((char *) qdisc - qdisc->padded); 945 } 946 947 static void qdisc_free_cb(struct rcu_head *head) 948 { 949 struct Qdisc *q = container_of(head, struct Qdisc, rcu); 950 951 qdisc_free(q); 952 } 953 954 static void qdisc_destroy(struct Qdisc *qdisc) 955 { 956 const struct Qdisc_ops *ops = qdisc->ops; 957 struct sk_buff *skb, *tmp; 958 959 #ifdef CONFIG_NET_SCHED 960 qdisc_hash_del(qdisc); 961 962 qdisc_put_stab(rtnl_dereference(qdisc->stab)); 963 #endif 964 gen_kill_estimator(&qdisc->rate_est); 965 if (ops->reset) 966 ops->reset(qdisc); 967 if (ops->destroy) 968 ops->destroy(qdisc); 969 970 module_put(ops->owner); 971 dev_put(qdisc_dev(qdisc)); 972 973 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) { 974 __skb_unlink(skb, &qdisc->gso_skb); 975 kfree_skb_list(skb); 976 } 977 978 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) { 979 __skb_unlink(skb, &qdisc->skb_bad_txq); 980 kfree_skb_list(skb); 981 } 982 983 call_rcu(&qdisc->rcu, qdisc_free_cb); 984 } 985 986 void qdisc_put(struct Qdisc *qdisc) 987 { 988 if (!qdisc) 989 return; 990 991 if (qdisc->flags & TCQ_F_BUILTIN || 992 !refcount_dec_and_test(&qdisc->refcnt)) 993 return; 994 995 qdisc_destroy(qdisc); 996 } 997 EXPORT_SYMBOL(qdisc_put); 998 999 /* Version of qdisc_put() that is called with rtnl mutex unlocked. 1000 * Intended to be used as optimization, this function only takes rtnl lock if 1001 * qdisc reference counter reached zero. 1002 */ 1003 1004 void qdisc_put_unlocked(struct Qdisc *qdisc) 1005 { 1006 if (qdisc->flags & TCQ_F_BUILTIN || 1007 !refcount_dec_and_rtnl_lock(&qdisc->refcnt)) 1008 return; 1009 1010 qdisc_destroy(qdisc); 1011 rtnl_unlock(); 1012 } 1013 EXPORT_SYMBOL(qdisc_put_unlocked); 1014 1015 /* Attach toplevel qdisc to device queue. */ 1016 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 1017 struct Qdisc *qdisc) 1018 { 1019 struct Qdisc *oqdisc = dev_queue->qdisc_sleeping; 1020 spinlock_t *root_lock; 1021 1022 root_lock = qdisc_lock(oqdisc); 1023 spin_lock_bh(root_lock); 1024 1025 /* ... and graft new one */ 1026 if (qdisc == NULL) 1027 qdisc = &noop_qdisc; 1028 dev_queue->qdisc_sleeping = qdisc; 1029 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc); 1030 1031 spin_unlock_bh(root_lock); 1032 1033 return oqdisc; 1034 } 1035 EXPORT_SYMBOL(dev_graft_qdisc); 1036 1037 static void attach_one_default_qdisc(struct net_device *dev, 1038 struct netdev_queue *dev_queue, 1039 void *_unused) 1040 { 1041 struct Qdisc *qdisc; 1042 const struct Qdisc_ops *ops = default_qdisc_ops; 1043 1044 if (dev->priv_flags & IFF_NO_QUEUE) 1045 ops = &noqueue_qdisc_ops; 1046 1047 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL); 1048 if (!qdisc) { 1049 netdev_info(dev, "activation failed\n"); 1050 return; 1051 } 1052 if (!netif_is_multiqueue(dev)) 1053 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; 1054 dev_queue->qdisc_sleeping = qdisc; 1055 } 1056 1057 static void attach_default_qdiscs(struct net_device *dev) 1058 { 1059 struct netdev_queue *txq; 1060 struct Qdisc *qdisc; 1061 1062 txq = netdev_get_tx_queue(dev, 0); 1063 1064 if (!netif_is_multiqueue(dev) || 1065 dev->priv_flags & IFF_NO_QUEUE) { 1066 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL); 1067 dev->qdisc = txq->qdisc_sleeping; 1068 qdisc_refcount_inc(dev->qdisc); 1069 } else { 1070 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL); 1071 if (qdisc) { 1072 dev->qdisc = qdisc; 1073 qdisc->ops->attach(qdisc); 1074 } 1075 } 1076 #ifdef CONFIG_NET_SCHED 1077 if (dev->qdisc != &noop_qdisc) 1078 qdisc_hash_add(dev->qdisc, false); 1079 #endif 1080 } 1081 1082 static void transition_one_qdisc(struct net_device *dev, 1083 struct netdev_queue *dev_queue, 1084 void *_need_watchdog) 1085 { 1086 struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping; 1087 int *need_watchdog_p = _need_watchdog; 1088 1089 if (!(new_qdisc->flags & TCQ_F_BUILTIN)) 1090 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state); 1091 1092 rcu_assign_pointer(dev_queue->qdisc, new_qdisc); 1093 if (need_watchdog_p) { 1094 dev_queue->trans_start = 0; 1095 *need_watchdog_p = 1; 1096 } 1097 } 1098 1099 void dev_activate(struct net_device *dev) 1100 { 1101 int need_watchdog; 1102 1103 /* No queueing discipline is attached to device; 1104 * create default one for devices, which need queueing 1105 * and noqueue_qdisc for virtual interfaces 1106 */ 1107 1108 if (dev->qdisc == &noop_qdisc) 1109 attach_default_qdiscs(dev); 1110 1111 if (!netif_carrier_ok(dev)) 1112 /* Delay activation until next carrier-on event */ 1113 return; 1114 1115 need_watchdog = 0; 1116 netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog); 1117 if (dev_ingress_queue(dev)) 1118 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL); 1119 1120 if (need_watchdog) { 1121 netif_trans_update(dev); 1122 dev_watchdog_up(dev); 1123 } 1124 } 1125 EXPORT_SYMBOL(dev_activate); 1126 1127 static void dev_deactivate_queue(struct net_device *dev, 1128 struct netdev_queue *dev_queue, 1129 void *_qdisc_default) 1130 { 1131 struct Qdisc *qdisc_default = _qdisc_default; 1132 struct Qdisc *qdisc; 1133 1134 qdisc = rtnl_dereference(dev_queue->qdisc); 1135 if (qdisc) { 1136 bool nolock = qdisc->flags & TCQ_F_NOLOCK; 1137 1138 if (nolock) 1139 spin_lock_bh(&qdisc->seqlock); 1140 spin_lock_bh(qdisc_lock(qdisc)); 1141 1142 if (!(qdisc->flags & TCQ_F_BUILTIN)) 1143 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state); 1144 1145 rcu_assign_pointer(dev_queue->qdisc, qdisc_default); 1146 qdisc_reset(qdisc); 1147 1148 spin_unlock_bh(qdisc_lock(qdisc)); 1149 if (nolock) 1150 spin_unlock_bh(&qdisc->seqlock); 1151 } 1152 } 1153 1154 static bool some_qdisc_is_busy(struct net_device *dev) 1155 { 1156 unsigned int i; 1157 1158 for (i = 0; i < dev->num_tx_queues; i++) { 1159 struct netdev_queue *dev_queue; 1160 spinlock_t *root_lock; 1161 struct Qdisc *q; 1162 int val; 1163 1164 dev_queue = netdev_get_tx_queue(dev, i); 1165 q = dev_queue->qdisc_sleeping; 1166 1167 root_lock = qdisc_lock(q); 1168 spin_lock_bh(root_lock); 1169 1170 val = (qdisc_is_running(q) || 1171 test_bit(__QDISC_STATE_SCHED, &q->state)); 1172 1173 spin_unlock_bh(root_lock); 1174 1175 if (val) 1176 return true; 1177 } 1178 return false; 1179 } 1180 1181 static void dev_qdisc_reset(struct net_device *dev, 1182 struct netdev_queue *dev_queue, 1183 void *none) 1184 { 1185 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 1186 1187 if (qdisc) 1188 qdisc_reset(qdisc); 1189 } 1190 1191 /** 1192 * dev_deactivate_many - deactivate transmissions on several devices 1193 * @head: list of devices to deactivate 1194 * 1195 * This function returns only when all outstanding transmissions 1196 * have completed, unless all devices are in dismantle phase. 1197 */ 1198 void dev_deactivate_many(struct list_head *head) 1199 { 1200 struct net_device *dev; 1201 1202 list_for_each_entry(dev, head, close_list) { 1203 netdev_for_each_tx_queue(dev, dev_deactivate_queue, 1204 &noop_qdisc); 1205 if (dev_ingress_queue(dev)) 1206 dev_deactivate_queue(dev, dev_ingress_queue(dev), 1207 &noop_qdisc); 1208 1209 dev_watchdog_down(dev); 1210 } 1211 1212 /* Wait for outstanding qdisc-less dev_queue_xmit calls. 1213 * This is avoided if all devices are in dismantle phase : 1214 * Caller will call synchronize_net() for us 1215 */ 1216 synchronize_net(); 1217 1218 /* Wait for outstanding qdisc_run calls. */ 1219 list_for_each_entry(dev, head, close_list) { 1220 while (some_qdisc_is_busy(dev)) 1221 yield(); 1222 /* The new qdisc is assigned at this point so we can safely 1223 * unwind stale skb lists and qdisc statistics 1224 */ 1225 netdev_for_each_tx_queue(dev, dev_qdisc_reset, NULL); 1226 if (dev_ingress_queue(dev)) 1227 dev_qdisc_reset(dev, dev_ingress_queue(dev), NULL); 1228 } 1229 } 1230 1231 void dev_deactivate(struct net_device *dev) 1232 { 1233 LIST_HEAD(single); 1234 1235 list_add(&dev->close_list, &single); 1236 dev_deactivate_many(&single); 1237 list_del(&single); 1238 } 1239 EXPORT_SYMBOL(dev_deactivate); 1240 1241 static int qdisc_change_tx_queue_len(struct net_device *dev, 1242 struct netdev_queue *dev_queue) 1243 { 1244 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 1245 const struct Qdisc_ops *ops = qdisc->ops; 1246 1247 if (ops->change_tx_queue_len) 1248 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len); 1249 return 0; 1250 } 1251 1252 int dev_qdisc_change_tx_queue_len(struct net_device *dev) 1253 { 1254 bool up = dev->flags & IFF_UP; 1255 unsigned int i; 1256 int ret = 0; 1257 1258 if (up) 1259 dev_deactivate(dev); 1260 1261 for (i = 0; i < dev->num_tx_queues; i++) { 1262 ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]); 1263 1264 /* TODO: revert changes on a partial failure */ 1265 if (ret) 1266 break; 1267 } 1268 1269 if (up) 1270 dev_activate(dev); 1271 return ret; 1272 } 1273 1274 static void dev_init_scheduler_queue(struct net_device *dev, 1275 struct netdev_queue *dev_queue, 1276 void *_qdisc) 1277 { 1278 struct Qdisc *qdisc = _qdisc; 1279 1280 rcu_assign_pointer(dev_queue->qdisc, qdisc); 1281 dev_queue->qdisc_sleeping = qdisc; 1282 } 1283 1284 void dev_init_scheduler(struct net_device *dev) 1285 { 1286 dev->qdisc = &noop_qdisc; 1287 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc); 1288 if (dev_ingress_queue(dev)) 1289 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc); 1290 1291 timer_setup(&dev->watchdog_timer, dev_watchdog, 0); 1292 } 1293 1294 static void shutdown_scheduler_queue(struct net_device *dev, 1295 struct netdev_queue *dev_queue, 1296 void *_qdisc_default) 1297 { 1298 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 1299 struct Qdisc *qdisc_default = _qdisc_default; 1300 1301 if (qdisc) { 1302 rcu_assign_pointer(dev_queue->qdisc, qdisc_default); 1303 dev_queue->qdisc_sleeping = qdisc_default; 1304 1305 qdisc_put(qdisc); 1306 } 1307 } 1308 1309 void dev_shutdown(struct net_device *dev) 1310 { 1311 netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); 1312 if (dev_ingress_queue(dev)) 1313 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc); 1314 qdisc_put(dev->qdisc); 1315 dev->qdisc = &noop_qdisc; 1316 1317 WARN_ON(timer_pending(&dev->watchdog_timer)); 1318 } 1319 1320 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1321 const struct tc_ratespec *conf, 1322 u64 rate64) 1323 { 1324 memset(r, 0, sizeof(*r)); 1325 r->overhead = conf->overhead; 1326 r->rate_bytes_ps = max_t(u64, conf->rate, rate64); 1327 r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK); 1328 r->mult = 1; 1329 /* 1330 * The deal here is to replace a divide by a reciprocal one 1331 * in fast path (a reciprocal divide is a multiply and a shift) 1332 * 1333 * Normal formula would be : 1334 * time_in_ns = (NSEC_PER_SEC * len) / rate_bps 1335 * 1336 * We compute mult/shift to use instead : 1337 * time_in_ns = (len * mult) >> shift; 1338 * 1339 * We try to get the highest possible mult value for accuracy, 1340 * but have to make sure no overflows will ever happen. 1341 */ 1342 if (r->rate_bytes_ps > 0) { 1343 u64 factor = NSEC_PER_SEC; 1344 1345 for (;;) { 1346 r->mult = div64_u64(factor, r->rate_bytes_ps); 1347 if (r->mult & (1U << 31) || factor & (1ULL << 63)) 1348 break; 1349 factor <<= 1; 1350 r->shift++; 1351 } 1352 } 1353 } 1354 EXPORT_SYMBOL(psched_ratecfg_precompute); 1355 1356 static void mini_qdisc_rcu_func(struct rcu_head *head) 1357 { 1358 } 1359 1360 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1361 struct tcf_proto *tp_head) 1362 { 1363 /* Protected with chain0->filter_chain_lock. 1364 * Can't access chain directly because tp_head can be NULL. 1365 */ 1366 struct mini_Qdisc *miniq_old = 1367 rcu_dereference_protected(*miniqp->p_miniq, 1); 1368 struct mini_Qdisc *miniq; 1369 1370 if (!tp_head) { 1371 RCU_INIT_POINTER(*miniqp->p_miniq, NULL); 1372 /* Wait for flying RCU callback before it is freed. */ 1373 rcu_barrier(); 1374 return; 1375 } 1376 1377 miniq = !miniq_old || miniq_old == &miniqp->miniq2 ? 1378 &miniqp->miniq1 : &miniqp->miniq2; 1379 1380 /* We need to make sure that readers won't see the miniq 1381 * we are about to modify. So wait until previous call_rcu callback 1382 * is done. 1383 */ 1384 rcu_barrier(); 1385 miniq->filter_list = tp_head; 1386 rcu_assign_pointer(*miniqp->p_miniq, miniq); 1387 1388 if (miniq_old) 1389 /* This is counterpart of the rcu barriers above. We need to 1390 * block potential new user of miniq_old until all readers 1391 * are not seeing it. 1392 */ 1393 call_rcu(&miniq_old->rcu, mini_qdisc_rcu_func); 1394 } 1395 EXPORT_SYMBOL(mini_qdisc_pair_swap); 1396 1397 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1398 struct mini_Qdisc __rcu **p_miniq) 1399 { 1400 miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats; 1401 miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats; 1402 miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats; 1403 miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats; 1404 miniqp->p_miniq = p_miniq; 1405 } 1406 EXPORT_SYMBOL(mini_qdisc_pair_init); 1407