1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <linux/mutex.h> 16 #include <linux/rwsem.h> 17 #include <linux/atomic.h> 18 #include <linux/hashtable.h> 19 #include <net/gen_stats.h> 20 #include <net/rtnetlink.h> 21 #include <net/flow_offload.h> 22 #include <linux/xarray.h> 23 24 struct Qdisc_ops; 25 struct qdisc_walker; 26 struct tcf_walker; 27 struct module; 28 struct bpf_flow_keys; 29 30 struct qdisc_rate_table { 31 struct tc_ratespec rate; 32 u32 data[256]; 33 struct qdisc_rate_table *next; 34 int refcnt; 35 }; 36 37 enum qdisc_state_t { 38 __QDISC_STATE_SCHED, 39 __QDISC_STATE_DEACTIVATED, 40 __QDISC_STATE_MISSED, 41 __QDISC_STATE_DRAINING, 42 }; 43 44 #define QDISC_STATE_MISSED BIT(__QDISC_STATE_MISSED) 45 #define QDISC_STATE_DRAINING BIT(__QDISC_STATE_DRAINING) 46 47 #define QDISC_STATE_NON_EMPTY (QDISC_STATE_MISSED | \ 48 QDISC_STATE_DRAINING) 49 50 struct qdisc_size_table { 51 struct rcu_head rcu; 52 struct list_head list; 53 struct tc_sizespec szopts; 54 int refcnt; 55 u16 data[]; 56 }; 57 58 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 59 struct qdisc_skb_head { 60 struct sk_buff *head; 61 struct sk_buff *tail; 62 __u32 qlen; 63 spinlock_t lock; 64 }; 65 66 struct Qdisc { 67 int (*enqueue)(struct sk_buff *skb, 68 struct Qdisc *sch, 69 struct sk_buff **to_free); 70 struct sk_buff * (*dequeue)(struct Qdisc *sch); 71 unsigned int flags; 72 #define TCQ_F_BUILTIN 1 73 #define TCQ_F_INGRESS 2 74 #define TCQ_F_CAN_BYPASS 4 75 #define TCQ_F_MQROOT 8 76 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 77 * q->dev_queue : It can test 78 * netif_xmit_frozen_or_stopped() before 79 * dequeueing next packet. 80 * Its true for MQ/MQPRIO slaves, or non 81 * multiqueue device. 82 */ 83 #define TCQ_F_WARN_NONWC (1 << 16) 84 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 85 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 86 * qdisc_tree_decrease_qlen() should stop. 87 */ 88 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 89 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 90 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 91 u32 limit; 92 const struct Qdisc_ops *ops; 93 struct qdisc_size_table __rcu *stab; 94 struct hlist_node hash; 95 u32 handle; 96 u32 parent; 97 98 struct netdev_queue *dev_queue; 99 100 struct net_rate_estimator __rcu *rate_est; 101 struct gnet_stats_basic_sync __percpu *cpu_bstats; 102 struct gnet_stats_queue __percpu *cpu_qstats; 103 int pad; 104 refcount_t refcnt; 105 106 /* 107 * For performance sake on SMP, we put highly modified fields at the end 108 */ 109 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 110 struct qdisc_skb_head q; 111 struct gnet_stats_basic_sync bstats; 112 struct gnet_stats_queue qstats; 113 bool running; /* must be written under qdisc spinlock */ 114 unsigned long state; 115 struct Qdisc *next_sched; 116 struct sk_buff_head skb_bad_txq; 117 118 atomic_long_t defer_count ____cacheline_aligned_in_smp; 119 struct llist_head defer_list; 120 121 spinlock_t seqlock; 122 123 struct rcu_head rcu; 124 netdevice_tracker dev_tracker; 125 struct lock_class_key root_lock_key; 126 /* private data */ 127 long privdata[] ____cacheline_aligned; 128 }; 129 130 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 131 { 132 if (qdisc->flags & TCQ_F_BUILTIN) 133 return; 134 refcount_inc(&qdisc->refcnt); 135 } 136 137 static inline bool qdisc_refcount_dec_if_one(struct Qdisc *qdisc) 138 { 139 if (qdisc->flags & TCQ_F_BUILTIN) 140 return true; 141 return refcount_dec_if_one(&qdisc->refcnt); 142 } 143 144 /* Intended to be used by unlocked users, when concurrent qdisc release is 145 * possible. 146 */ 147 148 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc) 149 { 150 if (qdisc->flags & TCQ_F_BUILTIN) 151 return qdisc; 152 if (refcount_inc_not_zero(&qdisc->refcnt)) 153 return qdisc; 154 return NULL; 155 } 156 157 /* For !TCQ_F_NOLOCK qdisc: callers must either call this within a qdisc 158 * root_lock section, or provide their own memory barriers -- ordering 159 * against qdisc_run_begin/end() atomic bit operations. 160 */ 161 static inline bool qdisc_is_running(struct Qdisc *qdisc) 162 { 163 if (qdisc->flags & TCQ_F_NOLOCK) 164 return spin_is_locked(&qdisc->seqlock); 165 return READ_ONCE(qdisc->running); 166 } 167 168 static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc) 169 { 170 return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY); 171 } 172 173 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 174 { 175 return q->flags & TCQ_F_CPUSTATS; 176 } 177 178 static inline bool qdisc_is_empty(const struct Qdisc *qdisc) 179 { 180 if (qdisc_is_percpu_stats(qdisc)) 181 return nolock_qdisc_is_empty(qdisc); 182 return !READ_ONCE(qdisc->q.qlen); 183 } 184 185 /* For !TCQ_F_NOLOCK qdisc, qdisc_run_begin/end() must be invoked with 186 * the qdisc root lock acquired. 187 */ 188 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 189 { 190 if (qdisc->flags & TCQ_F_NOLOCK) { 191 if (spin_trylock(&qdisc->seqlock)) 192 return true; 193 194 /* No need to insist if the MISSED flag was already set. 195 * Note that test_and_set_bit() also gives us memory ordering 196 * guarantees wrt potential earlier enqueue() and below 197 * spin_trylock(), both of which are necessary to prevent races 198 */ 199 if (test_and_set_bit(__QDISC_STATE_MISSED, &qdisc->state)) 200 return false; 201 202 /* Try to take the lock again to make sure that we will either 203 * grab it or the CPU that still has it will see MISSED set 204 * when testing it in qdisc_run_end() 205 */ 206 return spin_trylock(&qdisc->seqlock); 207 } 208 if (READ_ONCE(qdisc->running)) 209 return false; 210 WRITE_ONCE(qdisc->running, true); 211 return true; 212 } 213 214 static inline void qdisc_run_end(struct Qdisc *qdisc) 215 { 216 if (qdisc->flags & TCQ_F_NOLOCK) { 217 spin_unlock(&qdisc->seqlock); 218 219 /* spin_unlock() only has store-release semantic. The unlock 220 * and test_bit() ordering is a store-load ordering, so a full 221 * memory barrier is needed here. 222 */ 223 smp_mb(); 224 225 if (unlikely(test_bit(__QDISC_STATE_MISSED, 226 &qdisc->state))) 227 __netif_schedule(qdisc); 228 } else { 229 WRITE_ONCE(qdisc->running, false); 230 } 231 } 232 233 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 234 { 235 return qdisc->flags & TCQ_F_ONETXQUEUE; 236 } 237 238 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 239 { 240 return netdev_queue_dql_avail(txq); 241 } 242 243 struct Qdisc_class_ops { 244 unsigned int flags; 245 /* Child qdisc manipulation */ 246 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 247 int (*graft)(struct Qdisc *, unsigned long cl, 248 struct Qdisc *, struct Qdisc **, 249 struct netlink_ext_ack *extack); 250 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 251 void (*qlen_notify)(struct Qdisc *, unsigned long); 252 253 /* Class manipulation routines */ 254 unsigned long (*find)(struct Qdisc *, u32 classid); 255 int (*change)(struct Qdisc *, u32, u32, 256 struct nlattr **, unsigned long *, 257 struct netlink_ext_ack *); 258 int (*delete)(struct Qdisc *, unsigned long, 259 struct netlink_ext_ack *); 260 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 261 262 /* Filter manipulation */ 263 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 264 unsigned long arg, 265 struct netlink_ext_ack *extack); 266 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 267 u32 classid); 268 void (*unbind_tcf)(struct Qdisc *, unsigned long); 269 270 /* rtnetlink specific */ 271 int (*dump)(struct Qdisc *, unsigned long, 272 struct sk_buff *skb, struct tcmsg*); 273 int (*dump_stats)(struct Qdisc *, unsigned long, 274 struct gnet_dump *); 275 }; 276 277 /* Qdisc_class_ops flag values */ 278 279 /* Implements API that doesn't require rtnl lock */ 280 enum qdisc_class_ops_flags { 281 QDISC_CLASS_OPS_DOIT_UNLOCKED = 1, 282 }; 283 284 struct Qdisc_ops { 285 struct Qdisc_ops *next; 286 const struct Qdisc_class_ops *cl_ops; 287 char id[IFNAMSIZ]; 288 int priv_size; 289 unsigned int static_flags; 290 291 int (*enqueue)(struct sk_buff *skb, 292 struct Qdisc *sch, 293 struct sk_buff **to_free); 294 struct sk_buff * (*dequeue)(struct Qdisc *); 295 struct sk_buff * (*peek)(struct Qdisc *); 296 297 int (*init)(struct Qdisc *sch, struct nlattr *arg, 298 struct netlink_ext_ack *extack); 299 void (*reset)(struct Qdisc *); 300 void (*destroy)(struct Qdisc *); 301 int (*change)(struct Qdisc *sch, 302 struct nlattr *arg, 303 struct netlink_ext_ack *extack); 304 void (*attach)(struct Qdisc *sch); 305 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 306 void (*change_real_num_tx)(struct Qdisc *sch, 307 unsigned int new_real_tx); 308 309 int (*dump)(struct Qdisc *, struct sk_buff *); 310 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 311 312 void (*ingress_block_set)(struct Qdisc *sch, 313 u32 block_index); 314 void (*egress_block_set)(struct Qdisc *sch, 315 u32 block_index); 316 u32 (*ingress_block_get)(struct Qdisc *sch); 317 u32 (*egress_block_get)(struct Qdisc *sch); 318 319 struct module *owner; 320 }; 321 322 struct tcf_result { 323 union { 324 struct { 325 unsigned long class; 326 u32 classid; 327 }; 328 const struct tcf_proto *goto_tp; 329 }; 330 }; 331 332 struct tcf_chain; 333 334 struct tcf_proto_ops { 335 struct list_head head; 336 char kind[IFNAMSIZ]; 337 338 int (*classify)(struct sk_buff *, 339 const struct tcf_proto *, 340 struct tcf_result *); 341 int (*init)(struct tcf_proto*); 342 void (*destroy)(struct tcf_proto *tp, bool rtnl_held, 343 struct netlink_ext_ack *extack); 344 345 void* (*get)(struct tcf_proto*, u32 handle); 346 void (*put)(struct tcf_proto *tp, void *f); 347 int (*change)(struct net *net, struct sk_buff *, 348 struct tcf_proto*, unsigned long, 349 u32 handle, struct nlattr **, 350 void **, u32, 351 struct netlink_ext_ack *); 352 int (*delete)(struct tcf_proto *tp, void *arg, 353 bool *last, bool rtnl_held, 354 struct netlink_ext_ack *); 355 bool (*delete_empty)(struct tcf_proto *tp); 356 void (*walk)(struct tcf_proto *tp, 357 struct tcf_walker *arg, bool rtnl_held); 358 int (*reoffload)(struct tcf_proto *tp, bool add, 359 flow_setup_cb_t *cb, void *cb_priv, 360 struct netlink_ext_ack *extack); 361 void (*hw_add)(struct tcf_proto *tp, 362 void *type_data); 363 void (*hw_del)(struct tcf_proto *tp, 364 void *type_data); 365 void (*bind_class)(void *, u32, unsigned long, 366 void *, unsigned long); 367 void * (*tmplt_create)(struct net *net, 368 struct tcf_chain *chain, 369 struct nlattr **tca, 370 struct netlink_ext_ack *extack); 371 void (*tmplt_destroy)(void *tmplt_priv); 372 void (*tmplt_reoffload)(struct tcf_chain *chain, 373 bool add, 374 flow_setup_cb_t *cb, 375 void *cb_priv); 376 struct tcf_exts * (*get_exts)(const struct tcf_proto *tp, 377 u32 handle); 378 379 /* rtnetlink specific */ 380 int (*dump)(struct net*, struct tcf_proto*, void *, 381 struct sk_buff *skb, struct tcmsg*, 382 bool); 383 int (*terse_dump)(struct net *net, 384 struct tcf_proto *tp, void *fh, 385 struct sk_buff *skb, 386 struct tcmsg *t, bool rtnl_held); 387 int (*tmplt_dump)(struct sk_buff *skb, 388 struct net *net, 389 void *tmplt_priv); 390 391 struct module *owner; 392 int flags; 393 }; 394 395 /* Classifiers setting TCF_PROTO_OPS_DOIT_UNLOCKED in tcf_proto_ops->flags 396 * are expected to implement tcf_proto_ops->delete_empty(), otherwise race 397 * conditions can occur when filters are inserted/deleted simultaneously. 398 */ 399 enum tcf_proto_ops_flags { 400 TCF_PROTO_OPS_DOIT_UNLOCKED = 1, 401 }; 402 403 struct tcf_proto { 404 /* Fast access part */ 405 struct tcf_proto __rcu *next; 406 void __rcu *root; 407 408 /* called under RCU BH lock*/ 409 int (*classify)(struct sk_buff *, 410 const struct tcf_proto *, 411 struct tcf_result *); 412 __be16 protocol; 413 414 /* All the rest */ 415 u32 prio; 416 void *data; 417 const struct tcf_proto_ops *ops; 418 struct tcf_chain *chain; 419 /* Lock protects tcf_proto shared state and can be used by unlocked 420 * classifiers to protect their private data. 421 */ 422 spinlock_t lock; 423 bool deleting; 424 bool counted; 425 bool usesw; 426 refcount_t refcnt; 427 struct rcu_head rcu; 428 struct hlist_node destroy_ht_node; 429 }; 430 431 struct qdisc_skb_cb { 432 struct { 433 unsigned int pkt_len; 434 u16 slave_dev_queue_mapping; 435 u16 tc_classid; 436 }; 437 #define QDISC_CB_PRIV_LEN 20 438 unsigned char data[QDISC_CB_PRIV_LEN]; 439 }; 440 441 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 442 443 struct tcf_chain { 444 /* Protects filter_chain. */ 445 struct mutex filter_chain_lock; 446 struct tcf_proto __rcu *filter_chain; 447 struct list_head list; 448 struct tcf_block *block; 449 u32 index; /* chain index */ 450 unsigned int refcnt; 451 unsigned int action_refcnt; 452 bool explicitly_created; 453 bool flushing; 454 const struct tcf_proto_ops *tmplt_ops; 455 void *tmplt_priv; 456 struct rcu_head rcu; 457 }; 458 459 struct tcf_block { 460 struct xarray ports; /* datapath accessible */ 461 /* Lock protects tcf_block and lifetime-management data of chains 462 * attached to the block (refcnt, action_refcnt, explicitly_created). 463 */ 464 struct mutex lock; 465 struct list_head chain_list; 466 u32 index; /* block index for shared blocks */ 467 u32 classid; /* which class this block belongs to */ 468 refcount_t refcnt; 469 struct net *net; 470 struct Qdisc *q; 471 struct rw_semaphore cb_lock; /* protects cb_list and offload counters */ 472 struct flow_block flow_block; 473 struct list_head owner_list; 474 bool keep_dst; 475 atomic_t useswcnt; 476 atomic_t offloadcnt; /* Number of oddloaded filters */ 477 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 478 unsigned int lockeddevcnt; /* Number of devs that require rtnl lock. */ 479 struct { 480 struct tcf_chain *chain; 481 struct list_head filter_chain_list; 482 } chain0; 483 struct rcu_head rcu; 484 DECLARE_HASHTABLE(proto_destroy_ht, 7); 485 struct mutex proto_destroy_lock; /* Lock for proto_destroy hashtable. */ 486 }; 487 488 struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index); 489 490 static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain) 491 { 492 return lockdep_is_held(&chain->filter_chain_lock); 493 } 494 495 static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp) 496 { 497 return lockdep_is_held(&tp->lock); 498 } 499 500 #define tcf_chain_dereference(p, chain) \ 501 rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain)) 502 503 #define tcf_proto_dereference(p, tp) \ 504 rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp)) 505 506 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 507 { 508 struct qdisc_skb_cb *qcb; 509 510 BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*qcb)); 511 BUILD_BUG_ON(sizeof(qcb->data) < sz); 512 } 513 514 static inline int qdisc_qlen(const struct Qdisc *q) 515 { 516 return q->q.qlen; 517 } 518 519 static inline int qdisc_qlen_sum(const struct Qdisc *q) 520 { 521 __u32 qlen = q->qstats.qlen; 522 int i; 523 524 if (qdisc_is_percpu_stats(q)) { 525 for_each_possible_cpu(i) 526 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 527 } else { 528 qlen += q->q.qlen; 529 } 530 531 return qlen; 532 } 533 534 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 535 { 536 return (struct qdisc_skb_cb *)skb->cb; 537 } 538 539 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 540 { 541 return &qdisc->q.lock; 542 } 543 544 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 545 { 546 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 547 548 return q; 549 } 550 551 static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc) 552 { 553 return rcu_dereference_bh(qdisc->dev_queue->qdisc); 554 } 555 556 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 557 { 558 return rcu_dereference_rtnl(qdisc->dev_queue->qdisc_sleeping); 559 } 560 561 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 562 { 563 struct Qdisc *root = qdisc_root_sleeping(qdisc); 564 565 ASSERT_RTNL(); 566 return qdisc_lock(root); 567 } 568 569 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 570 { 571 return qdisc->dev_queue->dev; 572 } 573 574 static inline void sch_tree_lock(struct Qdisc *q) 575 { 576 if (q->flags & TCQ_F_MQROOT) 577 spin_lock_bh(qdisc_lock(q)); 578 else 579 spin_lock_bh(qdisc_root_sleeping_lock(q)); 580 } 581 582 static inline void sch_tree_unlock(struct Qdisc *q) 583 { 584 if (q->flags & TCQ_F_MQROOT) 585 spin_unlock_bh(qdisc_lock(q)); 586 else 587 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 588 } 589 590 extern struct Qdisc noop_qdisc; 591 extern struct Qdisc_ops noop_qdisc_ops; 592 extern struct Qdisc_ops pfifo_fast_ops; 593 extern const u8 sch_default_prio2band[TC_PRIO_MAX + 1]; 594 extern struct Qdisc_ops mq_qdisc_ops; 595 extern struct Qdisc_ops noqueue_qdisc_ops; 596 extern const struct Qdisc_ops *default_qdisc_ops; 597 static inline const struct Qdisc_ops * 598 get_default_qdisc_ops(const struct net_device *dev, int ntx) 599 { 600 return ntx < dev->real_num_tx_queues ? 601 default_qdisc_ops : &pfifo_fast_ops; 602 } 603 604 struct Qdisc_class_common { 605 u32 classid; 606 unsigned int filter_cnt; 607 struct hlist_node hnode; 608 }; 609 610 struct Qdisc_class_hash { 611 struct hlist_head *hash; 612 unsigned int hashsize; 613 unsigned int hashmask; 614 unsigned int hashelems; 615 }; 616 617 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 618 { 619 id ^= id >> 8; 620 id ^= id >> 4; 621 return id & mask; 622 } 623 624 static inline struct Qdisc_class_common * 625 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 626 { 627 struct Qdisc_class_common *cl; 628 unsigned int h; 629 630 if (!id) 631 return NULL; 632 633 h = qdisc_class_hash(id, hash->hashmask); 634 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 635 if (cl->classid == id) 636 return cl; 637 } 638 return NULL; 639 } 640 641 static inline bool qdisc_class_in_use(const struct Qdisc_class_common *cl) 642 { 643 return cl->filter_cnt > 0; 644 } 645 646 static inline void qdisc_class_get(struct Qdisc_class_common *cl) 647 { 648 unsigned int res; 649 650 if (check_add_overflow(cl->filter_cnt, 1, &res)) 651 WARN(1, "Qdisc class overflow"); 652 653 cl->filter_cnt = res; 654 } 655 656 static inline void qdisc_class_put(struct Qdisc_class_common *cl) 657 { 658 unsigned int res; 659 660 if (check_sub_overflow(cl->filter_cnt, 1, &res)) 661 WARN(1, "Qdisc class underflow"); 662 663 cl->filter_cnt = res; 664 } 665 666 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 667 { 668 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 669 670 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 671 } 672 673 int qdisc_class_hash_init(struct Qdisc_class_hash *); 674 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 675 struct Qdisc_class_common *); 676 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 677 struct Qdisc_class_common *); 678 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 679 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 680 681 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 682 void dev_qdisc_change_real_num_tx(struct net_device *dev, 683 unsigned int new_real_tx); 684 void dev_init_scheduler(struct net_device *dev); 685 void dev_shutdown(struct net_device *dev); 686 void dev_activate(struct net_device *dev); 687 void dev_deactivate(struct net_device *dev); 688 void dev_deactivate_many(struct list_head *head); 689 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 690 struct Qdisc *qdisc); 691 void qdisc_reset(struct Qdisc *qdisc); 692 void qdisc_destroy(struct Qdisc *qdisc); 693 void qdisc_put(struct Qdisc *qdisc); 694 void qdisc_put_unlocked(struct Qdisc *qdisc); 695 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); 696 #ifdef CONFIG_NET_SCHED 697 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 698 void *type_data); 699 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 700 struct Qdisc *new, struct Qdisc *old, 701 enum tc_setup_type type, void *type_data, 702 struct netlink_ext_ack *extack); 703 #else 704 static inline int 705 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 706 void *type_data) 707 { 708 q->flags &= ~TCQ_F_OFFLOADED; 709 return 0; 710 } 711 712 static inline void 713 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 714 struct Qdisc *new, struct Qdisc *old, 715 enum tc_setup_type type, void *type_data, 716 struct netlink_ext_ack *extack) 717 { 718 } 719 #endif 720 void qdisc_offload_query_caps(struct net_device *dev, 721 enum tc_setup_type type, 722 void *caps, size_t caps_len); 723 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 724 const struct Qdisc_ops *ops, 725 struct netlink_ext_ack *extack); 726 void qdisc_free(struct Qdisc *qdisc); 727 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 728 const struct Qdisc_ops *ops, u32 parentid, 729 struct netlink_ext_ack *extack); 730 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 731 const struct qdisc_size_table *stab); 732 int skb_do_redirect(struct sk_buff *); 733 734 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 735 { 736 #ifdef CONFIG_NET_XGRESS 737 return skb->tc_at_ingress; 738 #else 739 return false; 740 #endif 741 } 742 743 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 744 { 745 #ifdef CONFIG_NET_CLS_ACT 746 if (skb->tc_skip_classify) { 747 skb->tc_skip_classify = 0; 748 return true; 749 } 750 #endif 751 return false; 752 } 753 754 /* Reset all TX qdiscs greater than index of a device. */ 755 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 756 { 757 struct Qdisc *qdisc; 758 759 for (; i < dev->num_tx_queues; i++) { 760 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 761 if (qdisc) { 762 spin_lock_bh(qdisc_lock(qdisc)); 763 qdisc_reset(qdisc); 764 spin_unlock_bh(qdisc_lock(qdisc)); 765 } 766 } 767 } 768 769 /* Are all TX queues of the device empty? */ 770 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 771 { 772 unsigned int i; 773 774 rcu_read_lock(); 775 for (i = 0; i < dev->num_tx_queues; i++) { 776 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 777 const struct Qdisc *q = rcu_dereference(txq->qdisc); 778 779 if (!qdisc_is_empty(q)) { 780 rcu_read_unlock(); 781 return false; 782 } 783 } 784 rcu_read_unlock(); 785 return true; 786 } 787 788 /* Are any of the TX qdiscs changing? */ 789 static inline bool qdisc_tx_changing(const struct net_device *dev) 790 { 791 unsigned int i; 792 793 for (i = 0; i < dev->num_tx_queues; i++) { 794 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 795 796 if (rcu_access_pointer(txq->qdisc) != 797 rcu_access_pointer(txq->qdisc_sleeping)) 798 return true; 799 } 800 return false; 801 } 802 803 /* "noqueue" qdisc identified by not having any enqueue, see noqueue_init() */ 804 static inline bool qdisc_txq_has_no_queue(const struct netdev_queue *txq) 805 { 806 struct Qdisc *qdisc = rcu_access_pointer(txq->qdisc); 807 808 return qdisc->enqueue == NULL; 809 } 810 811 /* Is the device using the noop qdisc on all queues? */ 812 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 813 { 814 unsigned int i; 815 816 for (i = 0; i < dev->num_tx_queues; i++) { 817 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 818 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 819 return false; 820 } 821 return true; 822 } 823 824 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 825 { 826 return qdisc_skb_cb(skb)->pkt_len; 827 } 828 829 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 830 enum net_xmit_qdisc_t { 831 __NET_XMIT_STOLEN = 0x00010000, 832 __NET_XMIT_BYPASS = 0x00020000, 833 }; 834 835 #ifdef CONFIG_NET_CLS_ACT 836 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 837 #else 838 #define net_xmit_drop_count(e) (1) 839 #endif 840 841 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 842 const struct Qdisc *sch) 843 { 844 #ifdef CONFIG_NET_SCHED 845 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 846 847 if (stab) 848 __qdisc_calculate_pkt_len(skb, stab); 849 #endif 850 } 851 852 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 853 struct sk_buff **to_free) 854 { 855 return sch->enqueue(skb, sch, to_free); 856 } 857 858 static inline void _bstats_update(struct gnet_stats_basic_sync *bstats, 859 __u64 bytes, __u64 packets) 860 { 861 u64_stats_update_begin(&bstats->syncp); 862 u64_stats_add(&bstats->bytes, bytes); 863 u64_stats_add(&bstats->packets, packets); 864 u64_stats_update_end(&bstats->syncp); 865 } 866 867 static inline void bstats_update(struct gnet_stats_basic_sync *bstats, 868 const struct sk_buff *skb) 869 { 870 _bstats_update(bstats, 871 qdisc_pkt_len(skb), 872 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 873 } 874 875 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 876 const struct sk_buff *skb) 877 { 878 bstats_update(this_cpu_ptr(sch->cpu_bstats), skb); 879 } 880 881 static inline void qdisc_bstats_update(struct Qdisc *sch, 882 const struct sk_buff *skb) 883 { 884 bstats_update(&sch->bstats, skb); 885 } 886 887 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 888 const struct sk_buff *skb) 889 { 890 sch->qstats.backlog -= qdisc_pkt_len(skb); 891 } 892 893 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 894 const struct sk_buff *skb) 895 { 896 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 897 } 898 899 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 900 const struct sk_buff *skb) 901 { 902 sch->qstats.backlog += qdisc_pkt_len(skb); 903 } 904 905 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 906 const struct sk_buff *skb) 907 { 908 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 909 } 910 911 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 912 { 913 this_cpu_inc(sch->cpu_qstats->qlen); 914 } 915 916 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 917 { 918 this_cpu_dec(sch->cpu_qstats->qlen); 919 } 920 921 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 922 { 923 this_cpu_inc(sch->cpu_qstats->requeues); 924 } 925 926 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 927 { 928 sch->qstats.drops += count; 929 } 930 931 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 932 { 933 qstats->drops++; 934 } 935 936 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 937 { 938 qstats->overlimits++; 939 } 940 941 static inline void qdisc_qstats_drop(struct Qdisc *sch) 942 { 943 qstats_drop_inc(&sch->qstats); 944 } 945 946 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 947 { 948 this_cpu_inc(sch->cpu_qstats->drops); 949 } 950 951 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 952 { 953 sch->qstats.overlimits++; 954 } 955 956 static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch) 957 { 958 __u32 qlen = qdisc_qlen_sum(sch); 959 960 return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen); 961 } 962 963 static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch, __u32 *qlen, 964 __u32 *backlog) 965 { 966 struct gnet_stats_queue qstats = { 0 }; 967 968 gnet_stats_add_queue(&qstats, sch->cpu_qstats, &sch->qstats); 969 *qlen = qstats.qlen + qdisc_qlen(sch); 970 *backlog = qstats.backlog; 971 } 972 973 static inline void qdisc_purge_queue(struct Qdisc *sch) 974 { 975 __u32 qlen, backlog; 976 977 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog); 978 qdisc_reset(sch); 979 qdisc_tree_reduce_backlog(sch, qlen, backlog); 980 } 981 982 static inline void __qdisc_enqueue_tail(struct sk_buff *skb, 983 struct qdisc_skb_head *qh) 984 { 985 struct sk_buff *last = qh->tail; 986 987 if (last) { 988 skb->next = NULL; 989 last->next = skb; 990 qh->tail = skb; 991 } else { 992 qh->tail = skb; 993 qh->head = skb; 994 } 995 qh->qlen++; 996 } 997 998 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 999 { 1000 __qdisc_enqueue_tail(skb, &sch->q); 1001 qdisc_qstats_backlog_inc(sch, skb); 1002 return NET_XMIT_SUCCESS; 1003 } 1004 1005 static inline void __qdisc_enqueue_head(struct sk_buff *skb, 1006 struct qdisc_skb_head *qh) 1007 { 1008 skb->next = qh->head; 1009 1010 if (!qh->head) 1011 qh->tail = skb; 1012 qh->head = skb; 1013 qh->qlen++; 1014 } 1015 1016 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 1017 { 1018 struct sk_buff *skb = qh->head; 1019 1020 if (likely(skb != NULL)) { 1021 qh->head = skb->next; 1022 qh->qlen--; 1023 if (qh->head == NULL) 1024 qh->tail = NULL; 1025 skb->next = NULL; 1026 } 1027 1028 return skb; 1029 } 1030 1031 static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool direct) 1032 { 1033 struct sk_buff *skb; 1034 1035 skb = __skb_dequeue(&sch->gso_skb); 1036 if (skb) { 1037 sch->q.qlen--; 1038 qdisc_qstats_backlog_dec(sch, skb); 1039 return skb; 1040 } 1041 if (direct) { 1042 skb = __qdisc_dequeue_head(&sch->q); 1043 if (skb) 1044 qdisc_qstats_backlog_dec(sch, skb); 1045 return skb; 1046 } else { 1047 return sch->dequeue(sch); 1048 } 1049 } 1050 1051 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 1052 { 1053 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 1054 1055 if (likely(skb != NULL)) { 1056 qdisc_qstats_backlog_dec(sch, skb); 1057 qdisc_bstats_update(sch, skb); 1058 } 1059 1060 return skb; 1061 } 1062 1063 struct tc_skb_cb { 1064 struct qdisc_skb_cb qdisc_cb; 1065 u32 drop_reason; 1066 1067 u16 zone; /* Only valid if post_ct = true */ 1068 u16 mru; 1069 u8 post_ct:1; 1070 u8 post_ct_snat:1; 1071 u8 post_ct_dnat:1; 1072 }; 1073 1074 static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb) 1075 { 1076 struct tc_skb_cb *cb = (struct tc_skb_cb *)skb->cb; 1077 1078 BUILD_BUG_ON(sizeof(*cb) > sizeof_field(struct sk_buff, cb)); 1079 return cb; 1080 } 1081 1082 static inline enum skb_drop_reason 1083 tcf_get_drop_reason(const struct sk_buff *skb) 1084 { 1085 return tc_skb_cb(skb)->drop_reason; 1086 } 1087 1088 static inline void tcf_set_drop_reason(const struct sk_buff *skb, 1089 enum skb_drop_reason reason) 1090 { 1091 tc_skb_cb(skb)->drop_reason = reason; 1092 } 1093 1094 /* Instead of calling kfree_skb() while root qdisc lock is held, 1095 * queue the skb for future freeing at end of __dev_xmit_skb() 1096 */ 1097 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 1098 { 1099 skb->next = *to_free; 1100 *to_free = skb; 1101 } 1102 1103 static inline void __qdisc_drop_all(struct sk_buff *skb, 1104 struct sk_buff **to_free) 1105 { 1106 if (skb->prev) 1107 skb->prev->next = *to_free; 1108 else 1109 skb->next = *to_free; 1110 *to_free = skb; 1111 } 1112 1113 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 1114 struct qdisc_skb_head *qh, 1115 struct sk_buff **to_free) 1116 { 1117 struct sk_buff *skb = __qdisc_dequeue_head(qh); 1118 1119 if (likely(skb != NULL)) { 1120 unsigned int len = qdisc_pkt_len(skb); 1121 1122 qdisc_qstats_backlog_dec(sch, skb); 1123 __qdisc_drop(skb, to_free); 1124 return len; 1125 } 1126 1127 return 0; 1128 } 1129 1130 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 1131 { 1132 const struct qdisc_skb_head *qh = &sch->q; 1133 1134 return qh->head; 1135 } 1136 1137 /* generic pseudo peek method for non-work-conserving qdisc */ 1138 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 1139 { 1140 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1141 1142 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 1143 if (!skb) { 1144 skb = sch->dequeue(sch); 1145 1146 if (skb) { 1147 __skb_queue_head(&sch->gso_skb, skb); 1148 /* it's still part of the queue */ 1149 qdisc_qstats_backlog_inc(sch, skb); 1150 sch->q.qlen++; 1151 } 1152 } 1153 1154 return skb; 1155 } 1156 1157 static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, 1158 struct sk_buff *skb) 1159 { 1160 if (qdisc_is_percpu_stats(sch)) { 1161 qdisc_qstats_cpu_backlog_dec(sch, skb); 1162 qdisc_bstats_cpu_update(sch, skb); 1163 qdisc_qstats_cpu_qlen_dec(sch); 1164 } else { 1165 qdisc_qstats_backlog_dec(sch, skb); 1166 qdisc_bstats_update(sch, skb); 1167 sch->q.qlen--; 1168 } 1169 } 1170 1171 static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, 1172 unsigned int pkt_len) 1173 { 1174 if (qdisc_is_percpu_stats(sch)) { 1175 qdisc_qstats_cpu_qlen_inc(sch); 1176 this_cpu_add(sch->cpu_qstats->backlog, pkt_len); 1177 } else { 1178 sch->qstats.backlog += pkt_len; 1179 sch->q.qlen++; 1180 } 1181 } 1182 1183 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 1184 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 1185 { 1186 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1187 1188 if (skb) { 1189 skb = __skb_dequeue(&sch->gso_skb); 1190 if (qdisc_is_percpu_stats(sch)) { 1191 qdisc_qstats_cpu_backlog_dec(sch, skb); 1192 qdisc_qstats_cpu_qlen_dec(sch); 1193 } else { 1194 qdisc_qstats_backlog_dec(sch, skb); 1195 sch->q.qlen--; 1196 } 1197 } else { 1198 skb = sch->dequeue(sch); 1199 } 1200 1201 return skb; 1202 } 1203 1204 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 1205 { 1206 /* 1207 * We do not know the backlog in bytes of this list, it 1208 * is up to the caller to correct it 1209 */ 1210 ASSERT_RTNL(); 1211 if (qh->qlen) { 1212 rtnl_kfree_skbs(qh->head, qh->tail); 1213 1214 qh->head = NULL; 1215 qh->tail = NULL; 1216 qh->qlen = 0; 1217 } 1218 } 1219 1220 static inline void qdisc_reset_queue(struct Qdisc *sch) 1221 { 1222 __qdisc_reset_queue(&sch->q); 1223 } 1224 1225 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 1226 struct Qdisc **pold) 1227 { 1228 struct Qdisc *old; 1229 1230 sch_tree_lock(sch); 1231 old = *pold; 1232 *pold = new; 1233 if (old != NULL) 1234 qdisc_purge_queue(old); 1235 sch_tree_unlock(sch); 1236 1237 return old; 1238 } 1239 1240 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 1241 { 1242 rtnl_kfree_skbs(skb, skb); 1243 qdisc_qstats_drop(sch); 1244 } 1245 1246 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 1247 struct sk_buff **to_free) 1248 { 1249 __qdisc_drop(skb, to_free); 1250 qdisc_qstats_cpu_drop(sch); 1251 1252 return NET_XMIT_DROP; 1253 } 1254 1255 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 1256 struct sk_buff **to_free) 1257 { 1258 __qdisc_drop(skb, to_free); 1259 qdisc_qstats_drop(sch); 1260 1261 return NET_XMIT_DROP; 1262 } 1263 1264 static inline int qdisc_drop_reason(struct sk_buff *skb, struct Qdisc *sch, 1265 struct sk_buff **to_free, 1266 enum skb_drop_reason reason) 1267 { 1268 tcf_set_drop_reason(skb, reason); 1269 return qdisc_drop(skb, sch, to_free); 1270 } 1271 1272 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1273 struct sk_buff **to_free) 1274 { 1275 __qdisc_drop_all(skb, to_free); 1276 qdisc_qstats_drop(sch); 1277 1278 return NET_XMIT_DROP; 1279 } 1280 1281 struct psched_ratecfg { 1282 u64 rate_bytes_ps; /* bytes per second */ 1283 u32 mult; 1284 u16 overhead; 1285 u16 mpu; 1286 u8 linklayer; 1287 u8 shift; 1288 }; 1289 1290 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1291 unsigned int len) 1292 { 1293 len += r->overhead; 1294 1295 if (len < r->mpu) 1296 len = r->mpu; 1297 1298 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1299 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1300 1301 return ((u64)len * r->mult) >> r->shift; 1302 } 1303 1304 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1305 const struct tc_ratespec *conf, 1306 u64 rate64); 1307 1308 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1309 const struct psched_ratecfg *r) 1310 { 1311 memset(res, 0, sizeof(*res)); 1312 1313 /* legacy struct tc_ratespec has a 32bit @rate field 1314 * Qdisc using 64bit rate should add new attributes 1315 * in order to maintain compatibility. 1316 */ 1317 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1318 1319 res->overhead = r->overhead; 1320 res->mpu = r->mpu; 1321 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1322 } 1323 1324 struct psched_pktrate { 1325 u64 rate_pkts_ps; /* packets per second */ 1326 u32 mult; 1327 u8 shift; 1328 }; 1329 1330 static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r, 1331 unsigned int pkt_num) 1332 { 1333 return ((u64)pkt_num * r->mult) >> r->shift; 1334 } 1335 1336 void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64); 1337 1338 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1339 * The fast path only needs to access filter list and to update stats 1340 */ 1341 struct mini_Qdisc { 1342 struct tcf_proto *filter_list; 1343 struct tcf_block *block; 1344 struct gnet_stats_basic_sync __percpu *cpu_bstats; 1345 struct gnet_stats_queue __percpu *cpu_qstats; 1346 unsigned long rcu_state; 1347 }; 1348 1349 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1350 const struct sk_buff *skb) 1351 { 1352 bstats_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1353 } 1354 1355 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1356 { 1357 this_cpu_inc(miniq->cpu_qstats->drops); 1358 } 1359 1360 struct mini_Qdisc_pair { 1361 struct mini_Qdisc miniq1; 1362 struct mini_Qdisc miniq2; 1363 struct mini_Qdisc __rcu **p_miniq; 1364 }; 1365 1366 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1367 struct tcf_proto *tp_head); 1368 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1369 struct mini_Qdisc __rcu **p_miniq); 1370 void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, 1371 struct tcf_block *block); 1372 1373 void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx); 1374 1375 int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)); 1376 1377 /* Make sure qdisc is no longer in SCHED state. */ 1378 static inline void qdisc_synchronize(const struct Qdisc *q) 1379 { 1380 while (test_bit(__QDISC_STATE_SCHED, &q->state)) 1381 msleep(1); 1382 } 1383 1384 #endif 1385