1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <linux/mutex.h> 16 #include <linux/rwsem.h> 17 #include <linux/atomic.h> 18 #include <linux/hashtable.h> 19 #include <net/gen_stats.h> 20 #include <net/rtnetlink.h> 21 #include <net/flow_offload.h> 22 23 struct Qdisc_ops; 24 struct qdisc_walker; 25 struct tcf_walker; 26 struct module; 27 struct bpf_flow_keys; 28 29 struct qdisc_rate_table { 30 struct tc_ratespec rate; 31 u32 data[256]; 32 struct qdisc_rate_table *next; 33 int refcnt; 34 }; 35 36 enum qdisc_state_t { 37 __QDISC_STATE_SCHED, 38 __QDISC_STATE_DEACTIVATED, 39 __QDISC_STATE_MISSED, 40 }; 41 42 struct qdisc_size_table { 43 struct rcu_head rcu; 44 struct list_head list; 45 struct tc_sizespec szopts; 46 int refcnt; 47 u16 data[]; 48 }; 49 50 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 51 struct qdisc_skb_head { 52 struct sk_buff *head; 53 struct sk_buff *tail; 54 __u32 qlen; 55 spinlock_t lock; 56 }; 57 58 struct Qdisc { 59 int (*enqueue)(struct sk_buff *skb, 60 struct Qdisc *sch, 61 struct sk_buff **to_free); 62 struct sk_buff * (*dequeue)(struct Qdisc *sch); 63 unsigned int flags; 64 #define TCQ_F_BUILTIN 1 65 #define TCQ_F_INGRESS 2 66 #define TCQ_F_CAN_BYPASS 4 67 #define TCQ_F_MQROOT 8 68 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 69 * q->dev_queue : It can test 70 * netif_xmit_frozen_or_stopped() before 71 * dequeueing next packet. 72 * Its true for MQ/MQPRIO slaves, or non 73 * multiqueue device. 74 */ 75 #define TCQ_F_WARN_NONWC (1 << 16) 76 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 77 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 78 * qdisc_tree_decrease_qlen() should stop. 79 */ 80 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 81 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 82 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 83 u32 limit; 84 const struct Qdisc_ops *ops; 85 struct qdisc_size_table __rcu *stab; 86 struct hlist_node hash; 87 u32 handle; 88 u32 parent; 89 90 struct netdev_queue *dev_queue; 91 92 struct net_rate_estimator __rcu *rate_est; 93 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 94 struct gnet_stats_queue __percpu *cpu_qstats; 95 int pad; 96 refcount_t refcnt; 97 98 /* 99 * For performance sake on SMP, we put highly modified fields at the end 100 */ 101 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 102 struct qdisc_skb_head q; 103 struct gnet_stats_basic_packed bstats; 104 seqcount_t running; 105 struct gnet_stats_queue qstats; 106 unsigned long state; 107 struct Qdisc *next_sched; 108 struct sk_buff_head skb_bad_txq; 109 110 spinlock_t busylock ____cacheline_aligned_in_smp; 111 spinlock_t seqlock; 112 113 /* for NOLOCK qdisc, true if there are no enqueued skbs */ 114 bool empty; 115 struct rcu_head rcu; 116 117 /* private data */ 118 long privdata[] ____cacheline_aligned; 119 }; 120 121 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 122 { 123 if (qdisc->flags & TCQ_F_BUILTIN) 124 return; 125 refcount_inc(&qdisc->refcnt); 126 } 127 128 /* Intended to be used by unlocked users, when concurrent qdisc release is 129 * possible. 130 */ 131 132 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc) 133 { 134 if (qdisc->flags & TCQ_F_BUILTIN) 135 return qdisc; 136 if (refcount_inc_not_zero(&qdisc->refcnt)) 137 return qdisc; 138 return NULL; 139 } 140 141 static inline bool qdisc_is_running(struct Qdisc *qdisc) 142 { 143 if (qdisc->flags & TCQ_F_NOLOCK) 144 return spin_is_locked(&qdisc->seqlock); 145 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 146 } 147 148 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 149 { 150 return q->flags & TCQ_F_CPUSTATS; 151 } 152 153 static inline bool qdisc_is_empty(const struct Qdisc *qdisc) 154 { 155 if (qdisc_is_percpu_stats(qdisc)) 156 return READ_ONCE(qdisc->empty); 157 return !READ_ONCE(qdisc->q.qlen); 158 } 159 160 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 161 { 162 if (qdisc->flags & TCQ_F_NOLOCK) { 163 if (spin_trylock(&qdisc->seqlock)) 164 goto nolock_empty; 165 166 /* If the MISSED flag is set, it means other thread has 167 * set the MISSED flag before second spin_trylock(), so 168 * we can return false here to avoid multi cpus doing 169 * the set_bit() and second spin_trylock() concurrently. 170 */ 171 if (test_bit(__QDISC_STATE_MISSED, &qdisc->state)) 172 return false; 173 174 /* Set the MISSED flag before the second spin_trylock(), 175 * if the second spin_trylock() return false, it means 176 * other cpu holding the lock will do dequeuing for us 177 * or it will see the MISSED flag set after releasing 178 * lock and reschedule the net_tx_action() to do the 179 * dequeuing. 180 */ 181 set_bit(__QDISC_STATE_MISSED, &qdisc->state); 182 183 /* Retry again in case other CPU may not see the new flag 184 * after it releases the lock at the end of qdisc_run_end(). 185 */ 186 if (!spin_trylock(&qdisc->seqlock)) 187 return false; 188 189 nolock_empty: 190 WRITE_ONCE(qdisc->empty, false); 191 } else if (qdisc_is_running(qdisc)) { 192 return false; 193 } 194 /* Variant of write_seqcount_begin() telling lockdep a trylock 195 * was attempted. 196 */ 197 raw_write_seqcount_begin(&qdisc->running); 198 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 199 return true; 200 } 201 202 static inline void qdisc_run_end(struct Qdisc *qdisc) 203 { 204 write_seqcount_end(&qdisc->running); 205 if (qdisc->flags & TCQ_F_NOLOCK) { 206 spin_unlock(&qdisc->seqlock); 207 208 if (unlikely(test_bit(__QDISC_STATE_MISSED, 209 &qdisc->state))) { 210 clear_bit(__QDISC_STATE_MISSED, &qdisc->state); 211 __netif_schedule(qdisc); 212 } 213 } 214 } 215 216 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 217 { 218 return qdisc->flags & TCQ_F_ONETXQUEUE; 219 } 220 221 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 222 { 223 #ifdef CONFIG_BQL 224 /* Non-BQL migrated drivers will return 0, too. */ 225 return dql_avail(&txq->dql); 226 #else 227 return 0; 228 #endif 229 } 230 231 struct Qdisc_class_ops { 232 unsigned int flags; 233 /* Child qdisc manipulation */ 234 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 235 int (*graft)(struct Qdisc *, unsigned long cl, 236 struct Qdisc *, struct Qdisc **, 237 struct netlink_ext_ack *extack); 238 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 239 void (*qlen_notify)(struct Qdisc *, unsigned long); 240 241 /* Class manipulation routines */ 242 unsigned long (*find)(struct Qdisc *, u32 classid); 243 int (*change)(struct Qdisc *, u32, u32, 244 struct nlattr **, unsigned long *, 245 struct netlink_ext_ack *); 246 int (*delete)(struct Qdisc *, unsigned long, 247 struct netlink_ext_ack *); 248 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 249 250 /* Filter manipulation */ 251 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 252 unsigned long arg, 253 struct netlink_ext_ack *extack); 254 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 255 u32 classid); 256 void (*unbind_tcf)(struct Qdisc *, unsigned long); 257 258 /* rtnetlink specific */ 259 int (*dump)(struct Qdisc *, unsigned long, 260 struct sk_buff *skb, struct tcmsg*); 261 int (*dump_stats)(struct Qdisc *, unsigned long, 262 struct gnet_dump *); 263 }; 264 265 /* Qdisc_class_ops flag values */ 266 267 /* Implements API that doesn't require rtnl lock */ 268 enum qdisc_class_ops_flags { 269 QDISC_CLASS_OPS_DOIT_UNLOCKED = 1, 270 }; 271 272 struct Qdisc_ops { 273 struct Qdisc_ops *next; 274 const struct Qdisc_class_ops *cl_ops; 275 char id[IFNAMSIZ]; 276 int priv_size; 277 unsigned int static_flags; 278 279 int (*enqueue)(struct sk_buff *skb, 280 struct Qdisc *sch, 281 struct sk_buff **to_free); 282 struct sk_buff * (*dequeue)(struct Qdisc *); 283 struct sk_buff * (*peek)(struct Qdisc *); 284 285 int (*init)(struct Qdisc *sch, struct nlattr *arg, 286 struct netlink_ext_ack *extack); 287 void (*reset)(struct Qdisc *); 288 void (*destroy)(struct Qdisc *); 289 int (*change)(struct Qdisc *sch, 290 struct nlattr *arg, 291 struct netlink_ext_ack *extack); 292 void (*attach)(struct Qdisc *sch); 293 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 294 295 int (*dump)(struct Qdisc *, struct sk_buff *); 296 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 297 298 void (*ingress_block_set)(struct Qdisc *sch, 299 u32 block_index); 300 void (*egress_block_set)(struct Qdisc *sch, 301 u32 block_index); 302 u32 (*ingress_block_get)(struct Qdisc *sch); 303 u32 (*egress_block_get)(struct Qdisc *sch); 304 305 struct module *owner; 306 }; 307 308 309 struct tcf_result { 310 union { 311 struct { 312 unsigned long class; 313 u32 classid; 314 }; 315 const struct tcf_proto *goto_tp; 316 317 /* used in the skb_tc_reinsert function */ 318 struct { 319 bool ingress; 320 struct gnet_stats_queue *qstats; 321 }; 322 }; 323 }; 324 325 struct tcf_chain; 326 327 struct tcf_proto_ops { 328 struct list_head head; 329 char kind[IFNAMSIZ]; 330 331 int (*classify)(struct sk_buff *, 332 const struct tcf_proto *, 333 struct tcf_result *); 334 int (*init)(struct tcf_proto*); 335 void (*destroy)(struct tcf_proto *tp, bool rtnl_held, 336 struct netlink_ext_ack *extack); 337 338 void* (*get)(struct tcf_proto*, u32 handle); 339 void (*put)(struct tcf_proto *tp, void *f); 340 int (*change)(struct net *net, struct sk_buff *, 341 struct tcf_proto*, unsigned long, 342 u32 handle, struct nlattr **, 343 void **, bool, bool, 344 struct netlink_ext_ack *); 345 int (*delete)(struct tcf_proto *tp, void *arg, 346 bool *last, bool rtnl_held, 347 struct netlink_ext_ack *); 348 bool (*delete_empty)(struct tcf_proto *tp); 349 void (*walk)(struct tcf_proto *tp, 350 struct tcf_walker *arg, bool rtnl_held); 351 int (*reoffload)(struct tcf_proto *tp, bool add, 352 flow_setup_cb_t *cb, void *cb_priv, 353 struct netlink_ext_ack *extack); 354 void (*hw_add)(struct tcf_proto *tp, 355 void *type_data); 356 void (*hw_del)(struct tcf_proto *tp, 357 void *type_data); 358 void (*bind_class)(void *, u32, unsigned long, 359 void *, unsigned long); 360 void * (*tmplt_create)(struct net *net, 361 struct tcf_chain *chain, 362 struct nlattr **tca, 363 struct netlink_ext_ack *extack); 364 void (*tmplt_destroy)(void *tmplt_priv); 365 366 /* rtnetlink specific */ 367 int (*dump)(struct net*, struct tcf_proto*, void *, 368 struct sk_buff *skb, struct tcmsg*, 369 bool); 370 int (*terse_dump)(struct net *net, 371 struct tcf_proto *tp, void *fh, 372 struct sk_buff *skb, 373 struct tcmsg *t, bool rtnl_held); 374 int (*tmplt_dump)(struct sk_buff *skb, 375 struct net *net, 376 void *tmplt_priv); 377 378 struct module *owner; 379 int flags; 380 }; 381 382 /* Classifiers setting TCF_PROTO_OPS_DOIT_UNLOCKED in tcf_proto_ops->flags 383 * are expected to implement tcf_proto_ops->delete_empty(), otherwise race 384 * conditions can occur when filters are inserted/deleted simultaneously. 385 */ 386 enum tcf_proto_ops_flags { 387 TCF_PROTO_OPS_DOIT_UNLOCKED = 1, 388 }; 389 390 struct tcf_proto { 391 /* Fast access part */ 392 struct tcf_proto __rcu *next; 393 void __rcu *root; 394 395 /* called under RCU BH lock*/ 396 int (*classify)(struct sk_buff *, 397 const struct tcf_proto *, 398 struct tcf_result *); 399 __be16 protocol; 400 401 /* All the rest */ 402 u32 prio; 403 void *data; 404 const struct tcf_proto_ops *ops; 405 struct tcf_chain *chain; 406 /* Lock protects tcf_proto shared state and can be used by unlocked 407 * classifiers to protect their private data. 408 */ 409 spinlock_t lock; 410 bool deleting; 411 refcount_t refcnt; 412 struct rcu_head rcu; 413 struct hlist_node destroy_ht_node; 414 }; 415 416 struct qdisc_skb_cb { 417 struct { 418 unsigned int pkt_len; 419 u16 slave_dev_queue_mapping; 420 u16 tc_classid; 421 }; 422 #define QDISC_CB_PRIV_LEN 20 423 unsigned char data[QDISC_CB_PRIV_LEN]; 424 u16 mru; 425 bool post_ct; 426 }; 427 428 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 429 430 struct tcf_chain { 431 /* Protects filter_chain. */ 432 struct mutex filter_chain_lock; 433 struct tcf_proto __rcu *filter_chain; 434 struct list_head list; 435 struct tcf_block *block; 436 u32 index; /* chain index */ 437 unsigned int refcnt; 438 unsigned int action_refcnt; 439 bool explicitly_created; 440 bool flushing; 441 const struct tcf_proto_ops *tmplt_ops; 442 void *tmplt_priv; 443 struct rcu_head rcu; 444 }; 445 446 struct tcf_block { 447 /* Lock protects tcf_block and lifetime-management data of chains 448 * attached to the block (refcnt, action_refcnt, explicitly_created). 449 */ 450 struct mutex lock; 451 struct list_head chain_list; 452 u32 index; /* block index for shared blocks */ 453 u32 classid; /* which class this block belongs to */ 454 refcount_t refcnt; 455 struct net *net; 456 struct Qdisc *q; 457 struct rw_semaphore cb_lock; /* protects cb_list and offload counters */ 458 struct flow_block flow_block; 459 struct list_head owner_list; 460 bool keep_dst; 461 atomic_t offloadcnt; /* Number of oddloaded filters */ 462 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 463 unsigned int lockeddevcnt; /* Number of devs that require rtnl lock. */ 464 struct { 465 struct tcf_chain *chain; 466 struct list_head filter_chain_list; 467 } chain0; 468 struct rcu_head rcu; 469 DECLARE_HASHTABLE(proto_destroy_ht, 7); 470 struct mutex proto_destroy_lock; /* Lock for proto_destroy hashtable. */ 471 }; 472 473 static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain) 474 { 475 return lockdep_is_held(&chain->filter_chain_lock); 476 } 477 478 static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp) 479 { 480 return lockdep_is_held(&tp->lock); 481 } 482 483 #define tcf_chain_dereference(p, chain) \ 484 rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain)) 485 486 #define tcf_proto_dereference(p, tp) \ 487 rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp)) 488 489 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 490 { 491 struct qdisc_skb_cb *qcb; 492 493 BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*qcb)); 494 BUILD_BUG_ON(sizeof(qcb->data) < sz); 495 } 496 497 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 498 { 499 return this_cpu_ptr(q->cpu_qstats)->qlen; 500 } 501 502 static inline int qdisc_qlen(const struct Qdisc *q) 503 { 504 return q->q.qlen; 505 } 506 507 static inline int qdisc_qlen_sum(const struct Qdisc *q) 508 { 509 __u32 qlen = q->qstats.qlen; 510 int i; 511 512 if (qdisc_is_percpu_stats(q)) { 513 for_each_possible_cpu(i) 514 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 515 } else { 516 qlen += q->q.qlen; 517 } 518 519 return qlen; 520 } 521 522 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 523 { 524 return (struct qdisc_skb_cb *)skb->cb; 525 } 526 527 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 528 { 529 return &qdisc->q.lock; 530 } 531 532 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 533 { 534 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 535 536 return q; 537 } 538 539 static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc) 540 { 541 return rcu_dereference_bh(qdisc->dev_queue->qdisc); 542 } 543 544 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 545 { 546 return qdisc->dev_queue->qdisc_sleeping; 547 } 548 549 /* The qdisc root lock is a mechanism by which to top level 550 * of a qdisc tree can be locked from any qdisc node in the 551 * forest. This allows changing the configuration of some 552 * aspect of the qdisc tree while blocking out asynchronous 553 * qdisc access in the packet processing paths. 554 * 555 * It is only legal to do this when the root will not change 556 * on us. Otherwise we'll potentially lock the wrong qdisc 557 * root. This is enforced by holding the RTNL semaphore, which 558 * all users of this lock accessor must do. 559 */ 560 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 561 { 562 struct Qdisc *root = qdisc_root(qdisc); 563 564 ASSERT_RTNL(); 565 return qdisc_lock(root); 566 } 567 568 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 569 { 570 struct Qdisc *root = qdisc_root_sleeping(qdisc); 571 572 ASSERT_RTNL(); 573 return qdisc_lock(root); 574 } 575 576 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 577 { 578 struct Qdisc *root = qdisc_root_sleeping(qdisc); 579 580 ASSERT_RTNL(); 581 return &root->running; 582 } 583 584 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 585 { 586 return qdisc->dev_queue->dev; 587 } 588 589 static inline void sch_tree_lock(struct Qdisc *q) 590 { 591 if (q->flags & TCQ_F_MQROOT) 592 spin_lock_bh(qdisc_lock(q)); 593 else 594 spin_lock_bh(qdisc_root_sleeping_lock(q)); 595 } 596 597 static inline void sch_tree_unlock(struct Qdisc *q) 598 { 599 if (q->flags & TCQ_F_MQROOT) 600 spin_unlock_bh(qdisc_lock(q)); 601 else 602 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 603 } 604 605 extern struct Qdisc noop_qdisc; 606 extern struct Qdisc_ops noop_qdisc_ops; 607 extern struct Qdisc_ops pfifo_fast_ops; 608 extern struct Qdisc_ops mq_qdisc_ops; 609 extern struct Qdisc_ops noqueue_qdisc_ops; 610 extern const struct Qdisc_ops *default_qdisc_ops; 611 static inline const struct Qdisc_ops * 612 get_default_qdisc_ops(const struct net_device *dev, int ntx) 613 { 614 return ntx < dev->real_num_tx_queues ? 615 default_qdisc_ops : &pfifo_fast_ops; 616 } 617 618 struct Qdisc_class_common { 619 u32 classid; 620 struct hlist_node hnode; 621 }; 622 623 struct Qdisc_class_hash { 624 struct hlist_head *hash; 625 unsigned int hashsize; 626 unsigned int hashmask; 627 unsigned int hashelems; 628 }; 629 630 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 631 { 632 id ^= id >> 8; 633 id ^= id >> 4; 634 return id & mask; 635 } 636 637 static inline struct Qdisc_class_common * 638 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 639 { 640 struct Qdisc_class_common *cl; 641 unsigned int h; 642 643 if (!id) 644 return NULL; 645 646 h = qdisc_class_hash(id, hash->hashmask); 647 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 648 if (cl->classid == id) 649 return cl; 650 } 651 return NULL; 652 } 653 654 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 655 { 656 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 657 658 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 659 } 660 661 int qdisc_class_hash_init(struct Qdisc_class_hash *); 662 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 663 struct Qdisc_class_common *); 664 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 665 struct Qdisc_class_common *); 666 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 667 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 668 669 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 670 void dev_init_scheduler(struct net_device *dev); 671 void dev_shutdown(struct net_device *dev); 672 void dev_activate(struct net_device *dev); 673 void dev_deactivate(struct net_device *dev); 674 void dev_deactivate_many(struct list_head *head); 675 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 676 struct Qdisc *qdisc); 677 void qdisc_reset(struct Qdisc *qdisc); 678 void qdisc_put(struct Qdisc *qdisc); 679 void qdisc_put_unlocked(struct Qdisc *qdisc); 680 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); 681 #ifdef CONFIG_NET_SCHED 682 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 683 void *type_data); 684 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 685 struct Qdisc *new, struct Qdisc *old, 686 enum tc_setup_type type, void *type_data, 687 struct netlink_ext_ack *extack); 688 #else 689 static inline int 690 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 691 void *type_data) 692 { 693 q->flags &= ~TCQ_F_OFFLOADED; 694 return 0; 695 } 696 697 static inline void 698 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 699 struct Qdisc *new, struct Qdisc *old, 700 enum tc_setup_type type, void *type_data, 701 struct netlink_ext_ack *extack) 702 { 703 } 704 #endif 705 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 706 const struct Qdisc_ops *ops, 707 struct netlink_ext_ack *extack); 708 void qdisc_free(struct Qdisc *qdisc); 709 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 710 const struct Qdisc_ops *ops, u32 parentid, 711 struct netlink_ext_ack *extack); 712 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 713 const struct qdisc_size_table *stab); 714 int skb_do_redirect(struct sk_buff *); 715 716 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 717 { 718 #ifdef CONFIG_NET_CLS_ACT 719 return skb->tc_at_ingress; 720 #else 721 return false; 722 #endif 723 } 724 725 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 726 { 727 #ifdef CONFIG_NET_CLS_ACT 728 if (skb->tc_skip_classify) { 729 skb->tc_skip_classify = 0; 730 return true; 731 } 732 #endif 733 return false; 734 } 735 736 /* Reset all TX qdiscs greater than index of a device. */ 737 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 738 { 739 struct Qdisc *qdisc; 740 741 for (; i < dev->num_tx_queues; i++) { 742 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 743 if (qdisc) { 744 spin_lock_bh(qdisc_lock(qdisc)); 745 qdisc_reset(qdisc); 746 spin_unlock_bh(qdisc_lock(qdisc)); 747 } 748 } 749 } 750 751 /* Are all TX queues of the device empty? */ 752 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 753 { 754 unsigned int i; 755 756 rcu_read_lock(); 757 for (i = 0; i < dev->num_tx_queues; i++) { 758 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 759 const struct Qdisc *q = rcu_dereference(txq->qdisc); 760 761 if (!qdisc_is_empty(q)) { 762 rcu_read_unlock(); 763 return false; 764 } 765 } 766 rcu_read_unlock(); 767 return true; 768 } 769 770 /* Are any of the TX qdiscs changing? */ 771 static inline bool qdisc_tx_changing(const struct net_device *dev) 772 { 773 unsigned int i; 774 775 for (i = 0; i < dev->num_tx_queues; i++) { 776 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 777 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 778 return true; 779 } 780 return false; 781 } 782 783 /* Is the device using the noop qdisc on all queues? */ 784 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 785 { 786 unsigned int i; 787 788 for (i = 0; i < dev->num_tx_queues; i++) { 789 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 790 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 791 return false; 792 } 793 return true; 794 } 795 796 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 797 { 798 return qdisc_skb_cb(skb)->pkt_len; 799 } 800 801 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 802 enum net_xmit_qdisc_t { 803 __NET_XMIT_STOLEN = 0x00010000, 804 __NET_XMIT_BYPASS = 0x00020000, 805 }; 806 807 #ifdef CONFIG_NET_CLS_ACT 808 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 809 #else 810 #define net_xmit_drop_count(e) (1) 811 #endif 812 813 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 814 const struct Qdisc *sch) 815 { 816 #ifdef CONFIG_NET_SCHED 817 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 818 819 if (stab) 820 __qdisc_calculate_pkt_len(skb, stab); 821 #endif 822 } 823 824 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 825 struct sk_buff **to_free) 826 { 827 qdisc_calculate_pkt_len(skb, sch); 828 return sch->enqueue(skb, sch, to_free); 829 } 830 831 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 832 __u64 bytes, __u32 packets) 833 { 834 bstats->bytes += bytes; 835 bstats->packets += packets; 836 } 837 838 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 839 const struct sk_buff *skb) 840 { 841 _bstats_update(bstats, 842 qdisc_pkt_len(skb), 843 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 844 } 845 846 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 847 __u64 bytes, __u32 packets) 848 { 849 u64_stats_update_begin(&bstats->syncp); 850 _bstats_update(&bstats->bstats, bytes, packets); 851 u64_stats_update_end(&bstats->syncp); 852 } 853 854 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 855 const struct sk_buff *skb) 856 { 857 u64_stats_update_begin(&bstats->syncp); 858 bstats_update(&bstats->bstats, skb); 859 u64_stats_update_end(&bstats->syncp); 860 } 861 862 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 863 const struct sk_buff *skb) 864 { 865 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 866 } 867 868 static inline void qdisc_bstats_update(struct Qdisc *sch, 869 const struct sk_buff *skb) 870 { 871 bstats_update(&sch->bstats, skb); 872 } 873 874 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 875 const struct sk_buff *skb) 876 { 877 sch->qstats.backlog -= qdisc_pkt_len(skb); 878 } 879 880 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 881 const struct sk_buff *skb) 882 { 883 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 884 } 885 886 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 887 const struct sk_buff *skb) 888 { 889 sch->qstats.backlog += qdisc_pkt_len(skb); 890 } 891 892 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 893 const struct sk_buff *skb) 894 { 895 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 896 } 897 898 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 899 { 900 this_cpu_inc(sch->cpu_qstats->qlen); 901 } 902 903 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 904 { 905 this_cpu_dec(sch->cpu_qstats->qlen); 906 } 907 908 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 909 { 910 this_cpu_inc(sch->cpu_qstats->requeues); 911 } 912 913 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 914 { 915 sch->qstats.drops += count; 916 } 917 918 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 919 { 920 qstats->drops++; 921 } 922 923 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 924 { 925 qstats->overlimits++; 926 } 927 928 static inline void qdisc_qstats_drop(struct Qdisc *sch) 929 { 930 qstats_drop_inc(&sch->qstats); 931 } 932 933 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 934 { 935 this_cpu_inc(sch->cpu_qstats->drops); 936 } 937 938 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 939 { 940 sch->qstats.overlimits++; 941 } 942 943 static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch) 944 { 945 __u32 qlen = qdisc_qlen_sum(sch); 946 947 return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen); 948 } 949 950 static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch, __u32 *qlen, 951 __u32 *backlog) 952 { 953 struct gnet_stats_queue qstats = { 0 }; 954 __u32 len = qdisc_qlen_sum(sch); 955 956 __gnet_stats_copy_queue(&qstats, sch->cpu_qstats, &sch->qstats, len); 957 *qlen = qstats.qlen; 958 *backlog = qstats.backlog; 959 } 960 961 static inline void qdisc_tree_flush_backlog(struct Qdisc *sch) 962 { 963 __u32 qlen, backlog; 964 965 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog); 966 qdisc_tree_reduce_backlog(sch, qlen, backlog); 967 } 968 969 static inline void qdisc_purge_queue(struct Qdisc *sch) 970 { 971 __u32 qlen, backlog; 972 973 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog); 974 qdisc_reset(sch); 975 qdisc_tree_reduce_backlog(sch, qlen, backlog); 976 } 977 978 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 979 { 980 qh->head = NULL; 981 qh->tail = NULL; 982 qh->qlen = 0; 983 } 984 985 static inline void __qdisc_enqueue_tail(struct sk_buff *skb, 986 struct qdisc_skb_head *qh) 987 { 988 struct sk_buff *last = qh->tail; 989 990 if (last) { 991 skb->next = NULL; 992 last->next = skb; 993 qh->tail = skb; 994 } else { 995 qh->tail = skb; 996 qh->head = skb; 997 } 998 qh->qlen++; 999 } 1000 1001 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 1002 { 1003 __qdisc_enqueue_tail(skb, &sch->q); 1004 qdisc_qstats_backlog_inc(sch, skb); 1005 return NET_XMIT_SUCCESS; 1006 } 1007 1008 static inline void __qdisc_enqueue_head(struct sk_buff *skb, 1009 struct qdisc_skb_head *qh) 1010 { 1011 skb->next = qh->head; 1012 1013 if (!qh->head) 1014 qh->tail = skb; 1015 qh->head = skb; 1016 qh->qlen++; 1017 } 1018 1019 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 1020 { 1021 struct sk_buff *skb = qh->head; 1022 1023 if (likely(skb != NULL)) { 1024 qh->head = skb->next; 1025 qh->qlen--; 1026 if (qh->head == NULL) 1027 qh->tail = NULL; 1028 skb->next = NULL; 1029 } 1030 1031 return skb; 1032 } 1033 1034 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 1035 { 1036 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 1037 1038 if (likely(skb != NULL)) { 1039 qdisc_qstats_backlog_dec(sch, skb); 1040 qdisc_bstats_update(sch, skb); 1041 } 1042 1043 return skb; 1044 } 1045 1046 /* Instead of calling kfree_skb() while root qdisc lock is held, 1047 * queue the skb for future freeing at end of __dev_xmit_skb() 1048 */ 1049 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 1050 { 1051 skb->next = *to_free; 1052 *to_free = skb; 1053 } 1054 1055 static inline void __qdisc_drop_all(struct sk_buff *skb, 1056 struct sk_buff **to_free) 1057 { 1058 if (skb->prev) 1059 skb->prev->next = *to_free; 1060 else 1061 skb->next = *to_free; 1062 *to_free = skb; 1063 } 1064 1065 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 1066 struct qdisc_skb_head *qh, 1067 struct sk_buff **to_free) 1068 { 1069 struct sk_buff *skb = __qdisc_dequeue_head(qh); 1070 1071 if (likely(skb != NULL)) { 1072 unsigned int len = qdisc_pkt_len(skb); 1073 1074 qdisc_qstats_backlog_dec(sch, skb); 1075 __qdisc_drop(skb, to_free); 1076 return len; 1077 } 1078 1079 return 0; 1080 } 1081 1082 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 1083 { 1084 const struct qdisc_skb_head *qh = &sch->q; 1085 1086 return qh->head; 1087 } 1088 1089 /* generic pseudo peek method for non-work-conserving qdisc */ 1090 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 1091 { 1092 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1093 1094 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 1095 if (!skb) { 1096 skb = sch->dequeue(sch); 1097 1098 if (skb) { 1099 __skb_queue_head(&sch->gso_skb, skb); 1100 /* it's still part of the queue */ 1101 qdisc_qstats_backlog_inc(sch, skb); 1102 sch->q.qlen++; 1103 } 1104 } 1105 1106 return skb; 1107 } 1108 1109 static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, 1110 struct sk_buff *skb) 1111 { 1112 if (qdisc_is_percpu_stats(sch)) { 1113 qdisc_qstats_cpu_backlog_dec(sch, skb); 1114 qdisc_bstats_cpu_update(sch, skb); 1115 qdisc_qstats_cpu_qlen_dec(sch); 1116 } else { 1117 qdisc_qstats_backlog_dec(sch, skb); 1118 qdisc_bstats_update(sch, skb); 1119 sch->q.qlen--; 1120 } 1121 } 1122 1123 static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, 1124 unsigned int pkt_len) 1125 { 1126 if (qdisc_is_percpu_stats(sch)) { 1127 qdisc_qstats_cpu_qlen_inc(sch); 1128 this_cpu_add(sch->cpu_qstats->backlog, pkt_len); 1129 } else { 1130 sch->qstats.backlog += pkt_len; 1131 sch->q.qlen++; 1132 } 1133 } 1134 1135 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 1136 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 1137 { 1138 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1139 1140 if (skb) { 1141 skb = __skb_dequeue(&sch->gso_skb); 1142 if (qdisc_is_percpu_stats(sch)) { 1143 qdisc_qstats_cpu_backlog_dec(sch, skb); 1144 qdisc_qstats_cpu_qlen_dec(sch); 1145 } else { 1146 qdisc_qstats_backlog_dec(sch, skb); 1147 sch->q.qlen--; 1148 } 1149 } else { 1150 skb = sch->dequeue(sch); 1151 } 1152 1153 return skb; 1154 } 1155 1156 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 1157 { 1158 /* 1159 * We do not know the backlog in bytes of this list, it 1160 * is up to the caller to correct it 1161 */ 1162 ASSERT_RTNL(); 1163 if (qh->qlen) { 1164 rtnl_kfree_skbs(qh->head, qh->tail); 1165 1166 qh->head = NULL; 1167 qh->tail = NULL; 1168 qh->qlen = 0; 1169 } 1170 } 1171 1172 static inline void qdisc_reset_queue(struct Qdisc *sch) 1173 { 1174 __qdisc_reset_queue(&sch->q); 1175 sch->qstats.backlog = 0; 1176 } 1177 1178 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 1179 struct Qdisc **pold) 1180 { 1181 struct Qdisc *old; 1182 1183 sch_tree_lock(sch); 1184 old = *pold; 1185 *pold = new; 1186 if (old != NULL) 1187 qdisc_purge_queue(old); 1188 sch_tree_unlock(sch); 1189 1190 return old; 1191 } 1192 1193 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 1194 { 1195 rtnl_kfree_skbs(skb, skb); 1196 qdisc_qstats_drop(sch); 1197 } 1198 1199 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 1200 struct sk_buff **to_free) 1201 { 1202 __qdisc_drop(skb, to_free); 1203 qdisc_qstats_cpu_drop(sch); 1204 1205 return NET_XMIT_DROP; 1206 } 1207 1208 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 1209 struct sk_buff **to_free) 1210 { 1211 __qdisc_drop(skb, to_free); 1212 qdisc_qstats_drop(sch); 1213 1214 return NET_XMIT_DROP; 1215 } 1216 1217 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1218 struct sk_buff **to_free) 1219 { 1220 __qdisc_drop_all(skb, to_free); 1221 qdisc_qstats_drop(sch); 1222 1223 return NET_XMIT_DROP; 1224 } 1225 1226 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 1227 long it will take to send a packet given its size. 1228 */ 1229 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 1230 { 1231 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 1232 if (slot < 0) 1233 slot = 0; 1234 slot >>= rtab->rate.cell_log; 1235 if (slot > 255) 1236 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 1237 return rtab->data[slot]; 1238 } 1239 1240 struct psched_ratecfg { 1241 u64 rate_bytes_ps; /* bytes per second */ 1242 u32 mult; 1243 u16 overhead; 1244 u8 linklayer; 1245 u8 shift; 1246 }; 1247 1248 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1249 unsigned int len) 1250 { 1251 len += r->overhead; 1252 1253 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1254 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1255 1256 return ((u64)len * r->mult) >> r->shift; 1257 } 1258 1259 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1260 const struct tc_ratespec *conf, 1261 u64 rate64); 1262 1263 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1264 const struct psched_ratecfg *r) 1265 { 1266 memset(res, 0, sizeof(*res)); 1267 1268 /* legacy struct tc_ratespec has a 32bit @rate field 1269 * Qdisc using 64bit rate should add new attributes 1270 * in order to maintain compatibility. 1271 */ 1272 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1273 1274 res->overhead = r->overhead; 1275 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1276 } 1277 1278 struct psched_pktrate { 1279 u64 rate_pkts_ps; /* packets per second */ 1280 u32 mult; 1281 u8 shift; 1282 }; 1283 1284 static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r, 1285 unsigned int pkt_num) 1286 { 1287 return ((u64)pkt_num * r->mult) >> r->shift; 1288 } 1289 1290 void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64); 1291 1292 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1293 * The fast path only needs to access filter list and to update stats 1294 */ 1295 struct mini_Qdisc { 1296 struct tcf_proto *filter_list; 1297 struct tcf_block *block; 1298 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1299 struct gnet_stats_queue __percpu *cpu_qstats; 1300 struct rcu_head rcu; 1301 }; 1302 1303 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1304 const struct sk_buff *skb) 1305 { 1306 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1307 } 1308 1309 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1310 { 1311 this_cpu_inc(miniq->cpu_qstats->drops); 1312 } 1313 1314 struct mini_Qdisc_pair { 1315 struct mini_Qdisc miniq1; 1316 struct mini_Qdisc miniq2; 1317 struct mini_Qdisc __rcu **p_miniq; 1318 }; 1319 1320 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1321 struct tcf_proto *tp_head); 1322 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1323 struct mini_Qdisc __rcu **p_miniq); 1324 void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, 1325 struct tcf_block *block); 1326 1327 int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)); 1328 1329 #endif 1330