1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <linux/mutex.h> 16 #include <net/gen_stats.h> 17 #include <net/rtnetlink.h> 18 #include <net/flow_offload.h> 19 20 struct Qdisc_ops; 21 struct qdisc_walker; 22 struct tcf_walker; 23 struct module; 24 struct bpf_flow_keys; 25 26 struct qdisc_rate_table { 27 struct tc_ratespec rate; 28 u32 data[256]; 29 struct qdisc_rate_table *next; 30 int refcnt; 31 }; 32 33 enum qdisc_state_t { 34 __QDISC_STATE_SCHED, 35 __QDISC_STATE_DEACTIVATED, 36 }; 37 38 struct qdisc_size_table { 39 struct rcu_head rcu; 40 struct list_head list; 41 struct tc_sizespec szopts; 42 int refcnt; 43 u16 data[]; 44 }; 45 46 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 47 struct qdisc_skb_head { 48 struct sk_buff *head; 49 struct sk_buff *tail; 50 __u32 qlen; 51 spinlock_t lock; 52 }; 53 54 struct Qdisc { 55 int (*enqueue)(struct sk_buff *skb, 56 struct Qdisc *sch, 57 struct sk_buff **to_free); 58 struct sk_buff * (*dequeue)(struct Qdisc *sch); 59 unsigned int flags; 60 #define TCQ_F_BUILTIN 1 61 #define TCQ_F_INGRESS 2 62 #define TCQ_F_CAN_BYPASS 4 63 #define TCQ_F_MQROOT 8 64 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 65 * q->dev_queue : It can test 66 * netif_xmit_frozen_or_stopped() before 67 * dequeueing next packet. 68 * Its true for MQ/MQPRIO slaves, or non 69 * multiqueue device. 70 */ 71 #define TCQ_F_WARN_NONWC (1 << 16) 72 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 73 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 74 * qdisc_tree_decrease_qlen() should stop. 75 */ 76 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 77 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 78 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 79 u32 limit; 80 const struct Qdisc_ops *ops; 81 struct qdisc_size_table __rcu *stab; 82 struct hlist_node hash; 83 u32 handle; 84 u32 parent; 85 86 struct netdev_queue *dev_queue; 87 88 struct net_rate_estimator __rcu *rate_est; 89 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 90 struct gnet_stats_queue __percpu *cpu_qstats; 91 int padded; 92 refcount_t refcnt; 93 94 /* 95 * For performance sake on SMP, we put highly modified fields at the end 96 */ 97 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 98 struct qdisc_skb_head q; 99 struct gnet_stats_basic_packed bstats; 100 seqcount_t running; 101 struct gnet_stats_queue qstats; 102 unsigned long state; 103 struct Qdisc *next_sched; 104 struct sk_buff_head skb_bad_txq; 105 106 spinlock_t busylock ____cacheline_aligned_in_smp; 107 spinlock_t seqlock; 108 109 /* for NOLOCK qdisc, true if there are no enqueued skbs */ 110 bool empty; 111 struct rcu_head rcu; 112 }; 113 114 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 115 { 116 if (qdisc->flags & TCQ_F_BUILTIN) 117 return; 118 refcount_inc(&qdisc->refcnt); 119 } 120 121 /* Intended to be used by unlocked users, when concurrent qdisc release is 122 * possible. 123 */ 124 125 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc) 126 { 127 if (qdisc->flags & TCQ_F_BUILTIN) 128 return qdisc; 129 if (refcount_inc_not_zero(&qdisc->refcnt)) 130 return qdisc; 131 return NULL; 132 } 133 134 static inline bool qdisc_is_running(struct Qdisc *qdisc) 135 { 136 if (qdisc->flags & TCQ_F_NOLOCK) 137 return spin_is_locked(&qdisc->seqlock); 138 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 139 } 140 141 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 142 { 143 return q->flags & TCQ_F_CPUSTATS; 144 } 145 146 static inline bool qdisc_is_empty(const struct Qdisc *qdisc) 147 { 148 if (qdisc_is_percpu_stats(qdisc)) 149 return qdisc->empty; 150 return !qdisc->q.qlen; 151 } 152 153 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 154 { 155 if (qdisc->flags & TCQ_F_NOLOCK) { 156 if (!spin_trylock(&qdisc->seqlock)) 157 return false; 158 qdisc->empty = false; 159 } else if (qdisc_is_running(qdisc)) { 160 return false; 161 } 162 /* Variant of write_seqcount_begin() telling lockdep a trylock 163 * was attempted. 164 */ 165 raw_write_seqcount_begin(&qdisc->running); 166 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 167 return true; 168 } 169 170 static inline void qdisc_run_end(struct Qdisc *qdisc) 171 { 172 write_seqcount_end(&qdisc->running); 173 if (qdisc->flags & TCQ_F_NOLOCK) 174 spin_unlock(&qdisc->seqlock); 175 } 176 177 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 178 { 179 return qdisc->flags & TCQ_F_ONETXQUEUE; 180 } 181 182 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 183 { 184 #ifdef CONFIG_BQL 185 /* Non-BQL migrated drivers will return 0, too. */ 186 return dql_avail(&txq->dql); 187 #else 188 return 0; 189 #endif 190 } 191 192 struct Qdisc_class_ops { 193 unsigned int flags; 194 /* Child qdisc manipulation */ 195 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 196 int (*graft)(struct Qdisc *, unsigned long cl, 197 struct Qdisc *, struct Qdisc **, 198 struct netlink_ext_ack *extack); 199 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 200 void (*qlen_notify)(struct Qdisc *, unsigned long); 201 202 /* Class manipulation routines */ 203 unsigned long (*find)(struct Qdisc *, u32 classid); 204 int (*change)(struct Qdisc *, u32, u32, 205 struct nlattr **, unsigned long *, 206 struct netlink_ext_ack *); 207 int (*delete)(struct Qdisc *, unsigned long); 208 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 209 210 /* Filter manipulation */ 211 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 212 unsigned long arg, 213 struct netlink_ext_ack *extack); 214 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 215 u32 classid); 216 void (*unbind_tcf)(struct Qdisc *, unsigned long); 217 218 /* rtnetlink specific */ 219 int (*dump)(struct Qdisc *, unsigned long, 220 struct sk_buff *skb, struct tcmsg*); 221 int (*dump_stats)(struct Qdisc *, unsigned long, 222 struct gnet_dump *); 223 }; 224 225 /* Qdisc_class_ops flag values */ 226 227 /* Implements API that doesn't require rtnl lock */ 228 enum qdisc_class_ops_flags { 229 QDISC_CLASS_OPS_DOIT_UNLOCKED = 1, 230 }; 231 232 struct Qdisc_ops { 233 struct Qdisc_ops *next; 234 const struct Qdisc_class_ops *cl_ops; 235 char id[IFNAMSIZ]; 236 int priv_size; 237 unsigned int static_flags; 238 239 int (*enqueue)(struct sk_buff *skb, 240 struct Qdisc *sch, 241 struct sk_buff **to_free); 242 struct sk_buff * (*dequeue)(struct Qdisc *); 243 struct sk_buff * (*peek)(struct Qdisc *); 244 245 int (*init)(struct Qdisc *sch, struct nlattr *arg, 246 struct netlink_ext_ack *extack); 247 void (*reset)(struct Qdisc *); 248 void (*destroy)(struct Qdisc *); 249 int (*change)(struct Qdisc *sch, 250 struct nlattr *arg, 251 struct netlink_ext_ack *extack); 252 void (*attach)(struct Qdisc *sch); 253 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 254 255 int (*dump)(struct Qdisc *, struct sk_buff *); 256 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 257 258 void (*ingress_block_set)(struct Qdisc *sch, 259 u32 block_index); 260 void (*egress_block_set)(struct Qdisc *sch, 261 u32 block_index); 262 u32 (*ingress_block_get)(struct Qdisc *sch); 263 u32 (*egress_block_get)(struct Qdisc *sch); 264 265 struct module *owner; 266 }; 267 268 269 struct tcf_result { 270 union { 271 struct { 272 unsigned long class; 273 u32 classid; 274 }; 275 const struct tcf_proto *goto_tp; 276 277 /* used in the skb_tc_reinsert function */ 278 struct { 279 bool ingress; 280 struct gnet_stats_queue *qstats; 281 }; 282 }; 283 }; 284 285 struct tcf_chain; 286 287 struct tcf_proto_ops { 288 struct list_head head; 289 char kind[IFNAMSIZ]; 290 291 int (*classify)(struct sk_buff *, 292 const struct tcf_proto *, 293 struct tcf_result *); 294 int (*init)(struct tcf_proto*); 295 void (*destroy)(struct tcf_proto *tp, bool rtnl_held, 296 struct netlink_ext_ack *extack); 297 298 void* (*get)(struct tcf_proto*, u32 handle); 299 void (*put)(struct tcf_proto *tp, void *f); 300 int (*change)(struct net *net, struct sk_buff *, 301 struct tcf_proto*, unsigned long, 302 u32 handle, struct nlattr **, 303 void **, bool, bool, 304 struct netlink_ext_ack *); 305 int (*delete)(struct tcf_proto *tp, void *arg, 306 bool *last, bool rtnl_held, 307 struct netlink_ext_ack *); 308 void (*walk)(struct tcf_proto *tp, 309 struct tcf_walker *arg, bool rtnl_held); 310 int (*reoffload)(struct tcf_proto *tp, bool add, 311 flow_setup_cb_t *cb, void *cb_priv, 312 struct netlink_ext_ack *extack); 313 void (*bind_class)(void *, u32, unsigned long); 314 void * (*tmplt_create)(struct net *net, 315 struct tcf_chain *chain, 316 struct nlattr **tca, 317 struct netlink_ext_ack *extack); 318 void (*tmplt_destroy)(void *tmplt_priv); 319 320 /* rtnetlink specific */ 321 int (*dump)(struct net*, struct tcf_proto*, void *, 322 struct sk_buff *skb, struct tcmsg*, 323 bool); 324 int (*tmplt_dump)(struct sk_buff *skb, 325 struct net *net, 326 void *tmplt_priv); 327 328 struct module *owner; 329 int flags; 330 }; 331 332 enum tcf_proto_ops_flags { 333 TCF_PROTO_OPS_DOIT_UNLOCKED = 1, 334 }; 335 336 struct tcf_proto { 337 /* Fast access part */ 338 struct tcf_proto __rcu *next; 339 void __rcu *root; 340 341 /* called under RCU BH lock*/ 342 int (*classify)(struct sk_buff *, 343 const struct tcf_proto *, 344 struct tcf_result *); 345 __be16 protocol; 346 347 /* All the rest */ 348 u32 prio; 349 void *data; 350 const struct tcf_proto_ops *ops; 351 struct tcf_chain *chain; 352 /* Lock protects tcf_proto shared state and can be used by unlocked 353 * classifiers to protect their private data. 354 */ 355 spinlock_t lock; 356 bool deleting; 357 refcount_t refcnt; 358 struct rcu_head rcu; 359 }; 360 361 struct qdisc_skb_cb { 362 struct { 363 unsigned int pkt_len; 364 u16 slave_dev_queue_mapping; 365 u16 tc_classid; 366 }; 367 #define QDISC_CB_PRIV_LEN 20 368 unsigned char data[QDISC_CB_PRIV_LEN]; 369 }; 370 371 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 372 373 struct tcf_chain { 374 /* Protects filter_chain. */ 375 struct mutex filter_chain_lock; 376 struct tcf_proto __rcu *filter_chain; 377 struct list_head list; 378 struct tcf_block *block; 379 u32 index; /* chain index */ 380 unsigned int refcnt; 381 unsigned int action_refcnt; 382 bool explicitly_created; 383 bool flushing; 384 const struct tcf_proto_ops *tmplt_ops; 385 void *tmplt_priv; 386 struct rcu_head rcu; 387 }; 388 389 struct tcf_block { 390 /* Lock protects tcf_block and lifetime-management data of chains 391 * attached to the block (refcnt, action_refcnt, explicitly_created). 392 */ 393 struct mutex lock; 394 struct list_head chain_list; 395 u32 index; /* block index for shared blocks */ 396 refcount_t refcnt; 397 struct net *net; 398 struct Qdisc *q; 399 struct flow_block flow_block; 400 struct list_head owner_list; 401 bool keep_dst; 402 unsigned int offloadcnt; /* Number of oddloaded filters */ 403 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 404 struct { 405 struct tcf_chain *chain; 406 struct list_head filter_chain_list; 407 } chain0; 408 struct rcu_head rcu; 409 }; 410 411 #ifdef CONFIG_PROVE_LOCKING 412 static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain) 413 { 414 return lockdep_is_held(&chain->filter_chain_lock); 415 } 416 417 static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp) 418 { 419 return lockdep_is_held(&tp->lock); 420 } 421 #else 422 static inline bool lockdep_tcf_chain_is_locked(struct tcf_block *chain) 423 { 424 return true; 425 } 426 427 static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp) 428 { 429 return true; 430 } 431 #endif /* #ifdef CONFIG_PROVE_LOCKING */ 432 433 #define tcf_chain_dereference(p, chain) \ 434 rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain)) 435 436 #define tcf_proto_dereference(p, tp) \ 437 rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp)) 438 439 static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 440 { 441 if (*flags & TCA_CLS_FLAGS_IN_HW) 442 return; 443 *flags |= TCA_CLS_FLAGS_IN_HW; 444 block->offloadcnt++; 445 } 446 447 static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 448 { 449 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 450 return; 451 *flags &= ~TCA_CLS_FLAGS_IN_HW; 452 block->offloadcnt--; 453 } 454 455 static inline void 456 tc_cls_offload_cnt_update(struct tcf_block *block, u32 *cnt, 457 u32 *flags, bool add) 458 { 459 if (add) { 460 if (!*cnt) 461 tcf_block_offload_inc(block, flags); 462 (*cnt)++; 463 } else { 464 (*cnt)--; 465 if (!*cnt) 466 tcf_block_offload_dec(block, flags); 467 } 468 } 469 470 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 471 { 472 struct qdisc_skb_cb *qcb; 473 474 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 475 BUILD_BUG_ON(sizeof(qcb->data) < sz); 476 } 477 478 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 479 { 480 return this_cpu_ptr(q->cpu_qstats)->qlen; 481 } 482 483 static inline int qdisc_qlen(const struct Qdisc *q) 484 { 485 return q->q.qlen; 486 } 487 488 static inline int qdisc_qlen_sum(const struct Qdisc *q) 489 { 490 __u32 qlen = q->qstats.qlen; 491 int i; 492 493 if (qdisc_is_percpu_stats(q)) { 494 for_each_possible_cpu(i) 495 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 496 } else { 497 qlen += q->q.qlen; 498 } 499 500 return qlen; 501 } 502 503 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 504 { 505 return (struct qdisc_skb_cb *)skb->cb; 506 } 507 508 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 509 { 510 return &qdisc->q.lock; 511 } 512 513 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 514 { 515 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 516 517 return q; 518 } 519 520 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 521 { 522 return qdisc->dev_queue->qdisc_sleeping; 523 } 524 525 /* The qdisc root lock is a mechanism by which to top level 526 * of a qdisc tree can be locked from any qdisc node in the 527 * forest. This allows changing the configuration of some 528 * aspect of the qdisc tree while blocking out asynchronous 529 * qdisc access in the packet processing paths. 530 * 531 * It is only legal to do this when the root will not change 532 * on us. Otherwise we'll potentially lock the wrong qdisc 533 * root. This is enforced by holding the RTNL semaphore, which 534 * all users of this lock accessor must do. 535 */ 536 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 537 { 538 struct Qdisc *root = qdisc_root(qdisc); 539 540 ASSERT_RTNL(); 541 return qdisc_lock(root); 542 } 543 544 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 545 { 546 struct Qdisc *root = qdisc_root_sleeping(qdisc); 547 548 ASSERT_RTNL(); 549 return qdisc_lock(root); 550 } 551 552 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 553 { 554 struct Qdisc *root = qdisc_root_sleeping(qdisc); 555 556 ASSERT_RTNL(); 557 return &root->running; 558 } 559 560 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 561 { 562 return qdisc->dev_queue->dev; 563 } 564 565 static inline void sch_tree_lock(const struct Qdisc *q) 566 { 567 spin_lock_bh(qdisc_root_sleeping_lock(q)); 568 } 569 570 static inline void sch_tree_unlock(const struct Qdisc *q) 571 { 572 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 573 } 574 575 extern struct Qdisc noop_qdisc; 576 extern struct Qdisc_ops noop_qdisc_ops; 577 extern struct Qdisc_ops pfifo_fast_ops; 578 extern struct Qdisc_ops mq_qdisc_ops; 579 extern struct Qdisc_ops noqueue_qdisc_ops; 580 extern const struct Qdisc_ops *default_qdisc_ops; 581 static inline const struct Qdisc_ops * 582 get_default_qdisc_ops(const struct net_device *dev, int ntx) 583 { 584 return ntx < dev->real_num_tx_queues ? 585 default_qdisc_ops : &pfifo_fast_ops; 586 } 587 588 struct Qdisc_class_common { 589 u32 classid; 590 struct hlist_node hnode; 591 }; 592 593 struct Qdisc_class_hash { 594 struct hlist_head *hash; 595 unsigned int hashsize; 596 unsigned int hashmask; 597 unsigned int hashelems; 598 }; 599 600 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 601 { 602 id ^= id >> 8; 603 id ^= id >> 4; 604 return id & mask; 605 } 606 607 static inline struct Qdisc_class_common * 608 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 609 { 610 struct Qdisc_class_common *cl; 611 unsigned int h; 612 613 if (!id) 614 return NULL; 615 616 h = qdisc_class_hash(id, hash->hashmask); 617 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 618 if (cl->classid == id) 619 return cl; 620 } 621 return NULL; 622 } 623 624 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 625 { 626 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 627 628 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 629 } 630 631 int qdisc_class_hash_init(struct Qdisc_class_hash *); 632 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 633 struct Qdisc_class_common *); 634 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 635 struct Qdisc_class_common *); 636 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 637 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 638 639 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 640 void dev_init_scheduler(struct net_device *dev); 641 void dev_shutdown(struct net_device *dev); 642 void dev_activate(struct net_device *dev); 643 void dev_deactivate(struct net_device *dev); 644 void dev_deactivate_many(struct list_head *head); 645 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 646 struct Qdisc *qdisc); 647 void qdisc_reset(struct Qdisc *qdisc); 648 void qdisc_put(struct Qdisc *qdisc); 649 void qdisc_put_unlocked(struct Qdisc *qdisc); 650 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); 651 #ifdef CONFIG_NET_SCHED 652 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 653 void *type_data); 654 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 655 struct Qdisc *new, struct Qdisc *old, 656 enum tc_setup_type type, void *type_data, 657 struct netlink_ext_ack *extack); 658 #else 659 static inline int 660 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 661 void *type_data) 662 { 663 q->flags &= ~TCQ_F_OFFLOADED; 664 return 0; 665 } 666 667 static inline void 668 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 669 struct Qdisc *new, struct Qdisc *old, 670 enum tc_setup_type type, void *type_data, 671 struct netlink_ext_ack *extack) 672 { 673 } 674 #endif 675 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 676 const struct Qdisc_ops *ops, 677 struct netlink_ext_ack *extack); 678 void qdisc_free(struct Qdisc *qdisc); 679 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 680 const struct Qdisc_ops *ops, u32 parentid, 681 struct netlink_ext_ack *extack); 682 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 683 const struct qdisc_size_table *stab); 684 int skb_do_redirect(struct sk_buff *); 685 686 static inline void skb_reset_tc(struct sk_buff *skb) 687 { 688 #ifdef CONFIG_NET_CLS_ACT 689 skb->tc_redirected = 0; 690 #endif 691 } 692 693 static inline bool skb_is_tc_redirected(const struct sk_buff *skb) 694 { 695 #ifdef CONFIG_NET_CLS_ACT 696 return skb->tc_redirected; 697 #else 698 return false; 699 #endif 700 } 701 702 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 703 { 704 #ifdef CONFIG_NET_CLS_ACT 705 return skb->tc_at_ingress; 706 #else 707 return false; 708 #endif 709 } 710 711 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 712 { 713 #ifdef CONFIG_NET_CLS_ACT 714 if (skb->tc_skip_classify) { 715 skb->tc_skip_classify = 0; 716 return true; 717 } 718 #endif 719 return false; 720 } 721 722 /* Reset all TX qdiscs greater than index of a device. */ 723 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 724 { 725 struct Qdisc *qdisc; 726 727 for (; i < dev->num_tx_queues; i++) { 728 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 729 if (qdisc) { 730 spin_lock_bh(qdisc_lock(qdisc)); 731 qdisc_reset(qdisc); 732 spin_unlock_bh(qdisc_lock(qdisc)); 733 } 734 } 735 } 736 737 static inline void qdisc_reset_all_tx(struct net_device *dev) 738 { 739 qdisc_reset_all_tx_gt(dev, 0); 740 } 741 742 /* Are all TX queues of the device empty? */ 743 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 744 { 745 unsigned int i; 746 747 rcu_read_lock(); 748 for (i = 0; i < dev->num_tx_queues; i++) { 749 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 750 const struct Qdisc *q = rcu_dereference(txq->qdisc); 751 752 if (!qdisc_is_empty(q)) { 753 rcu_read_unlock(); 754 return false; 755 } 756 } 757 rcu_read_unlock(); 758 return true; 759 } 760 761 /* Are any of the TX qdiscs changing? */ 762 static inline bool qdisc_tx_changing(const struct net_device *dev) 763 { 764 unsigned int i; 765 766 for (i = 0; i < dev->num_tx_queues; i++) { 767 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 768 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 769 return true; 770 } 771 return false; 772 } 773 774 /* Is the device using the noop qdisc on all queues? */ 775 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 776 { 777 unsigned int i; 778 779 for (i = 0; i < dev->num_tx_queues; i++) { 780 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 781 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 782 return false; 783 } 784 return true; 785 } 786 787 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 788 { 789 return qdisc_skb_cb(skb)->pkt_len; 790 } 791 792 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 793 enum net_xmit_qdisc_t { 794 __NET_XMIT_STOLEN = 0x00010000, 795 __NET_XMIT_BYPASS = 0x00020000, 796 }; 797 798 #ifdef CONFIG_NET_CLS_ACT 799 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 800 #else 801 #define net_xmit_drop_count(e) (1) 802 #endif 803 804 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 805 const struct Qdisc *sch) 806 { 807 #ifdef CONFIG_NET_SCHED 808 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 809 810 if (stab) 811 __qdisc_calculate_pkt_len(skb, stab); 812 #endif 813 } 814 815 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 816 struct sk_buff **to_free) 817 { 818 qdisc_calculate_pkt_len(skb, sch); 819 return sch->enqueue(skb, sch, to_free); 820 } 821 822 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 823 __u64 bytes, __u32 packets) 824 { 825 bstats->bytes += bytes; 826 bstats->packets += packets; 827 } 828 829 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 830 const struct sk_buff *skb) 831 { 832 _bstats_update(bstats, 833 qdisc_pkt_len(skb), 834 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 835 } 836 837 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 838 __u64 bytes, __u32 packets) 839 { 840 u64_stats_update_begin(&bstats->syncp); 841 _bstats_update(&bstats->bstats, bytes, packets); 842 u64_stats_update_end(&bstats->syncp); 843 } 844 845 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 846 const struct sk_buff *skb) 847 { 848 u64_stats_update_begin(&bstats->syncp); 849 bstats_update(&bstats->bstats, skb); 850 u64_stats_update_end(&bstats->syncp); 851 } 852 853 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 854 const struct sk_buff *skb) 855 { 856 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 857 } 858 859 static inline void qdisc_bstats_update(struct Qdisc *sch, 860 const struct sk_buff *skb) 861 { 862 bstats_update(&sch->bstats, skb); 863 } 864 865 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 866 const struct sk_buff *skb) 867 { 868 sch->qstats.backlog -= qdisc_pkt_len(skb); 869 } 870 871 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 872 const struct sk_buff *skb) 873 { 874 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 875 } 876 877 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 878 const struct sk_buff *skb) 879 { 880 sch->qstats.backlog += qdisc_pkt_len(skb); 881 } 882 883 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 884 const struct sk_buff *skb) 885 { 886 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 887 } 888 889 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 890 { 891 this_cpu_inc(sch->cpu_qstats->qlen); 892 } 893 894 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 895 { 896 this_cpu_dec(sch->cpu_qstats->qlen); 897 } 898 899 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 900 { 901 this_cpu_inc(sch->cpu_qstats->requeues); 902 } 903 904 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 905 { 906 sch->qstats.drops += count; 907 } 908 909 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 910 { 911 qstats->drops++; 912 } 913 914 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 915 { 916 qstats->overlimits++; 917 } 918 919 static inline void qdisc_qstats_drop(struct Qdisc *sch) 920 { 921 qstats_drop_inc(&sch->qstats); 922 } 923 924 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 925 { 926 this_cpu_inc(sch->cpu_qstats->drops); 927 } 928 929 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 930 { 931 sch->qstats.overlimits++; 932 } 933 934 static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch) 935 { 936 __u32 qlen = qdisc_qlen_sum(sch); 937 938 return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen); 939 } 940 941 static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch, __u32 *qlen, 942 __u32 *backlog) 943 { 944 struct gnet_stats_queue qstats = { 0 }; 945 __u32 len = qdisc_qlen_sum(sch); 946 947 __gnet_stats_copy_queue(&qstats, sch->cpu_qstats, &sch->qstats, len); 948 *qlen = qstats.qlen; 949 *backlog = qstats.backlog; 950 } 951 952 static inline void qdisc_tree_flush_backlog(struct Qdisc *sch) 953 { 954 __u32 qlen, backlog; 955 956 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog); 957 qdisc_tree_reduce_backlog(sch, qlen, backlog); 958 } 959 960 static inline void qdisc_purge_queue(struct Qdisc *sch) 961 { 962 __u32 qlen, backlog; 963 964 qdisc_qstats_qlen_backlog(sch, &qlen, &backlog); 965 qdisc_reset(sch); 966 qdisc_tree_reduce_backlog(sch, qlen, backlog); 967 } 968 969 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 970 { 971 qh->head = NULL; 972 qh->tail = NULL; 973 qh->qlen = 0; 974 } 975 976 static inline void __qdisc_enqueue_tail(struct sk_buff *skb, 977 struct qdisc_skb_head *qh) 978 { 979 struct sk_buff *last = qh->tail; 980 981 if (last) { 982 skb->next = NULL; 983 last->next = skb; 984 qh->tail = skb; 985 } else { 986 qh->tail = skb; 987 qh->head = skb; 988 } 989 qh->qlen++; 990 } 991 992 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 993 { 994 __qdisc_enqueue_tail(skb, &sch->q); 995 qdisc_qstats_backlog_inc(sch, skb); 996 return NET_XMIT_SUCCESS; 997 } 998 999 static inline void __qdisc_enqueue_head(struct sk_buff *skb, 1000 struct qdisc_skb_head *qh) 1001 { 1002 skb->next = qh->head; 1003 1004 if (!qh->head) 1005 qh->tail = skb; 1006 qh->head = skb; 1007 qh->qlen++; 1008 } 1009 1010 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 1011 { 1012 struct sk_buff *skb = qh->head; 1013 1014 if (likely(skb != NULL)) { 1015 qh->head = skb->next; 1016 qh->qlen--; 1017 if (qh->head == NULL) 1018 qh->tail = NULL; 1019 skb->next = NULL; 1020 } 1021 1022 return skb; 1023 } 1024 1025 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 1026 { 1027 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 1028 1029 if (likely(skb != NULL)) { 1030 qdisc_qstats_backlog_dec(sch, skb); 1031 qdisc_bstats_update(sch, skb); 1032 } 1033 1034 return skb; 1035 } 1036 1037 /* Instead of calling kfree_skb() while root qdisc lock is held, 1038 * queue the skb for future freeing at end of __dev_xmit_skb() 1039 */ 1040 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 1041 { 1042 skb->next = *to_free; 1043 *to_free = skb; 1044 } 1045 1046 static inline void __qdisc_drop_all(struct sk_buff *skb, 1047 struct sk_buff **to_free) 1048 { 1049 if (skb->prev) 1050 skb->prev->next = *to_free; 1051 else 1052 skb->next = *to_free; 1053 *to_free = skb; 1054 } 1055 1056 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 1057 struct qdisc_skb_head *qh, 1058 struct sk_buff **to_free) 1059 { 1060 struct sk_buff *skb = __qdisc_dequeue_head(qh); 1061 1062 if (likely(skb != NULL)) { 1063 unsigned int len = qdisc_pkt_len(skb); 1064 1065 qdisc_qstats_backlog_dec(sch, skb); 1066 __qdisc_drop(skb, to_free); 1067 return len; 1068 } 1069 1070 return 0; 1071 } 1072 1073 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 1074 struct sk_buff **to_free) 1075 { 1076 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 1077 } 1078 1079 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 1080 { 1081 const struct qdisc_skb_head *qh = &sch->q; 1082 1083 return qh->head; 1084 } 1085 1086 /* generic pseudo peek method for non-work-conserving qdisc */ 1087 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 1088 { 1089 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1090 1091 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 1092 if (!skb) { 1093 skb = sch->dequeue(sch); 1094 1095 if (skb) { 1096 __skb_queue_head(&sch->gso_skb, skb); 1097 /* it's still part of the queue */ 1098 qdisc_qstats_backlog_inc(sch, skb); 1099 sch->q.qlen++; 1100 } 1101 } 1102 1103 return skb; 1104 } 1105 1106 static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, 1107 struct sk_buff *skb) 1108 { 1109 if (qdisc_is_percpu_stats(sch)) { 1110 qdisc_qstats_cpu_backlog_dec(sch, skb); 1111 qdisc_bstats_cpu_update(sch, skb); 1112 qdisc_qstats_cpu_qlen_dec(sch); 1113 } else { 1114 qdisc_qstats_backlog_dec(sch, skb); 1115 qdisc_bstats_update(sch, skb); 1116 sch->q.qlen--; 1117 } 1118 } 1119 1120 static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, 1121 unsigned int pkt_len) 1122 { 1123 if (qdisc_is_percpu_stats(sch)) { 1124 qdisc_qstats_cpu_qlen_inc(sch); 1125 this_cpu_add(sch->cpu_qstats->backlog, pkt_len); 1126 } else { 1127 sch->qstats.backlog += pkt_len; 1128 sch->q.qlen++; 1129 } 1130 } 1131 1132 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 1133 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 1134 { 1135 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1136 1137 if (skb) { 1138 skb = __skb_dequeue(&sch->gso_skb); 1139 if (qdisc_is_percpu_stats(sch)) { 1140 qdisc_qstats_cpu_backlog_dec(sch, skb); 1141 qdisc_qstats_cpu_qlen_dec(sch); 1142 } else { 1143 qdisc_qstats_backlog_dec(sch, skb); 1144 sch->q.qlen--; 1145 } 1146 } else { 1147 skb = sch->dequeue(sch); 1148 } 1149 1150 return skb; 1151 } 1152 1153 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 1154 { 1155 /* 1156 * We do not know the backlog in bytes of this list, it 1157 * is up to the caller to correct it 1158 */ 1159 ASSERT_RTNL(); 1160 if (qh->qlen) { 1161 rtnl_kfree_skbs(qh->head, qh->tail); 1162 1163 qh->head = NULL; 1164 qh->tail = NULL; 1165 qh->qlen = 0; 1166 } 1167 } 1168 1169 static inline void qdisc_reset_queue(struct Qdisc *sch) 1170 { 1171 __qdisc_reset_queue(&sch->q); 1172 sch->qstats.backlog = 0; 1173 } 1174 1175 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 1176 struct Qdisc **pold) 1177 { 1178 struct Qdisc *old; 1179 1180 sch_tree_lock(sch); 1181 old = *pold; 1182 *pold = new; 1183 if (old != NULL) 1184 qdisc_tree_flush_backlog(old); 1185 sch_tree_unlock(sch); 1186 1187 return old; 1188 } 1189 1190 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 1191 { 1192 rtnl_kfree_skbs(skb, skb); 1193 qdisc_qstats_drop(sch); 1194 } 1195 1196 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 1197 struct sk_buff **to_free) 1198 { 1199 __qdisc_drop(skb, to_free); 1200 qdisc_qstats_cpu_drop(sch); 1201 1202 return NET_XMIT_DROP; 1203 } 1204 1205 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 1206 struct sk_buff **to_free) 1207 { 1208 __qdisc_drop(skb, to_free); 1209 qdisc_qstats_drop(sch); 1210 1211 return NET_XMIT_DROP; 1212 } 1213 1214 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1215 struct sk_buff **to_free) 1216 { 1217 __qdisc_drop_all(skb, to_free); 1218 qdisc_qstats_drop(sch); 1219 1220 return NET_XMIT_DROP; 1221 } 1222 1223 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 1224 long it will take to send a packet given its size. 1225 */ 1226 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 1227 { 1228 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 1229 if (slot < 0) 1230 slot = 0; 1231 slot >>= rtab->rate.cell_log; 1232 if (slot > 255) 1233 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 1234 return rtab->data[slot]; 1235 } 1236 1237 struct psched_ratecfg { 1238 u64 rate_bytes_ps; /* bytes per second */ 1239 u32 mult; 1240 u16 overhead; 1241 u8 linklayer; 1242 u8 shift; 1243 }; 1244 1245 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1246 unsigned int len) 1247 { 1248 len += r->overhead; 1249 1250 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1251 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1252 1253 return ((u64)len * r->mult) >> r->shift; 1254 } 1255 1256 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1257 const struct tc_ratespec *conf, 1258 u64 rate64); 1259 1260 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1261 const struct psched_ratecfg *r) 1262 { 1263 memset(res, 0, sizeof(*res)); 1264 1265 /* legacy struct tc_ratespec has a 32bit @rate field 1266 * Qdisc using 64bit rate should add new attributes 1267 * in order to maintain compatibility. 1268 */ 1269 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1270 1271 res->overhead = r->overhead; 1272 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1273 } 1274 1275 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1276 * The fast path only needs to access filter list and to update stats 1277 */ 1278 struct mini_Qdisc { 1279 struct tcf_proto *filter_list; 1280 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1281 struct gnet_stats_queue __percpu *cpu_qstats; 1282 struct rcu_head rcu; 1283 }; 1284 1285 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1286 const struct sk_buff *skb) 1287 { 1288 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1289 } 1290 1291 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1292 { 1293 this_cpu_inc(miniq->cpu_qstats->drops); 1294 } 1295 1296 struct mini_Qdisc_pair { 1297 struct mini_Qdisc miniq1; 1298 struct mini_Qdisc miniq2; 1299 struct mini_Qdisc __rcu **p_miniq; 1300 }; 1301 1302 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1303 struct tcf_proto *tp_head); 1304 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1305 struct mini_Qdisc __rcu **p_miniq); 1306 1307 static inline void skb_tc_reinsert(struct sk_buff *skb, struct tcf_result *res) 1308 { 1309 struct gnet_stats_queue *stats = res->qstats; 1310 int ret; 1311 1312 if (res->ingress) 1313 ret = netif_receive_skb(skb); 1314 else 1315 ret = dev_queue_xmit(skb); 1316 if (ret && stats) 1317 qstats_overlimit_inc(res->qstats); 1318 } 1319 1320 #endif 1321