1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_SCHED_GENERIC_H 3 #define __NET_SCHED_GENERIC_H 4 5 #include <linux/netdevice.h> 6 #include <linux/types.h> 7 #include <linux/rcupdate.h> 8 #include <linux/pkt_sched.h> 9 #include <linux/pkt_cls.h> 10 #include <linux/percpu.h> 11 #include <linux/dynamic_queue_limits.h> 12 #include <linux/list.h> 13 #include <linux/refcount.h> 14 #include <linux/workqueue.h> 15 #include <net/gen_stats.h> 16 #include <net/rtnetlink.h> 17 18 struct Qdisc_ops; 19 struct qdisc_walker; 20 struct tcf_walker; 21 struct module; 22 struct bpf_flow_keys; 23 24 typedef int tc_setup_cb_t(enum tc_setup_type type, 25 void *type_data, void *cb_priv); 26 27 typedef int tc_indr_block_bind_cb_t(struct net_device *dev, void *cb_priv, 28 enum tc_setup_type type, void *type_data); 29 30 struct qdisc_rate_table { 31 struct tc_ratespec rate; 32 u32 data[256]; 33 struct qdisc_rate_table *next; 34 int refcnt; 35 }; 36 37 enum qdisc_state_t { 38 __QDISC_STATE_SCHED, 39 __QDISC_STATE_DEACTIVATED, 40 }; 41 42 struct qdisc_size_table { 43 struct rcu_head rcu; 44 struct list_head list; 45 struct tc_sizespec szopts; 46 int refcnt; 47 u16 data[]; 48 }; 49 50 /* similar to sk_buff_head, but skb->prev pointer is undefined. */ 51 struct qdisc_skb_head { 52 struct sk_buff *head; 53 struct sk_buff *tail; 54 __u32 qlen; 55 spinlock_t lock; 56 }; 57 58 struct Qdisc { 59 int (*enqueue)(struct sk_buff *skb, 60 struct Qdisc *sch, 61 struct sk_buff **to_free); 62 struct sk_buff * (*dequeue)(struct Qdisc *sch); 63 unsigned int flags; 64 #define TCQ_F_BUILTIN 1 65 #define TCQ_F_INGRESS 2 66 #define TCQ_F_CAN_BYPASS 4 67 #define TCQ_F_MQROOT 8 68 #define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for 69 * q->dev_queue : It can test 70 * netif_xmit_frozen_or_stopped() before 71 * dequeueing next packet. 72 * Its true for MQ/MQPRIO slaves, or non 73 * multiqueue device. 74 */ 75 #define TCQ_F_WARN_NONWC (1 << 16) 76 #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ 77 #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : 78 * qdisc_tree_decrease_qlen() should stop. 79 */ 80 #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ 81 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ 82 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */ 83 u32 limit; 84 const struct Qdisc_ops *ops; 85 struct qdisc_size_table __rcu *stab; 86 struct hlist_node hash; 87 u32 handle; 88 u32 parent; 89 90 struct netdev_queue *dev_queue; 91 92 struct net_rate_estimator __rcu *rate_est; 93 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 94 struct gnet_stats_queue __percpu *cpu_qstats; 95 int padded; 96 refcount_t refcnt; 97 98 /* 99 * For performance sake on SMP, we put highly modified fields at the end 100 */ 101 struct sk_buff_head gso_skb ____cacheline_aligned_in_smp; 102 struct qdisc_skb_head q; 103 struct gnet_stats_basic_packed bstats; 104 seqcount_t running; 105 struct gnet_stats_queue qstats; 106 unsigned long state; 107 struct Qdisc *next_sched; 108 struct sk_buff_head skb_bad_txq; 109 110 spinlock_t busylock ____cacheline_aligned_in_smp; 111 spinlock_t seqlock; 112 struct rcu_head rcu; 113 }; 114 115 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) 116 { 117 if (qdisc->flags & TCQ_F_BUILTIN) 118 return; 119 refcount_inc(&qdisc->refcnt); 120 } 121 122 /* Intended to be used by unlocked users, when concurrent qdisc release is 123 * possible. 124 */ 125 126 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc) 127 { 128 if (qdisc->flags & TCQ_F_BUILTIN) 129 return qdisc; 130 if (refcount_inc_not_zero(&qdisc->refcnt)) 131 return qdisc; 132 return NULL; 133 } 134 135 static inline bool qdisc_is_running(struct Qdisc *qdisc) 136 { 137 if (qdisc->flags & TCQ_F_NOLOCK) 138 return spin_is_locked(&qdisc->seqlock); 139 return (raw_read_seqcount(&qdisc->running) & 1) ? true : false; 140 } 141 142 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 143 { 144 if (qdisc->flags & TCQ_F_NOLOCK) { 145 if (!spin_trylock(&qdisc->seqlock)) 146 return false; 147 } else if (qdisc_is_running(qdisc)) { 148 return false; 149 } 150 /* Variant of write_seqcount_begin() telling lockdep a trylock 151 * was attempted. 152 */ 153 raw_write_seqcount_begin(&qdisc->running); 154 seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_); 155 return true; 156 } 157 158 static inline void qdisc_run_end(struct Qdisc *qdisc) 159 { 160 write_seqcount_end(&qdisc->running); 161 if (qdisc->flags & TCQ_F_NOLOCK) 162 spin_unlock(&qdisc->seqlock); 163 } 164 165 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) 166 { 167 return qdisc->flags & TCQ_F_ONETXQUEUE; 168 } 169 170 static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq) 171 { 172 #ifdef CONFIG_BQL 173 /* Non-BQL migrated drivers will return 0, too. */ 174 return dql_avail(&txq->dql); 175 #else 176 return 0; 177 #endif 178 } 179 180 struct Qdisc_class_ops { 181 /* Child qdisc manipulation */ 182 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 183 int (*graft)(struct Qdisc *, unsigned long cl, 184 struct Qdisc *, struct Qdisc **, 185 struct netlink_ext_ack *extack); 186 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 187 void (*qlen_notify)(struct Qdisc *, unsigned long); 188 189 /* Class manipulation routines */ 190 unsigned long (*find)(struct Qdisc *, u32 classid); 191 int (*change)(struct Qdisc *, u32, u32, 192 struct nlattr **, unsigned long *, 193 struct netlink_ext_ack *); 194 int (*delete)(struct Qdisc *, unsigned long); 195 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 196 197 /* Filter manipulation */ 198 struct tcf_block * (*tcf_block)(struct Qdisc *sch, 199 unsigned long arg, 200 struct netlink_ext_ack *extack); 201 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 202 u32 classid); 203 void (*unbind_tcf)(struct Qdisc *, unsigned long); 204 205 /* rtnetlink specific */ 206 int (*dump)(struct Qdisc *, unsigned long, 207 struct sk_buff *skb, struct tcmsg*); 208 int (*dump_stats)(struct Qdisc *, unsigned long, 209 struct gnet_dump *); 210 }; 211 212 struct Qdisc_ops { 213 struct Qdisc_ops *next; 214 const struct Qdisc_class_ops *cl_ops; 215 char id[IFNAMSIZ]; 216 int priv_size; 217 unsigned int static_flags; 218 219 int (*enqueue)(struct sk_buff *skb, 220 struct Qdisc *sch, 221 struct sk_buff **to_free); 222 struct sk_buff * (*dequeue)(struct Qdisc *); 223 struct sk_buff * (*peek)(struct Qdisc *); 224 225 int (*init)(struct Qdisc *sch, struct nlattr *arg, 226 struct netlink_ext_ack *extack); 227 void (*reset)(struct Qdisc *); 228 void (*destroy)(struct Qdisc *); 229 int (*change)(struct Qdisc *sch, 230 struct nlattr *arg, 231 struct netlink_ext_ack *extack); 232 void (*attach)(struct Qdisc *sch); 233 int (*change_tx_queue_len)(struct Qdisc *, unsigned int); 234 235 int (*dump)(struct Qdisc *, struct sk_buff *); 236 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 237 238 void (*ingress_block_set)(struct Qdisc *sch, 239 u32 block_index); 240 void (*egress_block_set)(struct Qdisc *sch, 241 u32 block_index); 242 u32 (*ingress_block_get)(struct Qdisc *sch); 243 u32 (*egress_block_get)(struct Qdisc *sch); 244 245 struct module *owner; 246 }; 247 248 249 struct tcf_result { 250 union { 251 struct { 252 unsigned long class; 253 u32 classid; 254 }; 255 const struct tcf_proto *goto_tp; 256 257 /* used by the TC_ACT_REINSERT action */ 258 struct { 259 bool ingress; 260 struct gnet_stats_queue *qstats; 261 }; 262 }; 263 }; 264 265 struct tcf_chain; 266 267 struct tcf_proto_ops { 268 struct list_head head; 269 char kind[IFNAMSIZ]; 270 271 int (*classify)(struct sk_buff *, 272 const struct tcf_proto *, 273 struct tcf_result *); 274 int (*init)(struct tcf_proto*); 275 void (*destroy)(struct tcf_proto *tp, 276 struct netlink_ext_ack *extack); 277 278 void* (*get)(struct tcf_proto*, u32 handle); 279 int (*change)(struct net *net, struct sk_buff *, 280 struct tcf_proto*, unsigned long, 281 u32 handle, struct nlattr **, 282 void **, bool, 283 struct netlink_ext_ack *); 284 int (*delete)(struct tcf_proto *tp, void *arg, 285 bool *last, 286 struct netlink_ext_ack *); 287 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 288 int (*reoffload)(struct tcf_proto *tp, bool add, 289 tc_setup_cb_t *cb, void *cb_priv, 290 struct netlink_ext_ack *extack); 291 void (*bind_class)(void *, u32, unsigned long); 292 void * (*tmplt_create)(struct net *net, 293 struct tcf_chain *chain, 294 struct nlattr **tca, 295 struct netlink_ext_ack *extack); 296 void (*tmplt_destroy)(void *tmplt_priv); 297 298 /* rtnetlink specific */ 299 int (*dump)(struct net*, struct tcf_proto*, void *, 300 struct sk_buff *skb, struct tcmsg*); 301 int (*tmplt_dump)(struct sk_buff *skb, 302 struct net *net, 303 void *tmplt_priv); 304 305 struct module *owner; 306 }; 307 308 struct tcf_proto { 309 /* Fast access part */ 310 struct tcf_proto __rcu *next; 311 void __rcu *root; 312 313 /* called under RCU BH lock*/ 314 int (*classify)(struct sk_buff *, 315 const struct tcf_proto *, 316 struct tcf_result *); 317 __be16 protocol; 318 319 /* All the rest */ 320 u32 prio; 321 void *data; 322 const struct tcf_proto_ops *ops; 323 struct tcf_chain *chain; 324 struct rcu_head rcu; 325 }; 326 327 struct qdisc_skb_cb { 328 union { 329 struct { 330 unsigned int pkt_len; 331 u16 slave_dev_queue_mapping; 332 u16 tc_classid; 333 }; 334 struct bpf_flow_keys *flow_keys; 335 }; 336 #define QDISC_CB_PRIV_LEN 20 337 unsigned char data[QDISC_CB_PRIV_LEN]; 338 }; 339 340 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); 341 342 struct tcf_chain { 343 struct tcf_proto __rcu *filter_chain; 344 struct list_head list; 345 struct tcf_block *block; 346 u32 index; /* chain index */ 347 unsigned int refcnt; 348 unsigned int action_refcnt; 349 bool explicitly_created; 350 const struct tcf_proto_ops *tmplt_ops; 351 void *tmplt_priv; 352 }; 353 354 struct tcf_block { 355 struct list_head chain_list; 356 u32 index; /* block index for shared blocks */ 357 refcount_t refcnt; 358 struct net *net; 359 struct Qdisc *q; 360 struct list_head cb_list; 361 struct list_head owner_list; 362 bool keep_dst; 363 unsigned int offloadcnt; /* Number of oddloaded filters */ 364 unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ 365 struct { 366 struct tcf_chain *chain; 367 struct list_head filter_chain_list; 368 } chain0; 369 struct rcu_head rcu; 370 }; 371 372 static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 373 { 374 if (*flags & TCA_CLS_FLAGS_IN_HW) 375 return; 376 *flags |= TCA_CLS_FLAGS_IN_HW; 377 block->offloadcnt++; 378 } 379 380 static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 381 { 382 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 383 return; 384 *flags &= ~TCA_CLS_FLAGS_IN_HW; 385 block->offloadcnt--; 386 } 387 388 static inline void 389 tc_cls_offload_cnt_update(struct tcf_block *block, u32 *cnt, 390 u32 *flags, bool add) 391 { 392 if (add) { 393 if (!*cnt) 394 tcf_block_offload_inc(block, flags); 395 (*cnt)++; 396 } else { 397 (*cnt)--; 398 if (!*cnt) 399 tcf_block_offload_dec(block, flags); 400 } 401 } 402 403 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz) 404 { 405 struct qdisc_skb_cb *qcb; 406 407 BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz); 408 BUILD_BUG_ON(sizeof(qcb->data) < sz); 409 } 410 411 static inline int qdisc_qlen_cpu(const struct Qdisc *q) 412 { 413 return this_cpu_ptr(q->cpu_qstats)->qlen; 414 } 415 416 static inline int qdisc_qlen(const struct Qdisc *q) 417 { 418 return q->q.qlen; 419 } 420 421 static inline int qdisc_qlen_sum(const struct Qdisc *q) 422 { 423 __u32 qlen = q->qstats.qlen; 424 int i; 425 426 if (q->flags & TCQ_F_NOLOCK) { 427 for_each_possible_cpu(i) 428 qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; 429 } else { 430 qlen += q->q.qlen; 431 } 432 433 return qlen; 434 } 435 436 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 437 { 438 return (struct qdisc_skb_cb *)skb->cb; 439 } 440 441 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 442 { 443 return &qdisc->q.lock; 444 } 445 446 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) 447 { 448 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); 449 450 return q; 451 } 452 453 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) 454 { 455 return qdisc->dev_queue->qdisc_sleeping; 456 } 457 458 /* The qdisc root lock is a mechanism by which to top level 459 * of a qdisc tree can be locked from any qdisc node in the 460 * forest. This allows changing the configuration of some 461 * aspect of the qdisc tree while blocking out asynchronous 462 * qdisc access in the packet processing paths. 463 * 464 * It is only legal to do this when the root will not change 465 * on us. Otherwise we'll potentially lock the wrong qdisc 466 * root. This is enforced by holding the RTNL semaphore, which 467 * all users of this lock accessor must do. 468 */ 469 static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) 470 { 471 struct Qdisc *root = qdisc_root(qdisc); 472 473 ASSERT_RTNL(); 474 return qdisc_lock(root); 475 } 476 477 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) 478 { 479 struct Qdisc *root = qdisc_root_sleeping(qdisc); 480 481 ASSERT_RTNL(); 482 return qdisc_lock(root); 483 } 484 485 static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc) 486 { 487 struct Qdisc *root = qdisc_root_sleeping(qdisc); 488 489 ASSERT_RTNL(); 490 return &root->running; 491 } 492 493 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) 494 { 495 return qdisc->dev_queue->dev; 496 } 497 498 static inline void sch_tree_lock(const struct Qdisc *q) 499 { 500 spin_lock_bh(qdisc_root_sleeping_lock(q)); 501 } 502 503 static inline void sch_tree_unlock(const struct Qdisc *q) 504 { 505 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 506 } 507 508 extern struct Qdisc noop_qdisc; 509 extern struct Qdisc_ops noop_qdisc_ops; 510 extern struct Qdisc_ops pfifo_fast_ops; 511 extern struct Qdisc_ops mq_qdisc_ops; 512 extern struct Qdisc_ops noqueue_qdisc_ops; 513 extern const struct Qdisc_ops *default_qdisc_ops; 514 static inline const struct Qdisc_ops * 515 get_default_qdisc_ops(const struct net_device *dev, int ntx) 516 { 517 return ntx < dev->real_num_tx_queues ? 518 default_qdisc_ops : &pfifo_fast_ops; 519 } 520 521 struct Qdisc_class_common { 522 u32 classid; 523 struct hlist_node hnode; 524 }; 525 526 struct Qdisc_class_hash { 527 struct hlist_head *hash; 528 unsigned int hashsize; 529 unsigned int hashmask; 530 unsigned int hashelems; 531 }; 532 533 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 534 { 535 id ^= id >> 8; 536 id ^= id >> 4; 537 return id & mask; 538 } 539 540 static inline struct Qdisc_class_common * 541 qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id) 542 { 543 struct Qdisc_class_common *cl; 544 unsigned int h; 545 546 if (!id) 547 return NULL; 548 549 h = qdisc_class_hash(id, hash->hashmask); 550 hlist_for_each_entry(cl, &hash->hash[h], hnode) { 551 if (cl->classid == id) 552 return cl; 553 } 554 return NULL; 555 } 556 557 static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid) 558 { 559 u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY; 560 561 return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL; 562 } 563 564 int qdisc_class_hash_init(struct Qdisc_class_hash *); 565 void qdisc_class_hash_insert(struct Qdisc_class_hash *, 566 struct Qdisc_class_common *); 567 void qdisc_class_hash_remove(struct Qdisc_class_hash *, 568 struct Qdisc_class_common *); 569 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 570 void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 571 572 int dev_qdisc_change_tx_queue_len(struct net_device *dev); 573 void dev_init_scheduler(struct net_device *dev); 574 void dev_shutdown(struct net_device *dev); 575 void dev_activate(struct net_device *dev); 576 void dev_deactivate(struct net_device *dev); 577 void dev_deactivate_many(struct list_head *head); 578 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 579 struct Qdisc *qdisc); 580 void qdisc_reset(struct Qdisc *qdisc); 581 void qdisc_put(struct Qdisc *qdisc); 582 void qdisc_put_unlocked(struct Qdisc *qdisc); 583 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); 584 #ifdef CONFIG_NET_SCHED 585 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 586 void *type_data); 587 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 588 struct Qdisc *new, struct Qdisc *old, 589 enum tc_setup_type type, void *type_data, 590 struct netlink_ext_ack *extack); 591 #else 592 static inline int 593 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, 594 void *type_data) 595 { 596 q->flags &= ~TCQ_F_OFFLOADED; 597 return 0; 598 } 599 600 static inline void 601 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 602 struct Qdisc *new, struct Qdisc *old, 603 enum tc_setup_type type, void *type_data, 604 struct netlink_ext_ack *extack) 605 { 606 } 607 #endif 608 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 609 const struct Qdisc_ops *ops, 610 struct netlink_ext_ack *extack); 611 void qdisc_free(struct Qdisc *qdisc); 612 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 613 const struct Qdisc_ops *ops, u32 parentid, 614 struct netlink_ext_ack *extack); 615 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 616 const struct qdisc_size_table *stab); 617 int skb_do_redirect(struct sk_buff *); 618 619 static inline void skb_reset_tc(struct sk_buff *skb) 620 { 621 #ifdef CONFIG_NET_CLS_ACT 622 skb->tc_redirected = 0; 623 #endif 624 } 625 626 static inline bool skb_is_tc_redirected(const struct sk_buff *skb) 627 { 628 #ifdef CONFIG_NET_CLS_ACT 629 return skb->tc_redirected; 630 #else 631 return false; 632 #endif 633 } 634 635 static inline bool skb_at_tc_ingress(const struct sk_buff *skb) 636 { 637 #ifdef CONFIG_NET_CLS_ACT 638 return skb->tc_at_ingress; 639 #else 640 return false; 641 #endif 642 } 643 644 static inline bool skb_skip_tc_classify(struct sk_buff *skb) 645 { 646 #ifdef CONFIG_NET_CLS_ACT 647 if (skb->tc_skip_classify) { 648 skb->tc_skip_classify = 0; 649 return true; 650 } 651 #endif 652 return false; 653 } 654 655 /* Reset all TX qdiscs greater than index of a device. */ 656 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 657 { 658 struct Qdisc *qdisc; 659 660 for (; i < dev->num_tx_queues; i++) { 661 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); 662 if (qdisc) { 663 spin_lock_bh(qdisc_lock(qdisc)); 664 qdisc_reset(qdisc); 665 spin_unlock_bh(qdisc_lock(qdisc)); 666 } 667 } 668 } 669 670 static inline void qdisc_reset_all_tx(struct net_device *dev) 671 { 672 qdisc_reset_all_tx_gt(dev, 0); 673 } 674 675 /* Are all TX queues of the device empty? */ 676 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 677 { 678 unsigned int i; 679 680 rcu_read_lock(); 681 for (i = 0; i < dev->num_tx_queues; i++) { 682 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 683 const struct Qdisc *q = rcu_dereference(txq->qdisc); 684 685 if (q->q.qlen) { 686 rcu_read_unlock(); 687 return false; 688 } 689 } 690 rcu_read_unlock(); 691 return true; 692 } 693 694 /* Are any of the TX qdiscs changing? */ 695 static inline bool qdisc_tx_changing(const struct net_device *dev) 696 { 697 unsigned int i; 698 699 for (i = 0; i < dev->num_tx_queues; i++) { 700 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 701 if (rcu_access_pointer(txq->qdisc) != txq->qdisc_sleeping) 702 return true; 703 } 704 return false; 705 } 706 707 /* Is the device using the noop qdisc on all queues? */ 708 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 709 { 710 unsigned int i; 711 712 for (i = 0; i < dev->num_tx_queues; i++) { 713 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 714 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) 715 return false; 716 } 717 return true; 718 } 719 720 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 721 { 722 return qdisc_skb_cb(skb)->pkt_len; 723 } 724 725 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 726 enum net_xmit_qdisc_t { 727 __NET_XMIT_STOLEN = 0x00010000, 728 __NET_XMIT_BYPASS = 0x00020000, 729 }; 730 731 #ifdef CONFIG_NET_CLS_ACT 732 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 733 #else 734 #define net_xmit_drop_count(e) (1) 735 #endif 736 737 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 738 const struct Qdisc *sch) 739 { 740 #ifdef CONFIG_NET_SCHED 741 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); 742 743 if (stab) 744 __qdisc_calculate_pkt_len(skb, stab); 745 #endif 746 } 747 748 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 749 struct sk_buff **to_free) 750 { 751 qdisc_calculate_pkt_len(skb, sch); 752 return sch->enqueue(skb, sch, to_free); 753 } 754 755 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) 756 { 757 return q->flags & TCQ_F_CPUSTATS; 758 } 759 760 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats, 761 __u64 bytes, __u32 packets) 762 { 763 bstats->bytes += bytes; 764 bstats->packets += packets; 765 } 766 767 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 768 const struct sk_buff *skb) 769 { 770 _bstats_update(bstats, 771 qdisc_pkt_len(skb), 772 skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1); 773 } 774 775 static inline void _bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 776 __u64 bytes, __u32 packets) 777 { 778 u64_stats_update_begin(&bstats->syncp); 779 _bstats_update(&bstats->bstats, bytes, packets); 780 u64_stats_update_end(&bstats->syncp); 781 } 782 783 static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats, 784 const struct sk_buff *skb) 785 { 786 u64_stats_update_begin(&bstats->syncp); 787 bstats_update(&bstats->bstats, skb); 788 u64_stats_update_end(&bstats->syncp); 789 } 790 791 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, 792 const struct sk_buff *skb) 793 { 794 bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb); 795 } 796 797 static inline void qdisc_bstats_update(struct Qdisc *sch, 798 const struct sk_buff *skb) 799 { 800 bstats_update(&sch->bstats, skb); 801 } 802 803 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, 804 const struct sk_buff *skb) 805 { 806 sch->qstats.backlog -= qdisc_pkt_len(skb); 807 } 808 809 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, 810 const struct sk_buff *skb) 811 { 812 this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 813 } 814 815 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, 816 const struct sk_buff *skb) 817 { 818 sch->qstats.backlog += qdisc_pkt_len(skb); 819 } 820 821 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, 822 const struct sk_buff *skb) 823 { 824 this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); 825 } 826 827 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) 828 { 829 this_cpu_inc(sch->cpu_qstats->qlen); 830 } 831 832 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) 833 { 834 this_cpu_dec(sch->cpu_qstats->qlen); 835 } 836 837 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) 838 { 839 this_cpu_inc(sch->cpu_qstats->requeues); 840 } 841 842 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 843 { 844 sch->qstats.drops += count; 845 } 846 847 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) 848 { 849 qstats->drops++; 850 } 851 852 static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) 853 { 854 qstats->overlimits++; 855 } 856 857 static inline void qdisc_qstats_drop(struct Qdisc *sch) 858 { 859 qstats_drop_inc(&sch->qstats); 860 } 861 862 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) 863 { 864 this_cpu_inc(sch->cpu_qstats->drops); 865 } 866 867 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) 868 { 869 sch->qstats.overlimits++; 870 } 871 872 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) 873 { 874 qh->head = NULL; 875 qh->tail = NULL; 876 qh->qlen = 0; 877 } 878 879 static inline void __qdisc_enqueue_tail(struct sk_buff *skb, 880 struct qdisc_skb_head *qh) 881 { 882 struct sk_buff *last = qh->tail; 883 884 if (last) { 885 skb->next = NULL; 886 last->next = skb; 887 qh->tail = skb; 888 } else { 889 qh->tail = skb; 890 qh->head = skb; 891 } 892 qh->qlen++; 893 } 894 895 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 896 { 897 __qdisc_enqueue_tail(skb, &sch->q); 898 qdisc_qstats_backlog_inc(sch, skb); 899 return NET_XMIT_SUCCESS; 900 } 901 902 static inline void __qdisc_enqueue_head(struct sk_buff *skb, 903 struct qdisc_skb_head *qh) 904 { 905 skb->next = qh->head; 906 907 if (!qh->head) 908 qh->tail = skb; 909 qh->head = skb; 910 qh->qlen++; 911 } 912 913 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) 914 { 915 struct sk_buff *skb = qh->head; 916 917 if (likely(skb != NULL)) { 918 qh->head = skb->next; 919 qh->qlen--; 920 if (qh->head == NULL) 921 qh->tail = NULL; 922 skb->next = NULL; 923 } 924 925 return skb; 926 } 927 928 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 929 { 930 struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); 931 932 if (likely(skb != NULL)) { 933 qdisc_qstats_backlog_dec(sch, skb); 934 qdisc_bstats_update(sch, skb); 935 } 936 937 return skb; 938 } 939 940 /* Instead of calling kfree_skb() while root qdisc lock is held, 941 * queue the skb for future freeing at end of __dev_xmit_skb() 942 */ 943 static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free) 944 { 945 skb->next = *to_free; 946 *to_free = skb; 947 } 948 949 static inline void __qdisc_drop_all(struct sk_buff *skb, 950 struct sk_buff **to_free) 951 { 952 if (skb->prev) 953 skb->prev->next = *to_free; 954 else 955 skb->next = *to_free; 956 *to_free = skb; 957 } 958 959 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 960 struct qdisc_skb_head *qh, 961 struct sk_buff **to_free) 962 { 963 struct sk_buff *skb = __qdisc_dequeue_head(qh); 964 965 if (likely(skb != NULL)) { 966 unsigned int len = qdisc_pkt_len(skb); 967 968 qdisc_qstats_backlog_dec(sch, skb); 969 __qdisc_drop(skb, to_free); 970 return len; 971 } 972 973 return 0; 974 } 975 976 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch, 977 struct sk_buff **to_free) 978 { 979 return __qdisc_queue_drop_head(sch, &sch->q, to_free); 980 } 981 982 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 983 { 984 const struct qdisc_skb_head *qh = &sch->q; 985 986 return qh->head; 987 } 988 989 /* generic pseudo peek method for non-work-conserving qdisc */ 990 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 991 { 992 struct sk_buff *skb = skb_peek(&sch->gso_skb); 993 994 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 995 if (!skb) { 996 skb = sch->dequeue(sch); 997 998 if (skb) { 999 __skb_queue_head(&sch->gso_skb, skb); 1000 /* it's still part of the queue */ 1001 qdisc_qstats_backlog_inc(sch, skb); 1002 sch->q.qlen++; 1003 } 1004 } 1005 1006 return skb; 1007 } 1008 1009 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 1010 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 1011 { 1012 struct sk_buff *skb = skb_peek(&sch->gso_skb); 1013 1014 if (skb) { 1015 skb = __skb_dequeue(&sch->gso_skb); 1016 qdisc_qstats_backlog_dec(sch, skb); 1017 sch->q.qlen--; 1018 } else { 1019 skb = sch->dequeue(sch); 1020 } 1021 1022 return skb; 1023 } 1024 1025 static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) 1026 { 1027 /* 1028 * We do not know the backlog in bytes of this list, it 1029 * is up to the caller to correct it 1030 */ 1031 ASSERT_RTNL(); 1032 if (qh->qlen) { 1033 rtnl_kfree_skbs(qh->head, qh->tail); 1034 1035 qh->head = NULL; 1036 qh->tail = NULL; 1037 qh->qlen = 0; 1038 } 1039 } 1040 1041 static inline void qdisc_reset_queue(struct Qdisc *sch) 1042 { 1043 __qdisc_reset_queue(&sch->q); 1044 sch->qstats.backlog = 0; 1045 } 1046 1047 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, 1048 struct Qdisc **pold) 1049 { 1050 struct Qdisc *old; 1051 1052 sch_tree_lock(sch); 1053 old = *pold; 1054 *pold = new; 1055 if (old != NULL) { 1056 unsigned int qlen = old->q.qlen; 1057 unsigned int backlog = old->qstats.backlog; 1058 1059 qdisc_reset(old); 1060 qdisc_tree_reduce_backlog(old, qlen, backlog); 1061 } 1062 sch_tree_unlock(sch); 1063 1064 return old; 1065 } 1066 1067 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 1068 { 1069 rtnl_kfree_skbs(skb, skb); 1070 qdisc_qstats_drop(sch); 1071 } 1072 1073 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, 1074 struct sk_buff **to_free) 1075 { 1076 __qdisc_drop(skb, to_free); 1077 qdisc_qstats_cpu_drop(sch); 1078 1079 return NET_XMIT_DROP; 1080 } 1081 1082 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, 1083 struct sk_buff **to_free) 1084 { 1085 __qdisc_drop(skb, to_free); 1086 qdisc_qstats_drop(sch); 1087 1088 return NET_XMIT_DROP; 1089 } 1090 1091 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, 1092 struct sk_buff **to_free) 1093 { 1094 __qdisc_drop_all(skb, to_free); 1095 qdisc_qstats_drop(sch); 1096 1097 return NET_XMIT_DROP; 1098 } 1099 1100 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 1101 long it will take to send a packet given its size. 1102 */ 1103 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 1104 { 1105 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 1106 if (slot < 0) 1107 slot = 0; 1108 slot >>= rtab->rate.cell_log; 1109 if (slot > 255) 1110 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 1111 return rtab->data[slot]; 1112 } 1113 1114 struct psched_ratecfg { 1115 u64 rate_bytes_ps; /* bytes per second */ 1116 u32 mult; 1117 u16 overhead; 1118 u8 linklayer; 1119 u8 shift; 1120 }; 1121 1122 static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, 1123 unsigned int len) 1124 { 1125 len += r->overhead; 1126 1127 if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) 1128 return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift; 1129 1130 return ((u64)len * r->mult) >> r->shift; 1131 } 1132 1133 void psched_ratecfg_precompute(struct psched_ratecfg *r, 1134 const struct tc_ratespec *conf, 1135 u64 rate64); 1136 1137 static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 1138 const struct psched_ratecfg *r) 1139 { 1140 memset(res, 0, sizeof(*res)); 1141 1142 /* legacy struct tc_ratespec has a 32bit @rate field 1143 * Qdisc using 64bit rate should add new attributes 1144 * in order to maintain compatibility. 1145 */ 1146 res->rate = min_t(u64, r->rate_bytes_ps, ~0U); 1147 1148 res->overhead = r->overhead; 1149 res->linklayer = (r->linklayer & TC_LINKLAYER_MASK); 1150 } 1151 1152 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. 1153 * The fast path only needs to access filter list and to update stats 1154 */ 1155 struct mini_Qdisc { 1156 struct tcf_proto *filter_list; 1157 struct gnet_stats_basic_cpu __percpu *cpu_bstats; 1158 struct gnet_stats_queue __percpu *cpu_qstats; 1159 struct rcu_head rcu; 1160 }; 1161 1162 static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, 1163 const struct sk_buff *skb) 1164 { 1165 bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb); 1166 } 1167 1168 static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq) 1169 { 1170 this_cpu_inc(miniq->cpu_qstats->drops); 1171 } 1172 1173 struct mini_Qdisc_pair { 1174 struct mini_Qdisc miniq1; 1175 struct mini_Qdisc miniq2; 1176 struct mini_Qdisc __rcu **p_miniq; 1177 }; 1178 1179 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, 1180 struct tcf_proto *tp_head); 1181 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, 1182 struct mini_Qdisc __rcu **p_miniq); 1183 1184 static inline void skb_tc_reinsert(struct sk_buff *skb, struct tcf_result *res) 1185 { 1186 struct gnet_stats_queue *stats = res->qstats; 1187 int ret; 1188 1189 if (res->ingress) 1190 ret = netif_receive_skb(skb); 1191 else 1192 ret = dev_queue_xmit(skb); 1193 if (ret && stats) 1194 qstats_overlimit_inc(res->qstats); 1195 } 1196 1197 #endif 1198