Lines Matching refs:qdisc
31 #include <trace/events/qdisc.h>
63 * qdisc_lock(qdisc) spinlock.
66 * - enqueue, dequeue are serialized via qdisc root lock
67 * - ingress filtering is also serialized via qdisc root lock
308 * required. Owning qdisc running bit guarantees that only one CPU
322 /* And release qdisc */
374 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
634 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
642 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
657 RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
685 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
691 qdisc->enqueue = NULL;
730 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
734 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
742 if (qdisc_is_percpu_stats(qdisc))
743 return qdisc_drop_cpu(skb, qdisc, to_free);
745 return qdisc_drop(skb, qdisc, to_free);
748 qdisc_update_stats_at_enqueue(qdisc, pkt_len);
752 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
754 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
769 qdisc_update_stats_at_dequeue(qdisc, skb);
771 READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {
777 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
778 clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
793 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
795 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
808 static void pfifo_fast_reset(struct Qdisc *qdisc)
811 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
827 if (qdisc_is_percpu_stats(qdisc)) {
831 q = per_cpu_ptr(qdisc->cpu_qstats, i);
838 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
851 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
854 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
855 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
872 qdisc->flags |= TCQ_F_CAN_BYPASS;
975 /* seqlock has the same scope of busylock, for NOLOCK qdisc */
1026 /* Under qdisc_lock(qdisc) and BH! */
1028 void qdisc_reset(struct Qdisc *qdisc)
1030 const struct Qdisc_ops *ops = qdisc->ops;
1032 trace_qdisc_reset(qdisc);
1035 ops->reset(qdisc);
1037 __skb_queue_purge(&qdisc->gso_skb);
1038 __skb_queue_purge(&qdisc->skb_bad_txq);
1040 qdisc->q.qlen = 0;
1041 qdisc->qstats.backlog = 0;
1045 void qdisc_free(struct Qdisc *qdisc)
1047 if (qdisc_is_percpu_stats(qdisc)) {
1048 free_percpu(qdisc->cpu_bstats);
1049 free_percpu(qdisc->cpu_qstats);
1052 kfree(qdisc);
1062 static void __qdisc_destroy(struct Qdisc *qdisc)
1064 const struct Qdisc_ops *ops = qdisc->ops;
1065 struct net_device *dev = qdisc_dev(qdisc);
1068 qdisc_hash_del(qdisc);
1070 qdisc_put_stab(rtnl_dereference(qdisc->stab));
1072 gen_kill_estimator(&qdisc->rate_est);
1074 qdisc_reset(qdisc);
1078 ops->destroy(qdisc);
1080 lockdep_unregister_key(&qdisc->root_lock_key);
1082 netdev_put(dev, &qdisc->dev_tracker);
1084 trace_qdisc_destroy(qdisc);
1086 call_rcu(&qdisc->rcu, qdisc_free_cb);
1089 void qdisc_destroy(struct Qdisc *qdisc)
1091 if (qdisc->flags & TCQ_F_BUILTIN)
1094 __qdisc_destroy(qdisc);
1097 void qdisc_put(struct Qdisc *qdisc)
1099 if (!qdisc)
1102 if (qdisc->flags & TCQ_F_BUILTIN ||
1103 !refcount_dec_and_test(&qdisc->refcnt))
1106 __qdisc_destroy(qdisc);
1112 * qdisc reference counter reached zero.
1115 void qdisc_put_unlocked(struct Qdisc *qdisc)
1117 if (qdisc->flags & TCQ_F_BUILTIN ||
1118 !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
1121 __qdisc_destroy(qdisc);
1126 /* Attach toplevel qdisc to device queue. */
1128 struct Qdisc *qdisc)
1137 if (qdisc == NULL)
1138 qdisc = &noop_qdisc;
1139 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1140 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1152 struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1155 if (qdisc) {
1156 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1159 qdisc_put(qdisc);
1167 struct Qdisc *qdisc;
1175 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
1176 if (!qdisc)
1180 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
1181 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1187 struct Qdisc *qdisc;
1194 qdisc = rtnl_dereference(txq->qdisc_sleeping);
1195 rcu_assign_pointer(dev->qdisc, qdisc);
1196 qdisc_refcount_inc(qdisc);
1198 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
1199 if (qdisc) {
1200 rcu_assign_pointer(dev->qdisc, qdisc);
1201 qdisc->ops->attach(qdisc);
1204 qdisc = rtnl_dereference(dev->qdisc);
1206 /* Detect default qdisc setup/init failed and fallback to "noqueue" */
1207 if (qdisc == &noop_qdisc) {
1208 netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
1213 qdisc = rtnl_dereference(txq->qdisc_sleeping);
1214 rcu_assign_pointer(dev->qdisc, qdisc);
1215 qdisc_refcount_inc(qdisc);
1220 if (qdisc != &noop_qdisc)
1221 qdisc_hash_add(qdisc, false);
1235 rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
1251 if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
1270 static void qdisc_deactivate(struct Qdisc *qdisc)
1272 if (qdisc->flags & TCQ_F_BUILTIN)
1275 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
1283 struct Qdisc *qdisc;
1285 qdisc = rtnl_dereference(dev_queue->qdisc);
1286 if (qdisc) {
1287 if (qdisc->enqueue)
1289 qdisc_deactivate(qdisc);
1290 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1298 struct Qdisc *qdisc;
1301 qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1302 if (!qdisc)
1305 nolock = qdisc->flags & TCQ_F_NOLOCK;
1308 spin_lock_bh(&qdisc->seqlock);
1309 spin_lock_bh(qdisc_lock(qdisc));
1311 qdisc_reset(qdisc);
1313 spin_unlock_bh(qdisc_lock(qdisc));
1315 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
1316 clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
1317 spin_unlock_bh(&qdisc->seqlock);
1370 /* Wait for outstanding qdisc enqueuing calls. */
1406 struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1407 const struct Qdisc_ops *ops = qdisc->ops;
1410 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
1417 struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
1419 if (qdisc->ops->change_real_num_tx)
1420 qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
1427 struct Qdisc *qdisc;
1431 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1435 if (qdisc != &noop_qdisc && !qdisc->handle)
1436 qdisc_hash_del(qdisc);
1439 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1440 if (qdisc != &noop_qdisc && !qdisc->handle)
1441 qdisc_hash_add(qdisc, false);
1473 struct Qdisc *qdisc = _qdisc;
1475 rcu_assign_pointer(dev_queue->qdisc, qdisc);
1476 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1481 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1494 qdisc_put(rtnl_dereference(dev->qdisc));
1495 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1609 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1612 miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
1613 miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
1614 miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
1615 miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;