72 	struct Qdisc *sch;	/* The Classic queue (C-queue) */  member
208 static void dualpi2_calculate_c_protection(struct Qdisc *sch,  in dualpi2_calculate_c_protection()  argument
213 	q->c_protection_init = (s32)psched_mtu(qdisc_dev(sch)) *  in dualpi2_calculate_c_protection()
249  *   delay, eventually resulting in taildrop behavior once sch->limit is
282 static bool must_drop(struct Qdisc *sch, struct dualpi2_sched_data *q,  in must_drop()  argument
289 	if (sch->qstats.backlog < 2 * psched_mtu(qdisc_dev(sch)))  in must_drop()
355 	if (TC_H_MAJ(skb->priority) == q->sch->handle &&  in dualpi2_skb_classify()
385 static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch,  in dualpi2_enqueue_skb()  argument
388 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_enqueue_skb()
391 	if (unlikely(qdisc_qlen(sch) >= sch->limit) ||  in dualpi2_enqueue_skb()
393 		qdisc_qstats_overlimit(sch);  in dualpi2_enqueue_skb()
396 		return qdisc_drop_reason(skb, sch, to_free,  in dualpi2_enqueue_skb()
400 	if (q->drop_early && must_drop(sch, q, skb)) {  in dualpi2_enqueue_skb()
401 		qdisc_drop_reason(skb, sch, to_free,  in dualpi2_enqueue_skb()
412 	if (qdisc_qlen(sch) > q->maxq)  in dualpi2_enqueue_skb()
413 		q->maxq = qdisc_qlen(sch);  in dualpi2_enqueue_skb()
420 		++sch->q.qlen;  in dualpi2_enqueue_skb()
421 		qdisc_qstats_backlog_inc(sch, skb);  in dualpi2_enqueue_skb()
430 	return qdisc_enqueue_tail(skb, sch);  in dualpi2_enqueue_skb()
441 static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,  in dualpi2_qdisc_enqueue()  argument
444 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_qdisc_enqueue()
450 			qdisc_qstats_drop(sch);  in dualpi2_qdisc_enqueue()
464 			return qdisc_drop(skb, sch, to_free);  in dualpi2_qdisc_enqueue()
481 			err = dualpi2_enqueue_skb(nskb, sch, to_free);  in dualpi2_qdisc_enqueue()
499 		qdisc_tree_reduce_backlog(sch, -cnt, -byte_len);  in dualpi2_qdisc_enqueue()
503 	return dualpi2_enqueue_skb(skb, sch, to_free);  in dualpi2_qdisc_enqueue()
516 static struct sk_buff *dequeue_packet(struct Qdisc *sch,  in dequeue_packet()  argument
525 	c_len = qdisc_qlen(sch) - qdisc_qlen(q->l_queue);  in dequeue_packet()
534 		--sch->q.qlen;  in dequeue_packet()
537 		skb = __qdisc_dequeue_head(&sch->q);  in dequeue_packet()
538 		WRITE_ONCE(q->c_head_ts, head_enqueue_time(sch));  in dequeue_packet()
547 	qdisc_qstats_backlog_dec(sch, skb);  in dequeue_packet()
575 			   struct Qdisc *sch, enum skb_drop_reason reason)  in drop_and_retry()  argument
580 	qdisc_qstats_drop(sch);  in drop_and_retry()
583 static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch)  in dualpi2_qdisc_dequeue()  argument
585 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_qdisc_dequeue()
592 	while ((skb = dequeue_packet(sch, q, &credit_change, now))) {  in dualpi2_qdisc_dequeue()
593 		if (!q->drop_early && must_drop(sch, q, skb)) {  in dualpi2_qdisc_dequeue()
594 			drop_and_retry(q, skb, sch,  in dualpi2_qdisc_dequeue()
601 			drop_and_retry(q, skb, sch,  in dualpi2_qdisc_dequeue()
607 		qdisc_bstats_update(sch, skb);  in dualpi2_qdisc_dequeue()
612 		qdisc_tree_reduce_backlog(sch, q->deferred_drops_cnt,  in dualpi2_qdisc_dequeue()
639 static u32 calculate_probability(struct Qdisc *sch)  in calculate_probability()  argument
641 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in calculate_probability()
677 static u32 get_memory_limit(struct Qdisc *sch, u32 limit)  in get_memory_limit()  argument
682 	u64 memlim = mul_u32_u32(limit, 2 * psched_mtu(qdisc_dev(sch)));  in get_memory_limit()
712 	struct Qdisc *sch = q->sch;  in dualpi2_timer()  local
716 	root_lock = qdisc_lock(qdisc_root_sleeping(sch));  in dualpi2_timer()
719 	WRITE_ONCE(q->pi2_prob, calculate_probability(sch));  in dualpi2_timer()
758 static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt,  in dualpi2_change()  argument
780 	q = qdisc_priv(sch);  in dualpi2_change()
781 	sch_tree_lock(sch);  in dualpi2_change()
786 		WRITE_ONCE(sch->limit, limit);  in dualpi2_change()
787 		WRITE_ONCE(q->memory_limit, get_memory_limit(sch, limit));  in dualpi2_change()
855 		dualpi2_calculate_c_protection(sch, q, wc);  in dualpi2_change()
870 	old_qlen = qdisc_qlen(sch);  in dualpi2_change()
871 	old_backlog = sch->qstats.backlog;  in dualpi2_change()
872 	while (qdisc_qlen(sch) > sch->limit ||  in dualpi2_change()
874 		struct sk_buff *skb = qdisc_dequeue_internal(sch, true);  in dualpi2_change()
877 		qdisc_qstats_backlog_dec(sch, skb);  in dualpi2_change()
878 		rtnl_qdisc_drop(skb, sch);  in dualpi2_change()
880 	qdisc_tree_reduce_backlog(sch, old_qlen - qdisc_qlen(sch),  in dualpi2_change()
881 				  old_backlog - sch->qstats.backlog);  in dualpi2_change()
883 	sch_tree_unlock(sch);  in dualpi2_change()
888 static void dualpi2_reset_default(struct Qdisc *sch)  in dualpi2_reset_default()  argument
890 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_reset_default()
892 	q->sch->limit = 10000;				/* Max 125ms at 1Gbps */  in dualpi2_reset_default()
893 	q->memory_limit = get_memory_limit(sch, q->sch->limit);  in dualpi2_reset_default()
903 	dualpi2_calculate_c_protection(q->sch, q, 10);	/* wc=10%, wl=90% */  in dualpi2_reset_default()
913 static int dualpi2_init(struct Qdisc *sch, struct nlattr *opt,  in dualpi2_init()  argument
916 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_init()
919 	q->l_queue = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,  in dualpi2_init()
920 				       TC_H_MAKE(sch->handle, 1), extack);  in dualpi2_init()
924 	err = tcf_block_get(&q->tcf_block, &q->tcf_filters, sch, extack);  in dualpi2_init()
928 	q->sch = sch;  in dualpi2_init()
929 	dualpi2_reset_default(sch);  in dualpi2_init()
934 		err = dualpi2_change(sch, opt, extack);  in dualpi2_init()
945 static int dualpi2_dump(struct Qdisc *sch, struct sk_buff *skb)  in dualpi2_dump()  argument
947 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_dump()
960 	    (nla_put_u32(skb, TCA_DUALPI2_LIMIT, READ_ONCE(sch->limit)) ||  in dualpi2_dump()
987 	    (nla_put_u32(skb, TCA_DUALPI2_LIMIT, READ_ONCE(sch->limit)) ||  in dualpi2_dump()
1021 static int dualpi2_dump_stats(struct Qdisc *sch, struct gnet_dump *d)  in dualpi2_dump_stats()  argument
1023 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_dump_stats()
1048 static void dualpi2_reset(struct Qdisc *sch)  in dualpi2_reset()  argument
1050 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_reset()
1052 	qdisc_reset_queue(sch);  in dualpi2_reset()
1067 static void dualpi2_destroy(struct Qdisc *sch)  in dualpi2_destroy()  argument
1069 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_destroy()
1078 static struct Qdisc *dualpi2_leaf(struct Qdisc *sch, unsigned long arg)  in dualpi2_leaf()  argument
1083 static unsigned long dualpi2_find(struct Qdisc *sch, u32 classid)  in dualpi2_find()  argument
1088 static unsigned long dualpi2_bind(struct Qdisc *sch, unsigned long parent,  in dualpi2_bind()  argument
1098 static struct tcf_block *dualpi2_tcf_block(struct Qdisc *sch, unsigned long cl,  in dualpi2_tcf_block()  argument
1101 	struct dualpi2_sched_data *q = qdisc_priv(sch);  in dualpi2_tcf_block()
1108 static void dualpi2_walk(struct Qdisc *sch, struct qdisc_walker *arg)  in dualpi2_walk()  argument
1121 		if (arg->fn(sch, i + 1, arg) < 0) {  in dualpi2_walk()