Lines Matching +full:1 +full:q
34 #define SFB_NUMBUCKETS (1 << SFB_BUCKET_SHIFT) /* N bins per Level */
35 #define SFB_BUCKET_MASK (SFB_NUMBUCKETS - 1)
69 u8 slot; /* current active bins (0 or 1) */
123 static void increment_one_qlen(u32 sfbhash, u32 slot, struct sfb_sched_data *q) in increment_one_qlen() argument
126 struct sfb_bucket *b = &q->bins[slot].bins[0][0]; in increment_one_qlen()
133 WRITE_ONCE(b[hash].qlen, b[hash].qlen + 1); in increment_one_qlen()
138 static void increment_qlen(const struct sfb_skb_cb *cb, struct sfb_sched_data *q) in increment_qlen() argument
144 increment_one_qlen(sfbhash, 0, q); in increment_qlen()
146 sfbhash = cb->hashes[1]; in increment_qlen()
148 increment_one_qlen(sfbhash, 1, q); in increment_qlen()
152 struct sfb_sched_data *q) in decrement_one_qlen() argument
155 struct sfb_bucket *b = &q->bins[slot].bins[0][0]; in decrement_one_qlen()
162 WRITE_ONCE(b[hash].qlen, b[hash].qlen - 1); in decrement_one_qlen()
167 static void decrement_qlen(const struct sk_buff *skb, struct sfb_sched_data *q) in decrement_qlen() argument
173 decrement_one_qlen(sfbhash, 0, q); in decrement_qlen()
175 sfbhash = sfb_hash(skb, 1); in decrement_qlen()
177 decrement_one_qlen(sfbhash, 1, q); in decrement_qlen()
180 static void decrement_prob(struct sfb_bucket *b, struct sfb_sched_data *q) in decrement_prob() argument
182 WRITE_ONCE(b->p_mark, prob_minus(b->p_mark, q->decrement)); in decrement_prob()
185 static void increment_prob(struct sfb_bucket *b, struct sfb_sched_data *q) in increment_prob() argument
187 WRITE_ONCE(b->p_mark, prob_plus(b->p_mark, q->increment)); in increment_prob()
190 static void sfb_zero_all_buckets(struct sfb_sched_data *q) in sfb_zero_all_buckets() argument
192 memset(&q->bins, 0, sizeof(q->bins)); in sfb_zero_all_buckets()
198 static u32 sfb_compute_qlen(u32 *prob_r, u32 *avgpm_r, const struct sfb_sched_data *q) in sfb_compute_qlen() argument
202 const struct sfb_bucket *b = &q->bins[q->slot].bins[0][0]; in sfb_compute_qlen()
221 static void sfb_init_perturbation(u32 slot, struct sfb_sched_data *q) in sfb_init_perturbation() argument
223 get_random_bytes(&q->bins[slot].perturbation, in sfb_init_perturbation()
224 sizeof(q->bins[slot].perturbation)); in sfb_init_perturbation()
227 static void sfb_swap_slot(struct sfb_sched_data *q) in sfb_swap_slot() argument
229 sfb_init_perturbation(q->slot, q); in sfb_swap_slot()
230 q->slot ^= 1; in sfb_swap_slot()
231 q->double_buffering = false; in sfb_swap_slot()
237 static bool sfb_rate_limit(struct sk_buff *skb, struct sfb_sched_data *q) in sfb_rate_limit() argument
239 if (q->penalty_rate == 0 || q->penalty_burst == 0) in sfb_rate_limit()
242 if (q->tokens_avail < 1) { in sfb_rate_limit()
243 unsigned long age = min(10UL * HZ, jiffies - q->token_time); in sfb_rate_limit()
245 q->tokens_avail = (age * q->penalty_rate) / HZ; in sfb_rate_limit()
246 if (q->tokens_avail > q->penalty_burst) in sfb_rate_limit()
247 q->tokens_avail = q->penalty_burst; in sfb_rate_limit()
248 q->token_time = jiffies; in sfb_rate_limit()
249 if (q->tokens_avail < 1) in sfb_rate_limit()
253 q->tokens_avail--; in sfb_rate_limit()
287 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_enqueue() local
289 struct Qdisc *child = q->qdisc; in sfb_enqueue()
296 u32 slot = q->slot; in sfb_enqueue()
299 if (unlikely(sch->q.qlen >= q->limit)) { in sfb_enqueue()
301 WRITE_ONCE(q->stats.queuedrop, in sfb_enqueue()
302 q->stats.queuedrop + 1); in sfb_enqueue()
306 if (q->rehash_interval > 0) { in sfb_enqueue()
307 unsigned long limit = q->rehash_time + q->rehash_interval; in sfb_enqueue()
310 sfb_swap_slot(q); in sfb_enqueue()
311 q->rehash_time = jiffies; in sfb_enqueue()
312 } else if (unlikely(!q->double_buffering && q->warmup_time > 0 && in sfb_enqueue()
313 time_after(jiffies, limit - q->warmup_time))) { in sfb_enqueue()
314 q->double_buffering = true; in sfb_enqueue()
318 fl = rcu_dereference_bh(q->filter_list); in sfb_enqueue()
325 sfbhash = siphash_1u32(salt, &q->bins[slot].perturbation); in sfb_enqueue()
327 sfbhash = skb_get_hash_perturb(skb, &q->bins[slot].perturbation); in sfb_enqueue()
332 sfbhash = 1; in sfb_enqueue()
337 struct sfb_bucket *b = &q->bins[slot].bins[i][hash]; in sfb_enqueue()
341 decrement_prob(b, q); in sfb_enqueue()
342 else if (b->qlen >= q->bin_size) in sfb_enqueue()
343 increment_prob(b, q); in sfb_enqueue()
350 slot ^= 1; in sfb_enqueue()
353 if (unlikely(minqlen >= q->max)) { in sfb_enqueue()
355 WRITE_ONCE(q->stats.bucketdrop, in sfb_enqueue()
356 q->stats.bucketdrop + 1); in sfb_enqueue()
362 if (q->double_buffering) { in sfb_enqueue()
364 &q->bins[slot].perturbation); in sfb_enqueue()
366 sfbhash = 1; in sfb_enqueue()
371 struct sfb_bucket *b = &q->bins[slot].bins[i][hash]; in sfb_enqueue()
375 decrement_prob(b, q); in sfb_enqueue()
376 else if (b->qlen >= q->bin_size) in sfb_enqueue()
377 increment_prob(b, q); in sfb_enqueue()
380 if (sfb_rate_limit(skb, q)) { in sfb_enqueue()
382 WRITE_ONCE(q->stats.penaltydrop, in sfb_enqueue()
383 q->stats.penaltydrop + 1); in sfb_enqueue()
399 WRITE_ONCE(q->stats.earlydrop, in sfb_enqueue()
400 q->stats.earlydrop + 1); in sfb_enqueue()
405 WRITE_ONCE(q->stats.marked, in sfb_enqueue()
406 q->stats.marked + 1); in sfb_enqueue()
408 WRITE_ONCE(q->stats.earlydrop, in sfb_enqueue()
409 q->stats.earlydrop + 1); in sfb_enqueue()
419 sch->q.qlen++; in sfb_enqueue()
420 increment_qlen(&cb, q); in sfb_enqueue()
422 WRITE_ONCE(q->stats.childdrop, in sfb_enqueue()
423 q->stats.childdrop + 1); in sfb_enqueue()
440 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_dequeue() local
441 struct Qdisc *child = q->qdisc; in sfb_dequeue()
449 sch->q.qlen--; in sfb_dequeue()
450 decrement_qlen(skb, q); in sfb_dequeue()
458 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_peek() local
459 struct Qdisc *child = q->qdisc; in sfb_peek()
468 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_reset() local
470 if (likely(q->qdisc)) in sfb_reset()
471 qdisc_reset(q->qdisc); in sfb_reset()
472 q->slot = 0; in sfb_reset()
473 q->double_buffering = false; in sfb_reset()
474 sfb_zero_all_buckets(q); in sfb_reset()
475 sfb_init_perturbation(0, q); in sfb_reset()
480 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_destroy() local
482 tcf_block_put(q->block); in sfb_destroy()
483 qdisc_put(q->qdisc); in sfb_destroy()
486 static const struct nla_policy sfb_policy[TCA_SFB_MAX + 1] = {
505 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_change() local
507 struct nlattr *tb[TCA_SFB_MAX + 1]; in sfb_change()
536 qdisc_purge_queue(q->qdisc); in sfb_change()
537 old = q->qdisc; in sfb_change()
538 q->qdisc = child; in sfb_change()
540 q->rehash_interval = msecs_to_jiffies(ctl->rehash_interval); in sfb_change()
541 q->warmup_time = msecs_to_jiffies(ctl->warmup_time); in sfb_change()
542 q->rehash_time = jiffies; in sfb_change()
543 q->limit = limit; in sfb_change()
544 q->increment = ctl->increment; in sfb_change()
545 q->decrement = ctl->decrement; in sfb_change()
546 q->max = ctl->max; in sfb_change()
547 q->bin_size = ctl->bin_size; in sfb_change()
548 q->penalty_rate = ctl->penalty_rate; in sfb_change()
549 q->penalty_burst = ctl->penalty_burst; in sfb_change()
550 q->tokens_avail = ctl->penalty_burst; in sfb_change()
551 q->token_time = jiffies; in sfb_change()
553 q->slot = 0; in sfb_change()
554 q->double_buffering = false; in sfb_change()
555 sfb_zero_all_buckets(q); in sfb_change()
556 sfb_init_perturbation(0, q); in sfb_change()
557 sfb_init_perturbation(1, q); in sfb_change()
568 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_init() local
571 err = tcf_block_get(&q->block, &q->filter_list, sch, extack); in sfb_init()
575 q->qdisc = &noop_qdisc; in sfb_init()
581 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_dump() local
584 .rehash_interval = jiffies_to_msecs(q->rehash_interval), in sfb_dump()
585 .warmup_time = jiffies_to_msecs(q->warmup_time), in sfb_dump()
586 .limit = q->limit, in sfb_dump()
587 .max = q->max, in sfb_dump()
588 .bin_size = q->bin_size, in sfb_dump()
589 .increment = q->increment, in sfb_dump()
590 .decrement = q->decrement, in sfb_dump()
591 .penalty_rate = q->penalty_rate, in sfb_dump()
592 .penalty_burst = q->penalty_burst, in sfb_dump()
595 sch->qstats.backlog = q->qdisc->qstats.backlog; in sfb_dump()
610 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_dump_stats() local
612 .earlydrop = READ_ONCE(q->stats.earlydrop), in sfb_dump_stats()
613 .penaltydrop = READ_ONCE(q->stats.penaltydrop), in sfb_dump_stats()
614 .bucketdrop = READ_ONCE(q->stats.bucketdrop), in sfb_dump_stats()
615 .queuedrop = READ_ONCE(q->stats.queuedrop), in sfb_dump_stats()
616 .childdrop = READ_ONCE(q->stats.childdrop), in sfb_dump_stats()
617 .marked = READ_ONCE(q->stats.marked), in sfb_dump_stats()
620 st.maxqlen = sfb_compute_qlen(&st.maxprob, &st.avgprob, q); in sfb_dump_stats()
634 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_graft() local
639 *old = qdisc_replace(sch, new, &q->qdisc); in sfb_graft()
645 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_leaf() local
647 return q->qdisc; in sfb_leaf()
652 return 1; in sfb_find()
675 tc_qdisc_stats_dump(sch, 1, walker); in sfb_walk()
682 struct sfb_sched_data *q = qdisc_priv(sch); in sfb_tcf_block() local
686 return q->block; in sfb_tcf_block()