1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/gen_estimator.c Simple rate estimator. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * Eric Dumazet <edumazet@google.com> 7 * 8 * Changes: 9 * Jamal Hadi Salim - moved it to net/core and reshulfed 10 * names to make it usable in general net subsystem. 11 */ 12 13 #include <linux/uaccess.h> 14 #include <linux/bitops.h> 15 #include <linux/module.h> 16 #include <linux/types.h> 17 #include <linux/kernel.h> 18 #include <linux/jiffies.h> 19 #include <linux/string.h> 20 #include <linux/mm.h> 21 #include <linux/socket.h> 22 #include <linux/sockios.h> 23 #include <linux/in.h> 24 #include <linux/errno.h> 25 #include <linux/interrupt.h> 26 #include <linux/netdevice.h> 27 #include <linux/skbuff.h> 28 #include <linux/rtnetlink.h> 29 #include <linux/init.h> 30 #include <linux/slab.h> 31 #include <linux/seqlock.h> 32 #include <net/sock.h> 33 #include <net/gen_stats.h> 34 35 /* This code is NOT intended to be used for statistics collection, 36 * its purpose is to provide a base for statistical multiplexing 37 * for controlled load service. 38 * If you need only statistics, run a user level daemon which 39 * periodically reads byte counters. 40 */ 41 42 struct net_rate_estimator { 43 struct gnet_stats_basic_sync *bstats; 44 spinlock_t *stats_lock; 45 bool running; 46 struct gnet_stats_basic_sync __percpu *cpu_bstats; 47 u8 ewma_log; 48 u8 intvl_log; /* period : (250ms << intvl_log) */ 49 50 seqcount_t seq; 51 u64 last_packets; 52 u64 last_bytes; 53 54 u64 avpps; 55 u64 avbps; 56 57 unsigned long next_jiffies; 58 struct timer_list timer; 59 struct rcu_head rcu; 60 }; 61 62 static void est_fetch_counters(struct net_rate_estimator *e, 63 struct gnet_stats_basic_sync *b) 64 { 65 gnet_stats_basic_sync_init(b); 66 if (e->stats_lock) 67 spin_lock(e->stats_lock); 68 69 gnet_stats_add_basic(b, e->cpu_bstats, e->bstats, e->running); 70 71 if (e->stats_lock) 72 spin_unlock(e->stats_lock); 73 74 } 75 76 static void est_timer(struct timer_list *t) 77 { 78 struct net_rate_estimator *est = timer_container_of(est, t, timer); 79 struct gnet_stats_basic_sync b; 80 u64 b_bytes, b_packets; 81 u64 rate, brate; 82 83 est_fetch_counters(est, &b); 84 b_bytes = u64_stats_read(&b.bytes); 85 b_packets = u64_stats_read(&b.packets); 86 87 brate = (b_bytes - est->last_bytes) << (10 - est->intvl_log); 88 brate = (brate >> est->ewma_log) - (est->avbps >> est->ewma_log); 89 90 rate = (b_packets - est->last_packets) << (10 - est->intvl_log); 91 rate = (rate >> est->ewma_log) - (est->avpps >> est->ewma_log); 92 93 preempt_disable_nested(); 94 write_seqcount_begin(&est->seq); 95 est->avbps += brate; 96 est->avpps += rate; 97 write_seqcount_end(&est->seq); 98 preempt_enable_nested(); 99 100 est->last_bytes = b_bytes; 101 est->last_packets = b_packets; 102 103 est->next_jiffies += ((HZ/4) << est->intvl_log); 104 105 if (unlikely(time_after_eq(jiffies, est->next_jiffies))) { 106 /* Ouch... timer was delayed. */ 107 est->next_jiffies = jiffies + 1; 108 } 109 mod_timer(&est->timer, est->next_jiffies); 110 } 111 112 /** 113 * gen_new_estimator - create a new rate estimator 114 * @bstats: basic statistics 115 * @cpu_bstats: bstats per cpu 116 * @rate_est: rate estimator statistics 117 * @lock: lock for statistics and control path 118 * @running: true if @bstats represents a running qdisc, thus @bstats' 119 * internal values might change during basic reads. Only used 120 * if @bstats_cpu is NULL 121 * @opt: rate estimator configuration TLV 122 * 123 * Creates a new rate estimator with &bstats as source and &rate_est 124 * as destination. A new timer with the interval specified in the 125 * configuration TLV is created. Upon each interval, the latest statistics 126 * will be read from &bstats and the estimated rate will be stored in 127 * &rate_est with the statistics lock grabbed during this period. 128 * 129 * Returns 0 on success or a negative error code. 130 * 131 */ 132 int gen_new_estimator(struct gnet_stats_basic_sync *bstats, 133 struct gnet_stats_basic_sync __percpu *cpu_bstats, 134 struct net_rate_estimator __rcu **rate_est, 135 spinlock_t *lock, 136 bool running, 137 struct nlattr *opt) 138 { 139 struct gnet_estimator *parm = nla_data(opt); 140 struct net_rate_estimator *old, *est; 141 struct gnet_stats_basic_sync b; 142 int intvl_log; 143 144 if (nla_len(opt) < sizeof(*parm)) 145 return -EINVAL; 146 147 /* allowed timer periods are : 148 * -2 : 250ms, -1 : 500ms, 0 : 1 sec 149 * 1 : 2 sec, 2 : 4 sec, 3 : 8 sec 150 */ 151 if (parm->interval < -2 || parm->interval > 3) 152 return -EINVAL; 153 154 if (parm->ewma_log == 0 || parm->ewma_log >= 31) 155 return -EINVAL; 156 157 est = kzalloc(sizeof(*est), GFP_KERNEL); 158 if (!est) 159 return -ENOBUFS; 160 161 seqcount_init(&est->seq); 162 intvl_log = parm->interval + 2; 163 est->bstats = bstats; 164 est->stats_lock = lock; 165 est->running = running; 166 est->ewma_log = parm->ewma_log; 167 est->intvl_log = intvl_log; 168 est->cpu_bstats = cpu_bstats; 169 170 if (lock) 171 local_bh_disable(); 172 est_fetch_counters(est, &b); 173 if (lock) 174 local_bh_enable(); 175 est->last_bytes = u64_stats_read(&b.bytes); 176 est->last_packets = u64_stats_read(&b.packets); 177 178 if (lock) 179 spin_lock_bh(lock); 180 old = rcu_dereference_protected(*rate_est, 1); 181 if (old) { 182 timer_delete_sync(&old->timer); 183 est->avbps = old->avbps; 184 est->avpps = old->avpps; 185 } 186 187 est->next_jiffies = jiffies + ((HZ/4) << intvl_log); 188 timer_setup(&est->timer, est_timer, 0); 189 mod_timer(&est->timer, est->next_jiffies); 190 191 rcu_assign_pointer(*rate_est, est); 192 if (lock) 193 spin_unlock_bh(lock); 194 if (old) 195 kfree_rcu(old, rcu); 196 return 0; 197 } 198 EXPORT_SYMBOL(gen_new_estimator); 199 200 /** 201 * gen_kill_estimator - remove a rate estimator 202 * @rate_est: rate estimator 203 * 204 * Removes the rate estimator. 205 * 206 */ 207 void gen_kill_estimator(struct net_rate_estimator __rcu **rate_est) 208 { 209 struct net_rate_estimator *est; 210 211 est = unrcu_pointer(xchg(rate_est, NULL)); 212 if (est) { 213 timer_shutdown_sync(&est->timer); 214 kfree_rcu(est, rcu); 215 } 216 } 217 EXPORT_SYMBOL(gen_kill_estimator); 218 219 /** 220 * gen_replace_estimator - replace rate estimator configuration 221 * @bstats: basic statistics 222 * @cpu_bstats: bstats per cpu 223 * @rate_est: rate estimator statistics 224 * @lock: lock for statistics and control path 225 * @running: true if @bstats represents a running qdisc, thus @bstats' 226 * internal values might change during basic reads. Only used 227 * if @cpu_bstats is NULL 228 * @opt: rate estimator configuration TLV 229 * 230 * Replaces the configuration of a rate estimator by calling 231 * gen_kill_estimator() and gen_new_estimator(). 232 * 233 * Returns 0 on success or a negative error code. 234 */ 235 int gen_replace_estimator(struct gnet_stats_basic_sync *bstats, 236 struct gnet_stats_basic_sync __percpu *cpu_bstats, 237 struct net_rate_estimator __rcu **rate_est, 238 spinlock_t *lock, 239 bool running, struct nlattr *opt) 240 { 241 return gen_new_estimator(bstats, cpu_bstats, rate_est, 242 lock, running, opt); 243 } 244 EXPORT_SYMBOL(gen_replace_estimator); 245 246 /** 247 * gen_estimator_active - test if estimator is currently in use 248 * @rate_est: rate estimator 249 * 250 * Returns true if estimator is active, and false if not. 251 */ 252 bool gen_estimator_active(struct net_rate_estimator __rcu **rate_est) 253 { 254 return !!rcu_access_pointer(*rate_est); 255 } 256 EXPORT_SYMBOL(gen_estimator_active); 257 258 bool gen_estimator_read(struct net_rate_estimator __rcu **rate_est, 259 struct gnet_stats_rate_est64 *sample) 260 { 261 struct net_rate_estimator *est; 262 unsigned seq; 263 264 rcu_read_lock(); 265 est = rcu_dereference(*rate_est); 266 if (!est) { 267 rcu_read_unlock(); 268 return false; 269 } 270 271 do { 272 seq = read_seqcount_begin(&est->seq); 273 sample->bps = est->avbps >> 8; 274 sample->pps = est->avpps >> 8; 275 } while (read_seqcount_retry(&est->seq, seq)); 276 277 rcu_read_unlock(); 278 return true; 279 } 280 EXPORT_SYMBOL(gen_estimator_read); 281