1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/uaccess.h>
18#include <linux/bitops.h>
19#include <linux/module.h>
20#include <linux/types.h>
21#include <linux/kernel.h>
22#include <linux/jiffies.h>
23#include <linux/string.h>
24#include <linux/mm.h>
25#include <linux/socket.h>
26#include <linux/sockios.h>
27#include <linux/in.h>
28#include <linux/errno.h>
29#include <linux/interrupt.h>
30#include <linux/netdevice.h>
31#include <linux/skbuff.h>
32#include <linux/rtnetlink.h>
33#include <linux/init.h>
34#include <linux/slab.h>
35#include <linux/seqlock.h>
36#include <net/sock.h>
37#include <net/gen_stats.h>
38
39
40
41
42
43
44
45
46struct net_rate_estimator {
47 struct gnet_stats_basic_packed *bstats;
48 spinlock_t *stats_lock;
49 seqcount_t *running;
50 struct gnet_stats_basic_cpu __percpu *cpu_bstats;
51 u8 ewma_log;
52 u8 intvl_log;
53
54 seqcount_t seq;
55 u32 last_packets;
56 u64 last_bytes;
57
58 u64 avpps;
59 u64 avbps;
60
61 unsigned long next_jiffies;
62 struct timer_list timer;
63 struct rcu_head rcu;
64};
65
66static void est_fetch_counters(struct net_rate_estimator *e,
67 struct gnet_stats_basic_packed *b)
68{
69 if (e->stats_lock)
70 spin_lock(e->stats_lock);
71
72 __gnet_stats_copy_basic(e->running, b, e->cpu_bstats, e->bstats);
73
74 if (e->stats_lock)
75 spin_unlock(e->stats_lock);
76
77}
78
79static void est_timer(unsigned long arg)
80{
81 struct net_rate_estimator *est = (struct net_rate_estimator *)arg;
82 struct gnet_stats_basic_packed b;
83 u64 rate, brate;
84
85 est_fetch_counters(est, &b);
86 brate = (b.bytes - est->last_bytes) << (8 - est->ewma_log);
87 brate -= (est->avbps >> est->ewma_log);
88
89 rate = (u64)(b.packets - est->last_packets) << (8 - est->ewma_log);
90 rate -= (est->avpps >> est->ewma_log);
91
92 write_seqcount_begin(&est->seq);
93 est->avbps += brate;
94 est->avpps += rate;
95 write_seqcount_end(&est->seq);
96
97 est->last_bytes = b.bytes;
98 est->last_packets = b.packets;
99
100 est->next_jiffies += ((HZ/4) << est->intvl_log);
101
102 if (unlikely(time_after_eq(jiffies, est->next_jiffies))) {
103
104 est->next_jiffies = jiffies + 1;
105 }
106 mod_timer(&est->timer, est->next_jiffies);
107}
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
128 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
129 struct net_rate_estimator __rcu **rate_est,
130 spinlock_t *stats_lock,
131 seqcount_t *running,
132 struct nlattr *opt)
133{
134 struct gnet_estimator *parm = nla_data(opt);
135 struct net_rate_estimator *old, *est;
136 struct gnet_stats_basic_packed b;
137 int intvl_log;
138
139 if (nla_len(opt) < sizeof(*parm))
140 return -EINVAL;
141
142
143
144
145
146 if (parm->interval < -2 || parm->interval > 3)
147 return -EINVAL;
148
149 est = kzalloc(sizeof(*est), GFP_KERNEL);
150 if (!est)
151 return -ENOBUFS;
152
153 seqcount_init(&est->seq);
154 intvl_log = parm->interval + 2;
155 est->bstats = bstats;
156 est->stats_lock = stats_lock;
157 est->running = running;
158 est->ewma_log = parm->ewma_log;
159 est->intvl_log = intvl_log;
160 est->cpu_bstats = cpu_bstats;
161
162 est_fetch_counters(est, &b);
163 est->last_bytes = b.bytes;
164 est->last_packets = b.packets;
165 old = rcu_dereference_protected(*rate_est, 1);
166 if (old) {
167 del_timer_sync(&old->timer);
168 est->avbps = old->avbps;
169 est->avpps = old->avpps;
170 }
171
172 est->next_jiffies = jiffies + ((HZ/4) << intvl_log);
173 setup_timer(&est->timer, est_timer, (unsigned long)est);
174 mod_timer(&est->timer, est->next_jiffies);
175
176 rcu_assign_pointer(*rate_est, est);
177 if (old)
178 kfree_rcu(old, rcu);
179 return 0;
180}
181EXPORT_SYMBOL(gen_new_estimator);
182
183
184
185
186
187
188
189
190void gen_kill_estimator(struct net_rate_estimator __rcu **rate_est)
191{
192 struct net_rate_estimator *est;
193
194 est = xchg((__force struct net_rate_estimator **)rate_est, NULL);
195 if (est) {
196 del_timer_sync(&est->timer);
197 kfree_rcu(est, rcu);
198 }
199}
200EXPORT_SYMBOL(gen_kill_estimator);
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
217 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
218 struct net_rate_estimator __rcu **rate_est,
219 spinlock_t *stats_lock,
220 seqcount_t *running, struct nlattr *opt)
221{
222 return gen_new_estimator(bstats, cpu_bstats, rate_est,
223 stats_lock, running, opt);
224}
225EXPORT_SYMBOL(gen_replace_estimator);
226
227
228
229
230
231
232
233bool gen_estimator_active(struct net_rate_estimator __rcu **rate_est)
234{
235 return !!rcu_access_pointer(*rate_est);
236}
237EXPORT_SYMBOL(gen_estimator_active);
238
239bool gen_estimator_read(struct net_rate_estimator __rcu **rate_est,
240 struct gnet_stats_rate_est64 *sample)
241{
242 struct net_rate_estimator *est;
243 unsigned seq;
244
245 rcu_read_lock();
246 est = rcu_dereference(*rate_est);
247 if (!est) {
248 rcu_read_unlock();
249 return false;
250 }
251
252 do {
253 seq = read_seqcount_begin(&est->seq);
254 sample->bps = est->avbps >> 8;
255 sample->pps = est->avpps >> 8;
256 } while (read_seqcount_retry(&est->seq, seq));
257
258 rcu_read_unlock();
259 return true;
260}
261EXPORT_SYMBOL(gen_estimator_read);
262