1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/mm.h>
28#include <linux/module.h>
29#include <linux/math64.h>
30#include <net/tcp.h>
31#include <linux/inet_diag.h>
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57static int nv_pad __read_mostly = 10;
58static int nv_pad_buffer __read_mostly = 2;
59static int nv_reset_period __read_mostly = 5;
60static int nv_min_cwnd __read_mostly = 2;
61static int nv_cong_dec_mult __read_mostly = 30 * 128 / 100;
62static int nv_ssthresh_factor __read_mostly = 8;
63static int nv_rtt_factor __read_mostly = 128;
64static int nv_loss_dec_factor __read_mostly = 512;
65static int nv_cwnd_growth_rate_neg __read_mostly = 8;
66static int nv_cwnd_growth_rate_pos __read_mostly;
67static int nv_dec_eval_min_calls __read_mostly = 60;
68static int nv_inc_eval_min_calls __read_mostly = 20;
69static int nv_ssthresh_eval_min_calls __read_mostly = 30;
70static int nv_stop_rtt_cnt __read_mostly = 10;
71static int nv_rtt_min_cnt __read_mostly = 2;
72
73module_param(nv_pad, int, 0644);
74MODULE_PARM_DESC(nv_pad, "max queued packets allowed in network");
75module_param(nv_reset_period, int, 0644);
76MODULE_PARM_DESC(nv_reset_period, "nv_min_rtt reset period (secs)");
77module_param(nv_min_cwnd, int, 0644);
78MODULE_PARM_DESC(nv_min_cwnd, "NV will not decrease cwnd below this value"
79 " without losses");
80
81
82struct tcpnv {
83 unsigned long nv_min_rtt_reset_jiffies;
84
85 s8 cwnd_growth_factor;
86
87 u8 available8;
88 u16 available16;
89 u8 nv_allow_cwnd_growth:1,
90 nv_reset:1,
91 nv_catchup:1;
92
93 u8 nv_eval_call_cnt;
94 u8 nv_min_cwnd;
95
96
97
98
99
100 u8 nv_rtt_cnt; ;
101 u32 nv_last_rtt;
102 u32 nv_min_rtt;
103 u32 nv_min_rtt_new;
104 u32 nv_rtt_max_rate;
105 u32 nv_rtt_start_seq;
106
107 u32 nv_last_snd_una;
108
109
110 u32 nv_no_cong_cnt;
111};
112
113#define NV_INIT_RTT U32_MAX
114#define NV_MIN_CWND 4
115#define NV_MIN_CWND_GROW 2
116#define NV_TSO_CWND_BOUND 80
117
118static inline void tcpnv_reset(struct tcpnv *ca, struct sock *sk)
119{
120 struct tcp_sock *tp = tcp_sk(sk);
121
122 ca->nv_reset = 0;
123 ca->nv_no_cong_cnt = 0;
124 ca->nv_rtt_cnt = 0;
125 ca->nv_last_rtt = 0;
126 ca->nv_rtt_max_rate = 0;
127 ca->nv_rtt_start_seq = tp->snd_una;
128 ca->nv_eval_call_cnt = 0;
129 ca->nv_last_snd_una = tp->snd_una;
130}
131
132static void tcpnv_init(struct sock *sk)
133{
134 struct tcpnv *ca = inet_csk_ca(sk);
135
136 tcpnv_reset(ca, sk);
137
138 ca->nv_allow_cwnd_growth = 1;
139 ca->nv_min_rtt_reset_jiffies = jiffies + 2 * HZ;
140 ca->nv_min_rtt = NV_INIT_RTT;
141 ca->nv_min_rtt_new = NV_INIT_RTT;
142 ca->nv_min_cwnd = NV_MIN_CWND;
143 ca->nv_catchup = 0;
144 ca->cwnd_growth_factor = 0;
145}
146
147static void tcpnv_cong_avoid(struct sock *sk, u32 ack, u32 acked)
148{
149 struct tcp_sock *tp = tcp_sk(sk);
150 struct tcpnv *ca = inet_csk_ca(sk);
151 u32 cnt;
152
153 if (!tcp_is_cwnd_limited(sk))
154 return;
155
156
157 if (!ca->nv_allow_cwnd_growth)
158 return;
159
160 if (tcp_in_slow_start(tp)) {
161 acked = tcp_slow_start(tp, acked);
162 if (!acked)
163 return;
164 }
165
166 if (ca->cwnd_growth_factor < 0) {
167 cnt = tp->snd_cwnd << -ca->cwnd_growth_factor;
168 tcp_cong_avoid_ai(tp, cnt, acked);
169 } else {
170 cnt = max(4U, tp->snd_cwnd >> ca->cwnd_growth_factor);
171 tcp_cong_avoid_ai(tp, cnt, acked);
172 }
173}
174
175static u32 tcpnv_recalc_ssthresh(struct sock *sk)
176{
177 const struct tcp_sock *tp = tcp_sk(sk);
178
179 return max((tp->snd_cwnd * nv_loss_dec_factor) >> 10, 2U);
180}
181
182static void tcpnv_state(struct sock *sk, u8 new_state)
183{
184 struct tcpnv *ca = inet_csk_ca(sk);
185
186 if (new_state == TCP_CA_Open && ca->nv_reset) {
187 tcpnv_reset(ca, sk);
188 } else if (new_state == TCP_CA_Loss || new_state == TCP_CA_CWR ||
189 new_state == TCP_CA_Recovery) {
190 ca->nv_reset = 1;
191 ca->nv_allow_cwnd_growth = 0;
192 if (new_state == TCP_CA_Loss) {
193
194 if (ca->cwnd_growth_factor > 0)
195 ca->cwnd_growth_factor = 0;
196
197 if (nv_cwnd_growth_rate_neg > 0 &&
198 ca->cwnd_growth_factor > -8)
199 ca->cwnd_growth_factor--;
200 }
201 }
202}
203
204
205
206static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample)
207{
208 const struct inet_connection_sock *icsk = inet_csk(sk);
209 struct tcp_sock *tp = tcp_sk(sk);
210 struct tcpnv *ca = inet_csk_ca(sk);
211 unsigned long now = jiffies;
212 s64 rate64 = 0;
213 u32 rate, max_win, cwnd_by_slope;
214 u32 avg_rtt;
215 u32 bytes_acked = 0;
216
217
218 if (sample->rtt_us < 0)
219 return;
220
221
222 if (icsk->icsk_ca_state != TCP_CA_Open &&
223 icsk->icsk_ca_state != TCP_CA_Disorder)
224 return;
225
226
227 if (ca->nv_catchup && tp->snd_cwnd >= nv_min_cwnd) {
228 ca->nv_catchup = 0;
229 ca->nv_allow_cwnd_growth = 0;
230 }
231
232 bytes_acked = tp->snd_una - ca->nv_last_snd_una;
233 ca->nv_last_snd_una = tp->snd_una;
234
235 if (sample->in_flight == 0)
236 return;
237
238
239 if (nv_rtt_factor > 0) {
240 if (ca->nv_last_rtt > 0) {
241 avg_rtt = (((u64)sample->rtt_us) * nv_rtt_factor +
242 ((u64)ca->nv_last_rtt)
243 * (256 - nv_rtt_factor)) >> 8;
244 } else {
245 avg_rtt = sample->rtt_us;
246 ca->nv_min_rtt = avg_rtt << 1;
247 }
248 ca->nv_last_rtt = avg_rtt;
249 } else {
250 avg_rtt = sample->rtt_us;
251 }
252
253
254 rate64 = ((u64)sample->in_flight) * 8000000;
255 rate = (u32)div64_u64(rate64, (u64)(avg_rtt ?: 1) * 100);
256
257
258
259
260
261 if (ca->nv_rtt_max_rate < rate)
262 ca->nv_rtt_max_rate = rate;
263
264
265 if (ca->nv_eval_call_cnt < 255)
266 ca->nv_eval_call_cnt++;
267
268
269 if (avg_rtt < ca->nv_min_rtt)
270 ca->nv_min_rtt = avg_rtt;
271
272
273 if (avg_rtt < ca->nv_min_rtt_new)
274 ca->nv_min_rtt_new = avg_rtt;
275
276
277
278
279
280
281
282
283
284
285 if (time_after_eq(now, ca->nv_min_rtt_reset_jiffies)) {
286 unsigned char rand;
287
288 ca->nv_min_rtt = ca->nv_min_rtt_new;
289 ca->nv_min_rtt_new = NV_INIT_RTT;
290 get_random_bytes(&rand, 1);
291 ca->nv_min_rtt_reset_jiffies =
292 now + ((nv_reset_period * (384 + rand) * HZ) >> 9);
293
294
295
296 ca->nv_min_cwnd = max(ca->nv_min_cwnd / 2, NV_MIN_CWND);
297 }
298
299
300 if (before(ca->nv_rtt_start_seq, tp->snd_una)) {
301 ca->nv_rtt_start_seq = tp->snd_nxt;
302 if (ca->nv_rtt_cnt < 0xff)
303
304 ca->nv_rtt_cnt++;
305
306
307
308
309
310
311 if (ca->nv_eval_call_cnt == 1 &&
312 bytes_acked >= (ca->nv_min_cwnd - 1) * tp->mss_cache &&
313 ca->nv_min_cwnd < (NV_TSO_CWND_BOUND + 1)) {
314 ca->nv_min_cwnd = min(ca->nv_min_cwnd
315 + NV_MIN_CWND_GROW,
316 NV_TSO_CWND_BOUND + 1);
317 ca->nv_rtt_start_seq = tp->snd_nxt +
318 ca->nv_min_cwnd * tp->mss_cache;
319 ca->nv_eval_call_cnt = 0;
320 ca->nv_allow_cwnd_growth = 1;
321 return;
322 }
323
324
325
326
327
328 cwnd_by_slope = (u32)
329 div64_u64(((u64)ca->nv_rtt_max_rate) * ca->nv_min_rtt,
330 (u64)(80000 * tp->mss_cache));
331 max_win = cwnd_by_slope + nv_pad;
332
333
334
335
336
337 if (tp->snd_cwnd > max_win) {
338
339
340
341
342
343
344
345 if (ca->nv_rtt_cnt < nv_rtt_min_cnt) {
346 return;
347 } else if (tp->snd_ssthresh == TCP_INFINITE_SSTHRESH) {
348 if (ca->nv_eval_call_cnt <
349 nv_ssthresh_eval_min_calls)
350 return;
351
352 } else if (ca->nv_eval_call_cnt <
353 nv_dec_eval_min_calls) {
354 if (ca->nv_allow_cwnd_growth &&
355 ca->nv_rtt_cnt > nv_stop_rtt_cnt)
356 ca->nv_allow_cwnd_growth = 0;
357 return;
358 }
359
360
361 ca->nv_allow_cwnd_growth = 0;
362 tp->snd_ssthresh =
363 (nv_ssthresh_factor * max_win) >> 3;
364 if (tp->snd_cwnd - max_win > 2) {
365
366 int dec;
367
368 dec = max(2U, ((tp->snd_cwnd - max_win) *
369 nv_cong_dec_mult) >> 7);
370 tp->snd_cwnd -= dec;
371 } else if (nv_cong_dec_mult > 0) {
372 tp->snd_cwnd = max_win;
373 }
374 if (ca->cwnd_growth_factor > 0)
375 ca->cwnd_growth_factor = 0;
376 ca->nv_no_cong_cnt = 0;
377 } else if (tp->snd_cwnd <= max_win - nv_pad_buffer) {
378
379 if (ca->nv_eval_call_cnt < nv_inc_eval_min_calls)
380 return;
381
382 ca->nv_allow_cwnd_growth = 1;
383 ca->nv_no_cong_cnt++;
384 if (ca->cwnd_growth_factor < 0 &&
385 nv_cwnd_growth_rate_neg > 0 &&
386 ca->nv_no_cong_cnt > nv_cwnd_growth_rate_neg) {
387 ca->cwnd_growth_factor++;
388 ca->nv_no_cong_cnt = 0;
389 } else if (ca->cwnd_growth_factor >= 0 &&
390 nv_cwnd_growth_rate_pos > 0 &&
391 ca->nv_no_cong_cnt >
392 nv_cwnd_growth_rate_pos) {
393 ca->cwnd_growth_factor++;
394 ca->nv_no_cong_cnt = 0;
395 }
396 } else {
397
398 return;
399 }
400
401
402 ca->nv_eval_call_cnt = 0;
403 ca->nv_rtt_cnt = 0;
404 ca->nv_rtt_max_rate = 0;
405
406
407
408
409
410 if (tp->snd_cwnd < nv_min_cwnd)
411 tp->snd_cwnd = nv_min_cwnd;
412 }
413}
414
415
416static size_t tcpnv_get_info(struct sock *sk, u32 ext, int *attr,
417 union tcp_cc_info *info)
418{
419 const struct tcpnv *ca = inet_csk_ca(sk);
420
421 if (ext & (1 << (INET_DIAG_VEGASINFO - 1))) {
422 info->vegas.tcpv_enabled = 1;
423 info->vegas.tcpv_rttcnt = ca->nv_rtt_cnt;
424 info->vegas.tcpv_rtt = ca->nv_last_rtt;
425 info->vegas.tcpv_minrtt = ca->nv_min_rtt;
426
427 *attr = INET_DIAG_VEGASINFO;
428 return sizeof(struct tcpvegas_info);
429 }
430 return 0;
431}
432
433static struct tcp_congestion_ops tcpnv __read_mostly = {
434 .init = tcpnv_init,
435 .ssthresh = tcpnv_recalc_ssthresh,
436 .cong_avoid = tcpnv_cong_avoid,
437 .set_state = tcpnv_state,
438 .undo_cwnd = tcp_reno_undo_cwnd,
439 .pkts_acked = tcpnv_acked,
440 .get_info = tcpnv_get_info,
441
442 .owner = THIS_MODULE,
443 .name = "nv",
444};
445
446static int __init tcpnv_register(void)
447{
448 BUILD_BUG_ON(sizeof(struct tcpnv) > ICSK_CA_PRIV_SIZE);
449
450 return tcp_register_congestion_control(&tcpnv);
451}
452
453static void __exit tcpnv_unregister(void)
454{
455 tcp_unregister_congestion_control(&tcpnv);
456}
457
458module_init(tcpnv_register);
459module_exit(tcpnv_unregister);
460
461MODULE_AUTHOR("Lawrence Brakmo");
462MODULE_LICENSE("GPL");
463MODULE_DESCRIPTION("TCP NV");
464MODULE_VERSION("1.0");
465