1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/mm.h>
28#include <linux/module.h>
29#include <linux/math64.h>
30#include <net/tcp.h>
31#include <linux/inet_diag.h>
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57static int nv_pad __read_mostly = 10;
58static int nv_pad_buffer __read_mostly = 2;
59static int nv_reset_period __read_mostly = 5;
60static int nv_min_cwnd __read_mostly = 2;
61static int nv_cong_dec_mult __read_mostly = 30 * 128 / 100;
62static int nv_ssthresh_factor __read_mostly = 8;
63static int nv_rtt_factor __read_mostly = 128;
64static int nv_loss_dec_factor __read_mostly = 819;
65static int nv_cwnd_growth_rate_neg __read_mostly = 8;
66static int nv_cwnd_growth_rate_pos __read_mostly;
67static int nv_dec_eval_min_calls __read_mostly = 60;
68static int nv_inc_eval_min_calls __read_mostly = 20;
69static int nv_ssthresh_eval_min_calls __read_mostly = 30;
70static int nv_stop_rtt_cnt __read_mostly = 10;
71static int nv_rtt_min_cnt __read_mostly = 2;
72
73module_param(nv_pad, int, 0644);
74MODULE_PARM_DESC(nv_pad, "max queued packets allowed in network");
75module_param(nv_reset_period, int, 0644);
76MODULE_PARM_DESC(nv_reset_period, "nv_min_rtt reset period (secs)");
77module_param(nv_min_cwnd, int, 0644);
78MODULE_PARM_DESC(nv_min_cwnd, "NV will not decrease cwnd below this value"
79 " without losses");
80
81
82struct tcpnv {
83 unsigned long nv_min_rtt_reset_jiffies;
84
85 s8 cwnd_growth_factor;
86
87 u8 available8;
88 u16 available16;
89 u8 nv_allow_cwnd_growth:1,
90 nv_reset:1,
91 nv_catchup:1;
92
93 u8 nv_eval_call_cnt;
94 u8 nv_min_cwnd;
95
96
97
98
99
100 u8 nv_rtt_cnt; ;
101 u32 nv_last_rtt;
102 u32 nv_min_rtt;
103 u32 nv_min_rtt_new;
104 u32 nv_base_rtt;
105
106 u32 nv_lower_bound_rtt;
107
108
109 u32 nv_rtt_max_rate;
110 u32 nv_rtt_start_seq;
111
112 u32 nv_last_snd_una;
113
114
115 u32 nv_no_cong_cnt;
116};
117
118#define NV_INIT_RTT U32_MAX
119#define NV_MIN_CWND 4
120#define NV_MIN_CWND_GROW 2
121#define NV_TSO_CWND_BOUND 80
122
123static inline void tcpnv_reset(struct tcpnv *ca, struct sock *sk)
124{
125 struct tcp_sock *tp = tcp_sk(sk);
126
127 ca->nv_reset = 0;
128 ca->nv_no_cong_cnt = 0;
129 ca->nv_rtt_cnt = 0;
130 ca->nv_last_rtt = 0;
131 ca->nv_rtt_max_rate = 0;
132 ca->nv_rtt_start_seq = tp->snd_una;
133 ca->nv_eval_call_cnt = 0;
134 ca->nv_last_snd_una = tp->snd_una;
135}
136
137static void tcpnv_init(struct sock *sk)
138{
139 struct tcpnv *ca = inet_csk_ca(sk);
140 int base_rtt;
141
142 tcpnv_reset(ca, sk);
143
144
145
146
147
148
149 base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT, 0, NULL);
150 if (base_rtt > 0) {
151 ca->nv_base_rtt = base_rtt;
152 ca->nv_lower_bound_rtt = (base_rtt * 205) >> 8;
153 } else {
154 ca->nv_base_rtt = 0;
155 ca->nv_lower_bound_rtt = 0;
156 }
157
158 ca->nv_allow_cwnd_growth = 1;
159 ca->nv_min_rtt_reset_jiffies = jiffies + 2 * HZ;
160 ca->nv_min_rtt = NV_INIT_RTT;
161 ca->nv_min_rtt_new = NV_INIT_RTT;
162 ca->nv_min_cwnd = NV_MIN_CWND;
163 ca->nv_catchup = 0;
164 ca->cwnd_growth_factor = 0;
165}
166
167
168
169
170inline u32 nv_get_bounded_rtt(struct tcpnv *ca, u32 val)
171{
172 if (ca->nv_lower_bound_rtt > 0 && val < ca->nv_lower_bound_rtt)
173 return ca->nv_lower_bound_rtt;
174 else if (ca->nv_base_rtt > 0 && val > ca->nv_base_rtt)
175 return ca->nv_base_rtt;
176 else
177 return val;
178}
179
180static void tcpnv_cong_avoid(struct sock *sk, u32 ack, u32 acked)
181{
182 struct tcp_sock *tp = tcp_sk(sk);
183 struct tcpnv *ca = inet_csk_ca(sk);
184 u32 cnt;
185
186 if (!tcp_is_cwnd_limited(sk))
187 return;
188
189
190 if (!ca->nv_allow_cwnd_growth)
191 return;
192
193 if (tcp_in_slow_start(tp)) {
194 acked = tcp_slow_start(tp, acked);
195 if (!acked)
196 return;
197 }
198
199 if (ca->cwnd_growth_factor < 0) {
200 cnt = tp->snd_cwnd << -ca->cwnd_growth_factor;
201 tcp_cong_avoid_ai(tp, cnt, acked);
202 } else {
203 cnt = max(4U, tp->snd_cwnd >> ca->cwnd_growth_factor);
204 tcp_cong_avoid_ai(tp, cnt, acked);
205 }
206}
207
208static u32 tcpnv_recalc_ssthresh(struct sock *sk)
209{
210 const struct tcp_sock *tp = tcp_sk(sk);
211
212 return max((tp->snd_cwnd * nv_loss_dec_factor) >> 10, 2U);
213}
214
215static void tcpnv_state(struct sock *sk, u8 new_state)
216{
217 struct tcpnv *ca = inet_csk_ca(sk);
218
219 if (new_state == TCP_CA_Open && ca->nv_reset) {
220 tcpnv_reset(ca, sk);
221 } else if (new_state == TCP_CA_Loss || new_state == TCP_CA_CWR ||
222 new_state == TCP_CA_Recovery) {
223 ca->nv_reset = 1;
224 ca->nv_allow_cwnd_growth = 0;
225 if (new_state == TCP_CA_Loss) {
226
227 if (ca->cwnd_growth_factor > 0)
228 ca->cwnd_growth_factor = 0;
229
230 if (nv_cwnd_growth_rate_neg > 0 &&
231 ca->cwnd_growth_factor > -8)
232 ca->cwnd_growth_factor--;
233 }
234 }
235}
236
237
238
239static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample)
240{
241 const struct inet_connection_sock *icsk = inet_csk(sk);
242 struct tcp_sock *tp = tcp_sk(sk);
243 struct tcpnv *ca = inet_csk_ca(sk);
244 unsigned long now = jiffies;
245 u64 rate64;
246 u32 rate, max_win, cwnd_by_slope;
247 u32 avg_rtt;
248 u32 bytes_acked = 0;
249
250
251 if (sample->rtt_us < 0)
252 return;
253
254
255 if (icsk->icsk_ca_state != TCP_CA_Open &&
256 icsk->icsk_ca_state != TCP_CA_Disorder)
257 return;
258
259
260 if (ca->nv_catchup && tp->snd_cwnd >= nv_min_cwnd) {
261 ca->nv_catchup = 0;
262 ca->nv_allow_cwnd_growth = 0;
263 }
264
265 bytes_acked = tp->snd_una - ca->nv_last_snd_una;
266 ca->nv_last_snd_una = tp->snd_una;
267
268 if (sample->in_flight == 0)
269 return;
270
271
272 if (nv_rtt_factor > 0) {
273 if (ca->nv_last_rtt > 0) {
274 avg_rtt = (((u64)sample->rtt_us) * nv_rtt_factor +
275 ((u64)ca->nv_last_rtt)
276 * (256 - nv_rtt_factor)) >> 8;
277 } else {
278 avg_rtt = sample->rtt_us;
279 ca->nv_min_rtt = avg_rtt << 1;
280 }
281 ca->nv_last_rtt = avg_rtt;
282 } else {
283 avg_rtt = sample->rtt_us;
284 }
285
286
287 rate64 = ((u64)sample->in_flight) * 80000;
288 do_div(rate64, avg_rtt ?: 1);
289 rate = (u32)rate64;
290
291
292
293
294
295 if (ca->nv_rtt_max_rate < rate)
296 ca->nv_rtt_max_rate = rate;
297
298
299 if (ca->nv_eval_call_cnt < 255)
300 ca->nv_eval_call_cnt++;
301
302
303 avg_rtt = nv_get_bounded_rtt(ca, avg_rtt);
304
305
306 if (avg_rtt < ca->nv_min_rtt)
307 ca->nv_min_rtt = avg_rtt;
308
309
310 if (avg_rtt < ca->nv_min_rtt_new)
311 ca->nv_min_rtt_new = avg_rtt;
312
313
314
315
316
317
318
319
320
321
322 if (time_after_eq(now, ca->nv_min_rtt_reset_jiffies)) {
323 unsigned char rand;
324
325 ca->nv_min_rtt = ca->nv_min_rtt_new;
326 ca->nv_min_rtt_new = NV_INIT_RTT;
327 get_random_bytes(&rand, 1);
328 ca->nv_min_rtt_reset_jiffies =
329 now + ((nv_reset_period * (384 + rand) * HZ) >> 9);
330
331
332
333 ca->nv_min_cwnd = max(ca->nv_min_cwnd / 2, NV_MIN_CWND);
334 }
335
336
337 if (before(ca->nv_rtt_start_seq, tp->snd_una)) {
338 ca->nv_rtt_start_seq = tp->snd_nxt;
339 if (ca->nv_rtt_cnt < 0xff)
340
341 ca->nv_rtt_cnt++;
342
343
344
345
346
347
348 if (ca->nv_eval_call_cnt == 1 &&
349 bytes_acked >= (ca->nv_min_cwnd - 1) * tp->mss_cache &&
350 ca->nv_min_cwnd < (NV_TSO_CWND_BOUND + 1)) {
351 ca->nv_min_cwnd = min(ca->nv_min_cwnd
352 + NV_MIN_CWND_GROW,
353 NV_TSO_CWND_BOUND + 1);
354 ca->nv_rtt_start_seq = tp->snd_nxt +
355 ca->nv_min_cwnd * tp->mss_cache;
356 ca->nv_eval_call_cnt = 0;
357 ca->nv_allow_cwnd_growth = 1;
358 return;
359 }
360
361
362
363
364
365 cwnd_by_slope = (u32)
366 div64_u64(((u64)ca->nv_rtt_max_rate) * ca->nv_min_rtt,
367 80000ULL * tp->mss_cache);
368 max_win = cwnd_by_slope + nv_pad;
369
370
371
372
373
374 if (tp->snd_cwnd > max_win) {
375
376
377
378
379
380
381
382 if (ca->nv_rtt_cnt < nv_rtt_min_cnt) {
383 return;
384 } else if (tp->snd_ssthresh == TCP_INFINITE_SSTHRESH) {
385 if (ca->nv_eval_call_cnt <
386 nv_ssthresh_eval_min_calls)
387 return;
388
389 } else if (ca->nv_eval_call_cnt <
390 nv_dec_eval_min_calls) {
391 if (ca->nv_allow_cwnd_growth &&
392 ca->nv_rtt_cnt > nv_stop_rtt_cnt)
393 ca->nv_allow_cwnd_growth = 0;
394 return;
395 }
396
397
398 ca->nv_allow_cwnd_growth = 0;
399 tp->snd_ssthresh =
400 (nv_ssthresh_factor * max_win) >> 3;
401 if (tp->snd_cwnd - max_win > 2) {
402
403 int dec;
404
405 dec = max(2U, ((tp->snd_cwnd - max_win) *
406 nv_cong_dec_mult) >> 7);
407 tp->snd_cwnd -= dec;
408 } else if (nv_cong_dec_mult > 0) {
409 tp->snd_cwnd = max_win;
410 }
411 if (ca->cwnd_growth_factor > 0)
412 ca->cwnd_growth_factor = 0;
413 ca->nv_no_cong_cnt = 0;
414 } else if (tp->snd_cwnd <= max_win - nv_pad_buffer) {
415
416 if (ca->nv_eval_call_cnt < nv_inc_eval_min_calls)
417 return;
418
419 ca->nv_allow_cwnd_growth = 1;
420 ca->nv_no_cong_cnt++;
421 if (ca->cwnd_growth_factor < 0 &&
422 nv_cwnd_growth_rate_neg > 0 &&
423 ca->nv_no_cong_cnt > nv_cwnd_growth_rate_neg) {
424 ca->cwnd_growth_factor++;
425 ca->nv_no_cong_cnt = 0;
426 } else if (ca->cwnd_growth_factor >= 0 &&
427 nv_cwnd_growth_rate_pos > 0 &&
428 ca->nv_no_cong_cnt >
429 nv_cwnd_growth_rate_pos) {
430 ca->cwnd_growth_factor++;
431 ca->nv_no_cong_cnt = 0;
432 }
433 } else {
434
435 return;
436 }
437
438
439 ca->nv_eval_call_cnt = 0;
440 ca->nv_rtt_cnt = 0;
441 ca->nv_rtt_max_rate = 0;
442
443
444
445
446
447 if (tp->snd_cwnd < nv_min_cwnd)
448 tp->snd_cwnd = nv_min_cwnd;
449 }
450}
451
452
453static size_t tcpnv_get_info(struct sock *sk, u32 ext, int *attr,
454 union tcp_cc_info *info)
455{
456 const struct tcpnv *ca = inet_csk_ca(sk);
457
458 if (ext & (1 << (INET_DIAG_VEGASINFO - 1))) {
459 info->vegas.tcpv_enabled = 1;
460 info->vegas.tcpv_rttcnt = ca->nv_rtt_cnt;
461 info->vegas.tcpv_rtt = ca->nv_last_rtt;
462 info->vegas.tcpv_minrtt = ca->nv_min_rtt;
463
464 *attr = INET_DIAG_VEGASINFO;
465 return sizeof(struct tcpvegas_info);
466 }
467 return 0;
468}
469
470static struct tcp_congestion_ops tcpnv __read_mostly = {
471 .init = tcpnv_init,
472 .ssthresh = tcpnv_recalc_ssthresh,
473 .cong_avoid = tcpnv_cong_avoid,
474 .set_state = tcpnv_state,
475 .undo_cwnd = tcp_reno_undo_cwnd,
476 .pkts_acked = tcpnv_acked,
477 .get_info = tcpnv_get_info,
478
479 .owner = THIS_MODULE,
480 .name = "nv",
481};
482
483static int __init tcpnv_register(void)
484{
485 BUILD_BUG_ON(sizeof(struct tcpnv) > ICSK_CA_PRIV_SIZE);
486
487 return tcp_register_congestion_control(&tcpnv);
488}
489
490static void __exit tcpnv_unregister(void)
491{
492 tcp_unregister_congestion_control(&tcpnv);
493}
494
495module_init(tcpnv_register);
496module_exit(tcpnv_unregister);
497
498MODULE_AUTHOR("Lawrence Brakmo");
499MODULE_LICENSE("GPL");
500MODULE_DESCRIPTION("TCP NV");
501MODULE_VERSION("1.0");
502