1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/kernel.h>
34#include <linux/slab.h>
35#include <linux/in.h>
36#include <linux/module.h>
37#include <net/tcp.h>
38#include <net/net_namespace.h>
39#include <net/netns/generic.h>
40
41#include "rds.h"
42#include "tcp.h"
43
44
45static DEFINE_SPINLOCK(rds_tcp_tc_list_lock);
46static LIST_HEAD(rds_tcp_tc_list);
47static unsigned int rds_tcp_tc_count;
48
49
50static DEFINE_SPINLOCK(rds_tcp_conn_lock);
51static LIST_HEAD(rds_tcp_conn_list);
52
53static struct kmem_cache *rds_tcp_conn_slab;
54
55static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
56 void __user *buffer, size_t *lenp,
57 loff_t *fpos);
58
59static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF;
60static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF;
61
62static struct ctl_table rds_tcp_sysctl_table[] = {
63#define RDS_TCP_SNDBUF 0
64 {
65 .procname = "rds_tcp_sndbuf",
66
67 .maxlen = sizeof(int),
68 .mode = 0644,
69 .proc_handler = rds_tcp_skbuf_handler,
70 .extra1 = &rds_tcp_min_sndbuf,
71 },
72#define RDS_TCP_RCVBUF 1
73 {
74 .procname = "rds_tcp_rcvbuf",
75
76 .maxlen = sizeof(int),
77 .mode = 0644,
78 .proc_handler = rds_tcp_skbuf_handler,
79 .extra1 = &rds_tcp_min_rcvbuf,
80 },
81 { }
82};
83
84
85void rds_tcp_nonagle(struct socket *sock)
86{
87 int val = 1;
88
89 kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&val,
90 sizeof(val));
91}
92
93u32 rds_tcp_snd_nxt(struct rds_tcp_connection *tc)
94{
95 return tcp_sk(tc->t_sock->sk)->snd_nxt;
96}
97
98u32 rds_tcp_snd_una(struct rds_tcp_connection *tc)
99{
100 return tcp_sk(tc->t_sock->sk)->snd_una;
101}
102
103void rds_tcp_restore_callbacks(struct socket *sock,
104 struct rds_tcp_connection *tc)
105{
106 rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc);
107 write_lock_bh(&sock->sk->sk_callback_lock);
108
109
110 spin_lock(&rds_tcp_tc_list_lock);
111 list_del_init(&tc->t_list_item);
112 rds_tcp_tc_count--;
113 spin_unlock(&rds_tcp_tc_list_lock);
114
115 tc->t_sock = NULL;
116
117 sock->sk->sk_write_space = tc->t_orig_write_space;
118 sock->sk->sk_data_ready = tc->t_orig_data_ready;
119 sock->sk->sk_state_change = tc->t_orig_state_change;
120 sock->sk->sk_user_data = NULL;
121
122 write_unlock_bh(&sock->sk->sk_callback_lock);
123}
124
125
126
127
128
129
130
131
132
133
134void rds_tcp_reset_callbacks(struct socket *sock,
135 struct rds_conn_path *cp)
136{
137 struct rds_tcp_connection *tc = cp->cp_transport_data;
138 struct socket *osock = tc->t_sock;
139
140 if (!osock)
141 goto newsock;
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163 atomic_set(&cp->cp_state, RDS_CONN_RESETTING);
164 wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags));
165 lock_sock(osock->sk);
166
167 cancel_delayed_work_sync(&cp->cp_send_w);
168 cancel_delayed_work_sync(&cp->cp_recv_w);
169 if (tc->t_tinc) {
170 rds_inc_put(&tc->t_tinc->ti_inc);
171 tc->t_tinc = NULL;
172 }
173 tc->t_tinc_hdr_rem = sizeof(struct rds_header);
174 tc->t_tinc_data_rem = 0;
175 rds_tcp_restore_callbacks(osock, tc);
176 release_sock(osock->sk);
177 sock_release(osock);
178newsock:
179 rds_send_path_reset(cp);
180 lock_sock(sock->sk);
181 rds_tcp_set_callbacks(sock, cp);
182 release_sock(sock->sk);
183}
184
185
186
187
188
189void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
190{
191 struct rds_tcp_connection *tc = cp->cp_transport_data;
192
193 rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
194 write_lock_bh(&sock->sk->sk_callback_lock);
195
196
197 spin_lock(&rds_tcp_tc_list_lock);
198 list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
199 rds_tcp_tc_count++;
200 spin_unlock(&rds_tcp_tc_list_lock);
201
202
203 if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
204 sock->sk->sk_data_ready = sock->sk->sk_user_data;
205
206 tc->t_sock = sock;
207 tc->t_cpath = cp;
208 tc->t_orig_data_ready = sock->sk->sk_data_ready;
209 tc->t_orig_write_space = sock->sk->sk_write_space;
210 tc->t_orig_state_change = sock->sk->sk_state_change;
211
212 sock->sk->sk_user_data = cp;
213 sock->sk->sk_data_ready = rds_tcp_data_ready;
214 sock->sk->sk_write_space = rds_tcp_write_space;
215 sock->sk->sk_state_change = rds_tcp_state_change;
216
217 write_unlock_bh(&sock->sk->sk_callback_lock);
218}
219
220static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len,
221 struct rds_info_iterator *iter,
222 struct rds_info_lengths *lens)
223{
224 struct rds_info_tcp_socket tsinfo;
225 struct rds_tcp_connection *tc;
226 unsigned long flags;
227 struct sockaddr_in sin;
228 int sinlen;
229 struct socket *sock;
230
231 spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);
232
233 if (len / sizeof(tsinfo) < rds_tcp_tc_count)
234 goto out;
235
236 list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
237
238 sock = tc->t_sock;
239 if (sock) {
240 sock->ops->getname(sock, (struct sockaddr *)&sin,
241 &sinlen, 0);
242 tsinfo.local_addr = sin.sin_addr.s_addr;
243 tsinfo.local_port = sin.sin_port;
244 sock->ops->getname(sock, (struct sockaddr *)&sin,
245 &sinlen, 1);
246 tsinfo.peer_addr = sin.sin_addr.s_addr;
247 tsinfo.peer_port = sin.sin_port;
248 }
249
250 tsinfo.hdr_rem = tc->t_tinc_hdr_rem;
251 tsinfo.data_rem = tc->t_tinc_data_rem;
252 tsinfo.last_sent_nxt = tc->t_last_sent_nxt;
253 tsinfo.last_expected_una = tc->t_last_expected_una;
254 tsinfo.last_seen_una = tc->t_last_seen_una;
255
256 rds_info_copy(iter, &tsinfo, sizeof(tsinfo));
257 }
258
259out:
260 lens->nr = rds_tcp_tc_count;
261 lens->each = sizeof(tsinfo);
262
263 spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
264}
265
266static int rds_tcp_laddr_check(struct net *net, __be32 addr)
267{
268 if (inet_addr_type(net, addr) == RTN_LOCAL)
269 return 0;
270 return -EADDRNOTAVAIL;
271}
272
273static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp)
274{
275 struct rds_tcp_connection *tc;
276 int i;
277
278 for (i = 0; i < RDS_MPATH_WORKERS; i++) {
279 tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp);
280 if (!tc)
281 return -ENOMEM;
282
283 mutex_init(&tc->t_conn_path_lock);
284 tc->t_sock = NULL;
285 tc->t_tinc = NULL;
286 tc->t_tinc_hdr_rem = sizeof(struct rds_header);
287 tc->t_tinc_data_rem = 0;
288
289 conn->c_path[i].cp_transport_data = tc;
290 tc->t_cpath = &conn->c_path[i];
291
292 spin_lock_irq(&rds_tcp_conn_lock);
293 list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list);
294 spin_unlock_irq(&rds_tcp_conn_lock);
295 rdsdebug("rds_conn_path [%d] tc %p\n", i,
296 conn->c_path[i].cp_transport_data);
297 }
298
299 return 0;
300}
301
302static void rds_tcp_conn_free(void *arg)
303{
304 struct rds_tcp_connection *tc = arg;
305 unsigned long flags;
306 rdsdebug("freeing tc %p\n", tc);
307
308 spin_lock_irqsave(&rds_tcp_conn_lock, flags);
309 list_del(&tc->t_tcp_node);
310 spin_unlock_irqrestore(&rds_tcp_conn_lock, flags);
311
312 kmem_cache_free(rds_tcp_conn_slab, tc);
313}
314
315static bool list_has_conn(struct list_head *list, struct rds_connection *conn)
316{
317 struct rds_tcp_connection *tc, *_tc;
318
319 list_for_each_entry_safe(tc, _tc, list, t_tcp_node) {
320 if (tc->t_cpath->cp_conn == conn)
321 return true;
322 }
323 return false;
324}
325
326static void rds_tcp_destroy_conns(void)
327{
328 struct rds_tcp_connection *tc, *_tc;
329 LIST_HEAD(tmp_list);
330
331
332 spin_lock_irq(&rds_tcp_conn_lock);
333 list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
334 if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn))
335 list_move_tail(&tc->t_tcp_node, &tmp_list);
336 }
337 spin_unlock_irq(&rds_tcp_conn_lock);
338
339 list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
340 rds_conn_destroy(tc->t_cpath->cp_conn);
341}
342
343static void rds_tcp_exit(void);
344
345struct rds_transport rds_tcp_transport = {
346 .laddr_check = rds_tcp_laddr_check,
347 .xmit_path_prepare = rds_tcp_xmit_path_prepare,
348 .xmit_path_complete = rds_tcp_xmit_path_complete,
349 .xmit = rds_tcp_xmit,
350 .recv_path = rds_tcp_recv_path,
351 .conn_alloc = rds_tcp_conn_alloc,
352 .conn_free = rds_tcp_conn_free,
353 .conn_path_connect = rds_tcp_conn_path_connect,
354 .conn_path_shutdown = rds_tcp_conn_path_shutdown,
355 .inc_copy_to_user = rds_tcp_inc_copy_to_user,
356 .inc_free = rds_tcp_inc_free,
357 .stats_info_copy = rds_tcp_stats_info_copy,
358 .exit = rds_tcp_exit,
359 .t_owner = THIS_MODULE,
360 .t_name = "tcp",
361 .t_type = RDS_TRANS_TCP,
362 .t_prefer_loopback = 1,
363 .t_mp_capable = 1,
364};
365
366static unsigned int rds_tcp_netid;
367
368
369struct rds_tcp_net {
370 struct socket *rds_tcp_listen_sock;
371 struct work_struct rds_tcp_accept_w;
372 struct ctl_table_header *rds_tcp_sysctl;
373 struct ctl_table *ctl_table;
374 int sndbuf_size;
375 int rcvbuf_size;
376};
377
378
379
380
381void rds_tcp_tune(struct socket *sock)
382{
383 struct sock *sk = sock->sk;
384 struct net *net = sock_net(sk);
385 struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
386
387 rds_tcp_nonagle(sock);
388 lock_sock(sk);
389 if (rtn->sndbuf_size > 0) {
390 sk->sk_sndbuf = rtn->sndbuf_size;
391 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
392 }
393 if (rtn->rcvbuf_size > 0) {
394 sk->sk_sndbuf = rtn->rcvbuf_size;
395 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
396 }
397 release_sock(sk);
398}
399
400static void rds_tcp_accept_worker(struct work_struct *work)
401{
402 struct rds_tcp_net *rtn = container_of(work,
403 struct rds_tcp_net,
404 rds_tcp_accept_w);
405
406 while (rds_tcp_accept_one(rtn->rds_tcp_listen_sock) == 0)
407 cond_resched();
408}
409
410void rds_tcp_accept_work(struct sock *sk)
411{
412 struct net *net = sock_net(sk);
413 struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
414
415 queue_work(rds_wq, &rtn->rds_tcp_accept_w);
416}
417
418static __net_init int rds_tcp_init_net(struct net *net)
419{
420 struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
421 struct ctl_table *tbl;
422 int err = 0;
423
424 memset(rtn, 0, sizeof(*rtn));
425
426
427
428
429 if (net == &init_net) {
430 tbl = rds_tcp_sysctl_table;
431 } else {
432 tbl = kmemdup(rds_tcp_sysctl_table,
433 sizeof(rds_tcp_sysctl_table), GFP_KERNEL);
434 if (!tbl) {
435 pr_warn("could not set allocate syctl table\n");
436 return -ENOMEM;
437 }
438 rtn->ctl_table = tbl;
439 }
440 tbl[RDS_TCP_SNDBUF].data = &rtn->sndbuf_size;
441 tbl[RDS_TCP_RCVBUF].data = &rtn->rcvbuf_size;
442 rtn->rds_tcp_sysctl = register_net_sysctl(net, "net/rds/tcp", tbl);
443 if (!rtn->rds_tcp_sysctl) {
444 pr_warn("could not register sysctl\n");
445 err = -ENOMEM;
446 goto fail;
447 }
448 rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net);
449 if (!rtn->rds_tcp_listen_sock) {
450 pr_warn("could not set up listen sock\n");
451 unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
452 rtn->rds_tcp_sysctl = NULL;
453 err = -EAFNOSUPPORT;
454 goto fail;
455 }
456 INIT_WORK(&rtn->rds_tcp_accept_w, rds_tcp_accept_worker);
457 return 0;
458
459fail:
460 if (net != &init_net)
461 kfree(tbl);
462 return err;
463}
464
465static void __net_exit rds_tcp_exit_net(struct net *net)
466{
467 struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
468
469 if (rtn->rds_tcp_sysctl)
470 unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
471
472 if (net != &init_net && rtn->ctl_table)
473 kfree(rtn->ctl_table);
474
475
476
477
478
479
480
481
482
483 if (rtn->rds_tcp_listen_sock) {
484 struct socket *lsock = rtn->rds_tcp_listen_sock;
485
486 rtn->rds_tcp_listen_sock = NULL;
487 rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w);
488 }
489}
490
491static struct pernet_operations rds_tcp_net_ops = {
492 .init = rds_tcp_init_net,
493 .exit = rds_tcp_exit_net,
494 .id = &rds_tcp_netid,
495 .size = sizeof(struct rds_tcp_net),
496};
497
498
499
500
501static void rds_tcp_conn_paths_destroy(struct rds_connection *conn)
502{
503 struct rds_conn_path *cp;
504 struct rds_tcp_connection *tc;
505 int i;
506 struct sock *sk;
507
508 for (i = 0; i < RDS_MPATH_WORKERS; i++) {
509 cp = &conn->c_path[i];
510 tc = cp->cp_transport_data;
511 if (!tc->t_sock)
512 continue;
513 sk = tc->t_sock->sk;
514 sk->sk_prot->disconnect(sk, 0);
515 tcp_done(sk);
516 }
517}
518
519static void rds_tcp_kill_sock(struct net *net)
520{
521 struct rds_tcp_connection *tc, *_tc;
522 LIST_HEAD(tmp_list);
523 struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
524 struct socket *lsock = rtn->rds_tcp_listen_sock;
525
526 rtn->rds_tcp_listen_sock = NULL;
527 rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w);
528 spin_lock_irq(&rds_tcp_conn_lock);
529 list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
530 struct net *c_net = tc->t_cpath->cp_conn->c_net;
531
532 if (net != c_net || !tc->t_sock)
533 continue;
534 if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn))
535 list_move_tail(&tc->t_tcp_node, &tmp_list);
536 }
537 spin_unlock_irq(&rds_tcp_conn_lock);
538 list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node) {
539 rds_tcp_conn_paths_destroy(tc->t_cpath->cp_conn);
540 rds_conn_destroy(tc->t_cpath->cp_conn);
541 }
542}
543
544void *rds_tcp_listen_sock_def_readable(struct net *net)
545{
546 struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
547 struct socket *lsock = rtn->rds_tcp_listen_sock;
548
549 if (!lsock)
550 return NULL;
551
552 return lsock->sk->sk_user_data;
553}
554
555static int rds_tcp_dev_event(struct notifier_block *this,
556 unsigned long event, void *ptr)
557{
558 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
559
560
561
562
563
564
565
566 if (event == NETDEV_UNREGISTER_FINAL &&
567 dev->ifindex == LOOPBACK_IFINDEX)
568 rds_tcp_kill_sock(dev_net(dev));
569
570 return NOTIFY_DONE;
571}
572
573static struct notifier_block rds_tcp_dev_notifier = {
574 .notifier_call = rds_tcp_dev_event,
575 .priority = -10,
576};
577
578
579
580
581
582
583static void rds_tcp_sysctl_reset(struct net *net)
584{
585 struct rds_tcp_connection *tc, *_tc;
586
587 spin_lock_irq(&rds_tcp_conn_lock);
588 list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
589 struct net *c_net = tc->t_cpath->cp_conn->c_net;
590
591 if (net != c_net || !tc->t_sock)
592 continue;
593
594
595 rds_conn_path_drop(tc->t_cpath, false);
596 }
597 spin_unlock_irq(&rds_tcp_conn_lock);
598}
599
600static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
601 void __user *buffer, size_t *lenp,
602 loff_t *fpos)
603{
604 struct net *net = current->nsproxy->net_ns;
605 int err;
606
607 err = proc_dointvec_minmax(ctl, write, buffer, lenp, fpos);
608 if (err < 0) {
609 pr_warn("Invalid input. Must be >= %d\n",
610 *(int *)(ctl->extra1));
611 return err;
612 }
613 if (write)
614 rds_tcp_sysctl_reset(net);
615 return 0;
616}
617
618static void rds_tcp_exit(void)
619{
620 rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
621 unregister_pernet_subsys(&rds_tcp_net_ops);
622 if (unregister_netdevice_notifier(&rds_tcp_dev_notifier))
623 pr_warn("could not unregister rds_tcp_dev_notifier\n");
624 rds_tcp_destroy_conns();
625 rds_trans_unregister(&rds_tcp_transport);
626 rds_tcp_recv_exit();
627 kmem_cache_destroy(rds_tcp_conn_slab);
628}
629module_exit(rds_tcp_exit);
630
631static int rds_tcp_init(void)
632{
633 int ret;
634
635 rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection",
636 sizeof(struct rds_tcp_connection),
637 0, 0, NULL);
638 if (!rds_tcp_conn_slab) {
639 ret = -ENOMEM;
640 goto out;
641 }
642
643 ret = rds_tcp_recv_init();
644 if (ret)
645 goto out_slab;
646
647 ret = register_pernet_subsys(&rds_tcp_net_ops);
648 if (ret)
649 goto out_recv;
650
651 ret = register_netdevice_notifier(&rds_tcp_dev_notifier);
652 if (ret) {
653 pr_warn("could not register rds_tcp_dev_notifier\n");
654 goto out_pernet;
655 }
656
657 rds_trans_register(&rds_tcp_transport);
658
659 rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
660
661 goto out;
662
663out_pernet:
664 unregister_pernet_subsys(&rds_tcp_net_ops);
665out_recv:
666 rds_tcp_recv_exit();
667out_slab:
668 kmem_cache_destroy(rds_tcp_conn_slab);
669out:
670 return ret;
671}
672module_init(rds_tcp_init);
673
674MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
675MODULE_DESCRIPTION("RDS: TCP transport");
676MODULE_LICENSE("Dual BSD/GPL");
677
678