1
2#ifndef _RDS_RDS_H
3#define _RDS_RDS_H
4
5#include <net/sock.h>
6#include <linux/scatterlist.h>
7#include <linux/highmem.h>
8#include <rdma/rdma_cm.h>
9#include <linux/mutex.h>
10#include <linux/rds.h>
11#include <linux/rhashtable.h>
12#include <linux/refcount.h>
13
14#include "info.h"
15
16
17
18
19#define RDS_PROTOCOL_3_0 0x0300
20#define RDS_PROTOCOL_3_1 0x0301
21#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
22#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
23#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
24#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
25
26
27
28
29
30
31
32#define RDS_PORT 18634
33
34#ifdef ATOMIC64_INIT
35#define KERNEL_HAS_ATOMIC64
36#endif
37
38#ifdef RDS_DEBUG
39#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
40#else
41
42static inline __printf(1, 2)
43void rdsdebug(char *fmt, ...)
44{
45}
46#endif
47
48
49#define ceil(x, y) \
50 ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
51
52#define RDS_FRAG_SHIFT 12
53#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
54
55
56#define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
57
58#define RDS_CONG_MAP_BYTES (65536 / 8)
59#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
60#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
61
62struct rds_cong_map {
63 struct rb_node m_rb_node;
64 __be32 m_addr;
65 wait_queue_head_t m_waitq;
66 struct list_head m_conn_list;
67 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
68};
69
70
71
72
73
74
75
76
77enum {
78 RDS_CONN_DOWN = 0,
79 RDS_CONN_CONNECTING,
80 RDS_CONN_DISCONNECTING,
81 RDS_CONN_UP,
82 RDS_CONN_RESETTING,
83 RDS_CONN_ERROR,
84};
85
86
87#define RDS_LL_SEND_FULL 0
88#define RDS_RECONNECT_PENDING 1
89#define RDS_IN_XMIT 2
90#define RDS_RECV_REFILL 3
91
92
93#define RDS_MPATH_WORKERS 8
94#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
95 (rs)->rs_hash_initval) & ((n) - 1))
96
97#define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
98
99
100struct rds_conn_path {
101 struct rds_connection *cp_conn;
102 struct rds_message *cp_xmit_rm;
103 unsigned long cp_xmit_sg;
104 unsigned int cp_xmit_hdr_off;
105 unsigned int cp_xmit_data_off;
106 unsigned int cp_xmit_atomic_sent;
107 unsigned int cp_xmit_rdma_sent;
108 unsigned int cp_xmit_data_sent;
109
110 spinlock_t cp_lock;
111 u64 cp_next_tx_seq;
112 struct list_head cp_send_queue;
113 struct list_head cp_retrans;
114
115 u64 cp_next_rx_seq;
116
117 void *cp_transport_data;
118
119 atomic_t cp_state;
120 unsigned long cp_send_gen;
121 unsigned long cp_flags;
122 unsigned long cp_reconnect_jiffies;
123 struct delayed_work cp_send_w;
124 struct delayed_work cp_recv_w;
125 struct delayed_work cp_conn_w;
126 struct work_struct cp_down_w;
127 struct mutex cp_cm_lock;
128 wait_queue_head_t cp_waitq;
129
130 unsigned int cp_unacked_packets;
131 unsigned int cp_unacked_bytes;
132 unsigned int cp_index;
133};
134
135
136struct rds_connection {
137 struct hlist_node c_hash_node;
138 __be32 c_laddr;
139 __be32 c_faddr;
140 unsigned int c_loopback:1,
141 c_ping_triggered:1,
142 c_destroy_in_prog:1,
143 c_pad_to_32:29;
144 int c_npaths;
145 struct rds_connection *c_passive;
146 struct rds_transport *c_trans;
147
148 struct rds_cong_map *c_lcong;
149 struct rds_cong_map *c_fcong;
150
151
152 unsigned int c_version;
153 struct net *c_net;
154
155 struct list_head c_map_item;
156 unsigned long c_map_queued;
157
158 struct rds_conn_path *c_path;
159 wait_queue_head_t c_hs_waitq;
160
161 u32 c_my_gen_num;
162 u32 c_peer_gen_num;
163};
164
165static inline
166struct net *rds_conn_net(struct rds_connection *conn)
167{
168 return conn->c_net;
169}
170
171static inline
172void rds_conn_net_set(struct rds_connection *conn, struct net *net)
173{
174 conn->c_net = get_net(net);
175}
176
177#define RDS_FLAG_CONG_BITMAP 0x01
178#define RDS_FLAG_ACK_REQUIRED 0x02
179#define RDS_FLAG_RETRANSMITTED 0x04
180#define RDS_MAX_ADV_CREDIT 255
181
182
183
184
185
186
187
188
189#define RDS_FLAG_PROBE_PORT 1
190#define RDS_HS_PROBE(sport, dport) \
191 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
192 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
193
194
195
196#define RDS_HEADER_EXT_SPACE 16
197
198struct rds_header {
199 __be64 h_sequence;
200 __be64 h_ack;
201 __be32 h_len;
202 __be16 h_sport;
203 __be16 h_dport;
204 u8 h_flags;
205 u8 h_credit;
206 u8 h_padding[4];
207 __sum16 h_csum;
208
209 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
210};
211
212
213
214
215#define RDS_EXTHDR_NONE 0
216
217
218
219
220
221
222
223
224
225
226
227#define RDS_EXTHDR_VERSION 1
228struct rds_ext_header_version {
229 __be32 h_version;
230};
231
232
233
234
235
236#define RDS_EXTHDR_RDMA 2
237struct rds_ext_header_rdma {
238 __be32 h_rdma_rkey;
239};
240
241
242
243
244
245
246#define RDS_EXTHDR_RDMA_DEST 3
247struct rds_ext_header_rdma_dest {
248 __be32 h_rdma_rkey;
249 __be32 h_rdma_offset;
250};
251
252
253
254
255#define RDS_EXTHDR_NPATHS 5
256#define RDS_EXTHDR_GEN_NUM 6
257
258#define __RDS_EXTHDR_MAX 16
259#define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
260#define RDS_MSG_RX_HDR 0
261#define RDS_MSG_RX_START 1
262#define RDS_MSG_RX_END 2
263#define RDS_MSG_RX_CMSG 3
264
265struct rds_incoming {
266 refcount_t i_refcount;
267 struct list_head i_item;
268 struct rds_connection *i_conn;
269 struct rds_conn_path *i_conn_path;
270 struct rds_header i_hdr;
271 unsigned long i_rx_jiffies;
272 __be32 i_saddr;
273
274 rds_rdma_cookie_t i_rdma_cookie;
275 struct timeval i_rx_tstamp;
276 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
277};
278
279struct rds_mr {
280 struct rb_node r_rb_node;
281 refcount_t r_refcount;
282 u32 r_key;
283
284
285 unsigned int r_use_once:1;
286 unsigned int r_invalidate:1;
287 unsigned int r_write:1;
288
289
290
291
292
293 unsigned long r_state;
294 struct rds_sock *r_sock;
295 struct rds_transport *r_trans;
296 void *r_trans_private;
297};
298
299
300#define RDS_MR_DEAD 0
301
302static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
303{
304 return r_key | (((u64) offset) << 32);
305}
306
307static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
308{
309 return cookie;
310}
311
312static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
313{
314 return cookie >> 32;
315}
316
317
318#define RDS_ATOMIC_TYPE_CSWP 0
319#define RDS_ATOMIC_TYPE_FADD 1
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350#define RDS_MSG_ON_SOCK 1
351#define RDS_MSG_ON_CONN 2
352#define RDS_MSG_HAS_ACK_SEQ 3
353#define RDS_MSG_ACK_REQUIRED 4
354#define RDS_MSG_RETRANSMITTED 5
355#define RDS_MSG_MAPPED 6
356#define RDS_MSG_PAGEVEC 7
357#define RDS_MSG_FLUSH 8
358
359struct rds_message {
360 refcount_t m_refcount;
361 struct list_head m_sock_item;
362 struct list_head m_conn_item;
363 struct rds_incoming m_inc;
364 u64 m_ack_seq;
365 __be32 m_daddr;
366 unsigned long m_flags;
367
368
369
370
371
372
373 spinlock_t m_rs_lock;
374 wait_queue_head_t m_flush_wait;
375
376 struct rds_sock *m_rs;
377
378
379 rds_rdma_cookie_t m_rdma_cookie;
380
381 unsigned int m_used_sgs;
382 unsigned int m_total_sgs;
383
384 void *m_final_op;
385
386 struct {
387 struct rm_atomic_op {
388 int op_type;
389 union {
390 struct {
391 uint64_t compare;
392 uint64_t swap;
393 uint64_t compare_mask;
394 uint64_t swap_mask;
395 } op_m_cswp;
396 struct {
397 uint64_t add;
398 uint64_t nocarry_mask;
399 } op_m_fadd;
400 };
401
402 u32 op_rkey;
403 u64 op_remote_addr;
404 unsigned int op_notify:1;
405 unsigned int op_recverr:1;
406 unsigned int op_mapped:1;
407 unsigned int op_silent:1;
408 unsigned int op_active:1;
409 struct scatterlist *op_sg;
410 struct rds_notifier *op_notifier;
411
412 struct rds_mr *op_rdma_mr;
413 } atomic;
414 struct rm_rdma_op {
415 u32 op_rkey;
416 u64 op_remote_addr;
417 unsigned int op_write:1;
418 unsigned int op_fence:1;
419 unsigned int op_notify:1;
420 unsigned int op_recverr:1;
421 unsigned int op_mapped:1;
422 unsigned int op_silent:1;
423 unsigned int op_active:1;
424 unsigned int op_bytes;
425 unsigned int op_nents;
426 unsigned int op_count;
427 struct scatterlist *op_sg;
428 struct rds_notifier *op_notifier;
429
430 struct rds_mr *op_rdma_mr;
431 } rdma;
432 struct rm_data_op {
433 unsigned int op_active:1;
434 unsigned int op_notify:1;
435 unsigned int op_nents;
436 unsigned int op_count;
437 unsigned int op_dmasg;
438 unsigned int op_dmaoff;
439 struct scatterlist *op_sg;
440 } data;
441 };
442};
443
444
445
446
447
448
449
450
451struct rds_notifier {
452 struct list_head n_list;
453 uint64_t n_user_token;
454 int n_status;
455};
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484struct rds_transport {
485 char t_name[TRANSNAMSIZ];
486 struct list_head t_item;
487 struct module *t_owner;
488 unsigned int t_prefer_loopback:1,
489 t_mp_capable:1;
490 unsigned int t_type;
491
492 int (*laddr_check)(struct net *net, __be32 addr);
493 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
494 void (*conn_free)(void *data);
495 int (*conn_path_connect)(struct rds_conn_path *cp);
496 void (*conn_path_shutdown)(struct rds_conn_path *conn);
497 void (*xmit_path_prepare)(struct rds_conn_path *cp);
498 void (*xmit_path_complete)(struct rds_conn_path *cp);
499 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
500 unsigned int hdr_off, unsigned int sg, unsigned int off);
501 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
502 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
503 int (*recv_path)(struct rds_conn_path *cp);
504 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
505 void (*inc_free)(struct rds_incoming *inc);
506
507 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
508 struct rdma_cm_event *event);
509 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id);
510 void (*cm_connect_complete)(struct rds_connection *conn,
511 struct rdma_cm_event *event);
512
513 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
514 unsigned int avail);
515 void (*exit)(void);
516 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
517 struct rds_sock *rs, u32 *key_ret);
518 void (*sync_mr)(void *trans_private, int direction);
519 void (*free_mr)(void *trans_private, int invalidate);
520 void (*flush_mrs)(void);
521};
522
523struct rds_sock {
524 struct sock rs_sk;
525
526 u64 rs_user_addr;
527 u64 rs_user_bytes;
528
529
530
531
532
533 struct rhash_head rs_bound_node;
534 u64 rs_bound_key;
535 __be32 rs_bound_addr;
536 __be32 rs_conn_addr;
537 __be16 rs_bound_port;
538 __be16 rs_conn_port;
539 struct rds_transport *rs_transport;
540
541
542
543
544
545 struct rds_connection *rs_conn;
546
547
548 int rs_congested;
549
550 int rs_seen_congestion;
551
552
553 spinlock_t rs_lock;
554 struct list_head rs_send_queue;
555 u32 rs_snd_bytes;
556 int rs_rcv_bytes;
557 struct list_head rs_notify_queue;
558
559
560
561
562
563
564 uint64_t rs_cong_mask;
565 uint64_t rs_cong_notify;
566 struct list_head rs_cong_list;
567 unsigned long rs_cong_track;
568
569
570
571
572
573 rwlock_t rs_recv_lock;
574 struct list_head rs_recv_queue;
575
576
577 struct list_head rs_item;
578
579
580 spinlock_t rs_rdma_lock;
581 struct rb_root rs_rdma_keys;
582
583
584 unsigned char rs_recverr,
585 rs_cong_monitor;
586 u32 rs_hash_initval;
587
588
589 u8 rs_rx_traces;
590 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
591};
592
593static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
594{
595 return container_of(sk, struct rds_sock, rs_sk);
596}
597static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
598{
599 return &rs->rs_sk;
600}
601
602
603
604
605
606
607static inline int rds_sk_sndbuf(struct rds_sock *rs)
608{
609 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
610}
611static inline int rds_sk_rcvbuf(struct rds_sock *rs)
612{
613 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
614}
615
616struct rds_statistics {
617 uint64_t s_conn_reset;
618 uint64_t s_recv_drop_bad_checksum;
619 uint64_t s_recv_drop_old_seq;
620 uint64_t s_recv_drop_no_sock;
621 uint64_t s_recv_drop_dead_sock;
622 uint64_t s_recv_deliver_raced;
623 uint64_t s_recv_delivered;
624 uint64_t s_recv_queued;
625 uint64_t s_recv_immediate_retry;
626 uint64_t s_recv_delayed_retry;
627 uint64_t s_recv_ack_required;
628 uint64_t s_recv_rdma_bytes;
629 uint64_t s_recv_ping;
630 uint64_t s_send_queue_empty;
631 uint64_t s_send_queue_full;
632 uint64_t s_send_lock_contention;
633 uint64_t s_send_lock_queue_raced;
634 uint64_t s_send_immediate_retry;
635 uint64_t s_send_delayed_retry;
636 uint64_t s_send_drop_acked;
637 uint64_t s_send_ack_required;
638 uint64_t s_send_queued;
639 uint64_t s_send_rdma;
640 uint64_t s_send_rdma_bytes;
641 uint64_t s_send_pong;
642 uint64_t s_page_remainder_hit;
643 uint64_t s_page_remainder_miss;
644 uint64_t s_copy_to_user;
645 uint64_t s_copy_from_user;
646 uint64_t s_cong_update_queued;
647 uint64_t s_cong_update_received;
648 uint64_t s_cong_send_error;
649 uint64_t s_cong_send_blocked;
650 uint64_t s_recv_bytes_added_to_socket;
651 uint64_t s_recv_bytes_removed_from_socket;
652
653};
654
655
656void rds_sock_addref(struct rds_sock *rs);
657void rds_sock_put(struct rds_sock *rs);
658void rds_wake_sk_sleep(struct rds_sock *rs);
659static inline void __rds_wake_sk_sleep(struct sock *sk)
660{
661 wait_queue_head_t *waitq = sk_sleep(sk);
662
663 if (!sock_flag(sk, SOCK_DEAD) && waitq)
664 wake_up(waitq);
665}
666extern wait_queue_head_t rds_poll_waitq;
667
668
669
670int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
671void rds_remove_bound(struct rds_sock *rs);
672struct rds_sock *rds_find_bound(__be32 addr, __be16 port);
673int rds_bind_lock_init(void);
674void rds_bind_lock_destroy(void);
675
676
677int rds_cong_get_maps(struct rds_connection *conn);
678void rds_cong_add_conn(struct rds_connection *conn);
679void rds_cong_remove_conn(struct rds_connection *conn);
680void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
681void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
682int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
683void rds_cong_queue_updates(struct rds_cong_map *map);
684void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
685int rds_cong_updated_since(unsigned long *recent);
686void rds_cong_add_socket(struct rds_sock *);
687void rds_cong_remove_socket(struct rds_sock *);
688void rds_cong_exit(void);
689struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
690
691
692extern u32 rds_gen_num;
693int rds_conn_init(void);
694void rds_conn_exit(void);
695struct rds_connection *rds_conn_create(struct net *net,
696 __be32 laddr, __be32 faddr,
697 struct rds_transport *trans, gfp_t gfp);
698struct rds_connection *rds_conn_create_outgoing(struct net *net,
699 __be32 laddr, __be32 faddr,
700 struct rds_transport *trans, gfp_t gfp);
701void rds_conn_shutdown(struct rds_conn_path *cpath);
702void rds_conn_destroy(struct rds_connection *conn);
703void rds_conn_drop(struct rds_connection *conn);
704void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
705void rds_conn_connect_if_down(struct rds_connection *conn);
706void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
707void rds_for_each_conn_info(struct socket *sock, unsigned int len,
708 struct rds_info_iterator *iter,
709 struct rds_info_lengths *lens,
710 int (*visitor)(struct rds_connection *, void *),
711 size_t item_len);
712
713__printf(2, 3)
714void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
715#define rds_conn_path_error(cp, fmt...) \
716 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
717
718static inline int
719rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
720{
721 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
722}
723
724static inline int
725rds_conn_transition(struct rds_connection *conn, int old, int new)
726{
727 WARN_ON(conn->c_trans->t_mp_capable);
728 return rds_conn_path_transition(&conn->c_path[0], old, new);
729}
730
731static inline int
732rds_conn_path_state(struct rds_conn_path *cp)
733{
734 return atomic_read(&cp->cp_state);
735}
736
737static inline int
738rds_conn_state(struct rds_connection *conn)
739{
740 WARN_ON(conn->c_trans->t_mp_capable);
741 return rds_conn_path_state(&conn->c_path[0]);
742}
743
744static inline int
745rds_conn_path_up(struct rds_conn_path *cp)
746{
747 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
748}
749
750static inline int
751rds_conn_up(struct rds_connection *conn)
752{
753 WARN_ON(conn->c_trans->t_mp_capable);
754 return rds_conn_path_up(&conn->c_path[0]);
755}
756
757static inline int
758rds_conn_path_connecting(struct rds_conn_path *cp)
759{
760 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
761}
762
763static inline int
764rds_conn_connecting(struct rds_connection *conn)
765{
766 WARN_ON(conn->c_trans->t_mp_capable);
767 return rds_conn_path_connecting(&conn->c_path[0]);
768}
769
770
771struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
772struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
773int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from);
774struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
775void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
776 __be16 dport, u64 seq);
777int rds_message_add_extension(struct rds_header *hdr,
778 unsigned int type, const void *data, unsigned int len);
779int rds_message_next_extension(struct rds_header *hdr,
780 unsigned int *pos, void *buf, unsigned int *buflen);
781int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
782int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
783void rds_message_inc_free(struct rds_incoming *inc);
784void rds_message_addref(struct rds_message *rm);
785void rds_message_put(struct rds_message *rm);
786void rds_message_wait(struct rds_message *rm);
787void rds_message_unmapped(struct rds_message *rm);
788
789static inline void rds_message_make_checksum(struct rds_header *hdr)
790{
791 hdr->h_csum = 0;
792 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
793}
794
795static inline int rds_message_verify_checksum(const struct rds_header *hdr)
796{
797 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
798}
799
800
801
802int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
803 gfp_t gfp);
804void rds_page_exit(void);
805
806
807void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
808 __be32 saddr);
809void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
810 __be32 saddr);
811void rds_inc_put(struct rds_incoming *inc);
812void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
813 struct rds_incoming *inc, gfp_t gfp);
814int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
815 int msg_flags);
816void rds_clear_recv_queue(struct rds_sock *rs);
817int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
818void rds_inc_info_copy(struct rds_incoming *inc,
819 struct rds_info_iterator *iter,
820 __be32 saddr, __be32 daddr, int flip);
821
822
823int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
824void rds_send_path_reset(struct rds_conn_path *conn);
825int rds_send_xmit(struct rds_conn_path *cp);
826struct sockaddr_in;
827void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest);
828typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
829void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
830 is_acked_func is_acked);
831void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
832 is_acked_func is_acked);
833void rds_send_ping(struct rds_connection *conn, int cp_index);
834int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
835
836
837void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
838int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
839int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
840int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
841void rds_rdma_drop_keys(struct rds_sock *rs);
842int rds_rdma_extra_size(struct rds_rdma_args *args);
843int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
844 struct cmsghdr *cmsg);
845int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
846 struct cmsghdr *cmsg);
847int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
848 struct cmsghdr *cmsg);
849int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
850 struct cmsghdr *cmsg);
851void rds_rdma_free_op(struct rm_rdma_op *ro);
852void rds_atomic_free_op(struct rm_atomic_op *ao);
853void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
854void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
855int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
856 struct cmsghdr *cmsg);
857
858void __rds_put_mr_final(struct rds_mr *mr);
859static inline void rds_mr_put(struct rds_mr *mr)
860{
861 if (refcount_dec_and_test(&mr->r_refcount))
862 __rds_put_mr_final(mr);
863}
864
865
866DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
867#define rds_stats_inc_which(which, member) do { \
868 per_cpu(which, get_cpu()).member++; \
869 put_cpu(); \
870} while (0)
871#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
872#define rds_stats_add_which(which, member, count) do { \
873 per_cpu(which, get_cpu()).member += count; \
874 put_cpu(); \
875} while (0)
876#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
877int rds_stats_init(void);
878void rds_stats_exit(void);
879void rds_stats_info_copy(struct rds_info_iterator *iter,
880 uint64_t *values, const char *const *names,
881 size_t nr);
882
883
884int rds_sysctl_init(void);
885void rds_sysctl_exit(void);
886extern unsigned long rds_sysctl_sndbuf_min;
887extern unsigned long rds_sysctl_sndbuf_default;
888extern unsigned long rds_sysctl_sndbuf_max;
889extern unsigned long rds_sysctl_reconnect_min_jiffies;
890extern unsigned long rds_sysctl_reconnect_max_jiffies;
891extern unsigned int rds_sysctl_max_unacked_packets;
892extern unsigned int rds_sysctl_max_unacked_bytes;
893extern unsigned int rds_sysctl_ping_enable;
894extern unsigned long rds_sysctl_trace_flags;
895extern unsigned int rds_sysctl_trace_level;
896
897
898int rds_threads_init(void);
899void rds_threads_exit(void);
900extern struct workqueue_struct *rds_wq;
901void rds_queue_reconnect(struct rds_conn_path *cp);
902void rds_connect_worker(struct work_struct *);
903void rds_shutdown_worker(struct work_struct *);
904void rds_send_worker(struct work_struct *);
905void rds_recv_worker(struct work_struct *);
906void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
907void rds_connect_complete(struct rds_connection *conn);
908
909
910void rds_trans_register(struct rds_transport *trans);
911void rds_trans_unregister(struct rds_transport *trans);
912struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
913void rds_trans_put(struct rds_transport *trans);
914unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
915 unsigned int avail);
916struct rds_transport *rds_trans_get(int t_type);
917int rds_trans_init(void);
918void rds_trans_exit(void);
919
920#endif
921