1#ifndef _RDS_RDS_H
2#define _RDS_RDS_H
3
4#include <net/sock.h>
5#include <linux/scatterlist.h>
6#include <linux/highmem.h>
7#include <rdma/rdma_cm.h>
8#include <linux/mutex.h>
9#include <linux/rds.h>
10#include <linux/rhashtable.h>
11
12#include "info.h"
13
14
15
16
17#define RDS_PROTOCOL_3_0 0x0300
18#define RDS_PROTOCOL_3_1 0x0301
19#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
20#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
21#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
22#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
23
24
25
26
27
28
29
30#define RDS_PORT 18634
31
32#ifdef ATOMIC64_INIT
33#define KERNEL_HAS_ATOMIC64
34#endif
35
36#ifdef DEBUG
37#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
38#else
39
40static inline __printf(1, 2)
41void rdsdebug(char *fmt, ...)
42{
43}
44#endif
45
46
47#define ceil(x, y) \
48 ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
49
50#define RDS_FRAG_SHIFT 12
51#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
52
53#define RDS_CONG_MAP_BYTES (65536 / 8)
54#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
55#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
56
57struct rds_cong_map {
58 struct rb_node m_rb_node;
59 __be32 m_addr;
60 wait_queue_head_t m_waitq;
61 struct list_head m_conn_list;
62 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
63};
64
65
66
67
68
69
70
71
72enum {
73 RDS_CONN_DOWN = 0,
74 RDS_CONN_CONNECTING,
75 RDS_CONN_DISCONNECTING,
76 RDS_CONN_UP,
77 RDS_CONN_ERROR,
78};
79
80
81#define RDS_LL_SEND_FULL 0
82#define RDS_RECONNECT_PENDING 1
83#define RDS_IN_XMIT 2
84#define RDS_RECV_REFILL 3
85
86struct rds_connection {
87 struct hlist_node c_hash_node;
88 __be32 c_laddr;
89 __be32 c_faddr;
90 unsigned int c_loopback:1,
91 c_outgoing:1,
92 c_pad_to_32:30;
93 struct rds_connection *c_passive;
94
95 struct rds_cong_map *c_lcong;
96 struct rds_cong_map *c_fcong;
97
98 struct rds_message *c_xmit_rm;
99 unsigned long c_xmit_sg;
100 unsigned int c_xmit_hdr_off;
101 unsigned int c_xmit_data_off;
102 unsigned int c_xmit_atomic_sent;
103 unsigned int c_xmit_rdma_sent;
104 unsigned int c_xmit_data_sent;
105
106 spinlock_t c_lock;
107 u64 c_next_tx_seq;
108 struct list_head c_send_queue;
109 struct list_head c_retrans;
110
111 u64 c_next_rx_seq;
112
113 struct rds_transport *c_trans;
114 void *c_transport_data;
115
116 atomic_t c_state;
117 unsigned long c_send_gen;
118 unsigned long c_flags;
119 unsigned long c_reconnect_jiffies;
120 struct delayed_work c_send_w;
121 struct delayed_work c_recv_w;
122 struct delayed_work c_conn_w;
123 struct work_struct c_down_w;
124 struct mutex c_cm_lock;
125 wait_queue_head_t c_waitq;
126
127 struct list_head c_map_item;
128 unsigned long c_map_queued;
129
130 unsigned int c_unacked_packets;
131 unsigned int c_unacked_bytes;
132
133
134 unsigned int c_version;
135 possible_net_t c_net;
136};
137
138static inline
139struct net *rds_conn_net(struct rds_connection *conn)
140{
141 return read_pnet(&conn->c_net);
142}
143
144static inline
145void rds_conn_net_set(struct rds_connection *conn, struct net *net)
146{
147 write_pnet(&conn->c_net, net);
148}
149
150#define RDS_FLAG_CONG_BITMAP 0x01
151#define RDS_FLAG_ACK_REQUIRED 0x02
152#define RDS_FLAG_RETRANSMITTED 0x04
153#define RDS_MAX_ADV_CREDIT 255
154
155
156
157
158#define RDS_HEADER_EXT_SPACE 16
159
160struct rds_header {
161 __be64 h_sequence;
162 __be64 h_ack;
163 __be32 h_len;
164 __be16 h_sport;
165 __be16 h_dport;
166 u8 h_flags;
167 u8 h_credit;
168 u8 h_padding[4];
169 __sum16 h_csum;
170
171 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
172};
173
174
175
176
177#define RDS_EXTHDR_NONE 0
178
179
180
181
182
183
184
185
186
187
188
189#define RDS_EXTHDR_VERSION 1
190struct rds_ext_header_version {
191 __be32 h_version;
192};
193
194
195
196
197
198#define RDS_EXTHDR_RDMA 2
199struct rds_ext_header_rdma {
200 __be32 h_rdma_rkey;
201};
202
203
204
205
206
207
208#define RDS_EXTHDR_RDMA_DEST 3
209struct rds_ext_header_rdma_dest {
210 __be32 h_rdma_rkey;
211 __be32 h_rdma_offset;
212};
213
214#define __RDS_EXTHDR_MAX 16
215
216struct rds_incoming {
217 atomic_t i_refcount;
218 struct list_head i_item;
219 struct rds_connection *i_conn;
220 struct rds_header i_hdr;
221 unsigned long i_rx_jiffies;
222 __be32 i_saddr;
223
224 rds_rdma_cookie_t i_rdma_cookie;
225 struct timeval i_rx_tstamp;
226};
227
228struct rds_mr {
229 struct rb_node r_rb_node;
230 atomic_t r_refcount;
231 u32 r_key;
232
233
234 unsigned int r_use_once:1;
235 unsigned int r_invalidate:1;
236 unsigned int r_write:1;
237
238
239
240
241
242 unsigned long r_state;
243 struct rds_sock *r_sock;
244 struct rds_transport *r_trans;
245 void *r_trans_private;
246};
247
248
249#define RDS_MR_DEAD 0
250
251static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
252{
253 return r_key | (((u64) offset) << 32);
254}
255
256static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
257{
258 return cookie;
259}
260
261static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
262{
263 return cookie >> 32;
264}
265
266
267#define RDS_ATOMIC_TYPE_CSWP 0
268#define RDS_ATOMIC_TYPE_FADD 1
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299#define RDS_MSG_ON_SOCK 1
300#define RDS_MSG_ON_CONN 2
301#define RDS_MSG_HAS_ACK_SEQ 3
302#define RDS_MSG_ACK_REQUIRED 4
303#define RDS_MSG_RETRANSMITTED 5
304#define RDS_MSG_MAPPED 6
305#define RDS_MSG_PAGEVEC 7
306
307struct rds_message {
308 atomic_t m_refcount;
309 struct list_head m_sock_item;
310 struct list_head m_conn_item;
311 struct rds_incoming m_inc;
312 u64 m_ack_seq;
313 __be32 m_daddr;
314 unsigned long m_flags;
315
316
317
318
319
320
321 spinlock_t m_rs_lock;
322 wait_queue_head_t m_flush_wait;
323
324 struct rds_sock *m_rs;
325
326
327 rds_rdma_cookie_t m_rdma_cookie;
328
329 unsigned int m_used_sgs;
330 unsigned int m_total_sgs;
331
332 void *m_final_op;
333
334 struct {
335 struct rm_atomic_op {
336 int op_type;
337 union {
338 struct {
339 uint64_t compare;
340 uint64_t swap;
341 uint64_t compare_mask;
342 uint64_t swap_mask;
343 } op_m_cswp;
344 struct {
345 uint64_t add;
346 uint64_t nocarry_mask;
347 } op_m_fadd;
348 };
349
350 u32 op_rkey;
351 u64 op_remote_addr;
352 unsigned int op_notify:1;
353 unsigned int op_recverr:1;
354 unsigned int op_mapped:1;
355 unsigned int op_silent:1;
356 unsigned int op_active:1;
357 struct scatterlist *op_sg;
358 struct rds_notifier *op_notifier;
359
360 struct rds_mr *op_rdma_mr;
361 } atomic;
362 struct rm_rdma_op {
363 u32 op_rkey;
364 u64 op_remote_addr;
365 unsigned int op_write:1;
366 unsigned int op_fence:1;
367 unsigned int op_notify:1;
368 unsigned int op_recverr:1;
369 unsigned int op_mapped:1;
370 unsigned int op_silent:1;
371 unsigned int op_active:1;
372 unsigned int op_bytes;
373 unsigned int op_nents;
374 unsigned int op_count;
375 struct scatterlist *op_sg;
376 struct rds_notifier *op_notifier;
377
378 struct rds_mr *op_rdma_mr;
379 } rdma;
380 struct rm_data_op {
381 unsigned int op_active:1;
382 unsigned int op_nents;
383 unsigned int op_count;
384 unsigned int op_dmasg;
385 unsigned int op_dmaoff;
386 struct scatterlist *op_sg;
387 } data;
388 };
389};
390
391
392
393
394
395
396
397
398struct rds_notifier {
399 struct list_head n_list;
400 uint64_t n_user_token;
401 int n_status;
402};
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431struct rds_transport {
432 char t_name[TRANSNAMSIZ];
433 struct list_head t_item;
434 struct module *t_owner;
435 unsigned int t_prefer_loopback:1;
436 unsigned int t_type;
437
438 int (*laddr_check)(struct net *net, __be32 addr);
439 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
440 void (*conn_free)(void *data);
441 int (*conn_connect)(struct rds_connection *conn);
442 void (*conn_shutdown)(struct rds_connection *conn);
443 void (*xmit_prepare)(struct rds_connection *conn);
444 void (*xmit_complete)(struct rds_connection *conn);
445 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
446 unsigned int hdr_off, unsigned int sg, unsigned int off);
447 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
448 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
449 int (*recv)(struct rds_connection *conn);
450 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
451 void (*inc_free)(struct rds_incoming *inc);
452
453 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
454 struct rdma_cm_event *event);
455 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id);
456 void (*cm_connect_complete)(struct rds_connection *conn,
457 struct rdma_cm_event *event);
458
459 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
460 unsigned int avail);
461 void (*exit)(void);
462 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
463 struct rds_sock *rs, u32 *key_ret);
464 void (*sync_mr)(void *trans_private, int direction);
465 void (*free_mr)(void *trans_private, int invalidate);
466 void (*flush_mrs)(void);
467};
468
469struct rds_sock {
470 struct sock rs_sk;
471
472 u64 rs_user_addr;
473 u64 rs_user_bytes;
474
475
476
477
478
479 struct rhash_head rs_bound_node;
480 u64 rs_bound_key;
481 __be32 rs_bound_addr;
482 __be32 rs_conn_addr;
483 __be16 rs_bound_port;
484 __be16 rs_conn_port;
485 struct rds_transport *rs_transport;
486
487
488
489
490
491 struct rds_connection *rs_conn;
492
493
494 int rs_congested;
495
496 int rs_seen_congestion;
497
498
499 spinlock_t rs_lock;
500 struct list_head rs_send_queue;
501 u32 rs_snd_bytes;
502 int rs_rcv_bytes;
503 struct list_head rs_notify_queue;
504
505
506
507
508
509
510 uint64_t rs_cong_mask;
511 uint64_t rs_cong_notify;
512 struct list_head rs_cong_list;
513 unsigned long rs_cong_track;
514
515
516
517
518
519 rwlock_t rs_recv_lock;
520 struct list_head rs_recv_queue;
521
522
523 struct list_head rs_item;
524
525
526 spinlock_t rs_rdma_lock;
527 struct rb_root rs_rdma_keys;
528
529
530 unsigned char rs_recverr,
531 rs_cong_monitor;
532};
533
534static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
535{
536 return container_of(sk, struct rds_sock, rs_sk);
537}
538static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
539{
540 return &rs->rs_sk;
541}
542
543
544
545
546
547
548static inline int rds_sk_sndbuf(struct rds_sock *rs)
549{
550 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
551}
552static inline int rds_sk_rcvbuf(struct rds_sock *rs)
553{
554 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
555}
556
557struct rds_statistics {
558 uint64_t s_conn_reset;
559 uint64_t s_recv_drop_bad_checksum;
560 uint64_t s_recv_drop_old_seq;
561 uint64_t s_recv_drop_no_sock;
562 uint64_t s_recv_drop_dead_sock;
563 uint64_t s_recv_deliver_raced;
564 uint64_t s_recv_delivered;
565 uint64_t s_recv_queued;
566 uint64_t s_recv_immediate_retry;
567 uint64_t s_recv_delayed_retry;
568 uint64_t s_recv_ack_required;
569 uint64_t s_recv_rdma_bytes;
570 uint64_t s_recv_ping;
571 uint64_t s_send_queue_empty;
572 uint64_t s_send_queue_full;
573 uint64_t s_send_lock_contention;
574 uint64_t s_send_lock_queue_raced;
575 uint64_t s_send_immediate_retry;
576 uint64_t s_send_delayed_retry;
577 uint64_t s_send_drop_acked;
578 uint64_t s_send_ack_required;
579 uint64_t s_send_queued;
580 uint64_t s_send_rdma;
581 uint64_t s_send_rdma_bytes;
582 uint64_t s_send_pong;
583 uint64_t s_page_remainder_hit;
584 uint64_t s_page_remainder_miss;
585 uint64_t s_copy_to_user;
586 uint64_t s_copy_from_user;
587 uint64_t s_cong_update_queued;
588 uint64_t s_cong_update_received;
589 uint64_t s_cong_send_error;
590 uint64_t s_cong_send_blocked;
591};
592
593
594void rds_sock_addref(struct rds_sock *rs);
595void rds_sock_put(struct rds_sock *rs);
596void rds_wake_sk_sleep(struct rds_sock *rs);
597static inline void __rds_wake_sk_sleep(struct sock *sk)
598{
599 wait_queue_head_t *waitq = sk_sleep(sk);
600
601 if (!sock_flag(sk, SOCK_DEAD) && waitq)
602 wake_up(waitq);
603}
604extern wait_queue_head_t rds_poll_waitq;
605
606
607
608int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
609void rds_remove_bound(struct rds_sock *rs);
610struct rds_sock *rds_find_bound(__be32 addr, __be16 port);
611int rds_bind_lock_init(void);
612void rds_bind_lock_destroy(void);
613
614
615int rds_cong_get_maps(struct rds_connection *conn);
616void rds_cong_add_conn(struct rds_connection *conn);
617void rds_cong_remove_conn(struct rds_connection *conn);
618void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
619void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
620int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
621void rds_cong_queue_updates(struct rds_cong_map *map);
622void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
623int rds_cong_updated_since(unsigned long *recent);
624void rds_cong_add_socket(struct rds_sock *);
625void rds_cong_remove_socket(struct rds_sock *);
626void rds_cong_exit(void);
627struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
628
629
630int rds_conn_init(void);
631void rds_conn_exit(void);
632struct rds_connection *rds_conn_create(struct net *net,
633 __be32 laddr, __be32 faddr,
634 struct rds_transport *trans, gfp_t gfp);
635struct rds_connection *rds_conn_create_outgoing(struct net *net,
636 __be32 laddr, __be32 faddr,
637 struct rds_transport *trans, gfp_t gfp);
638void rds_conn_shutdown(struct rds_connection *conn);
639void rds_conn_destroy(struct rds_connection *conn);
640void rds_conn_drop(struct rds_connection *conn);
641void rds_conn_connect_if_down(struct rds_connection *conn);
642void rds_for_each_conn_info(struct socket *sock, unsigned int len,
643 struct rds_info_iterator *iter,
644 struct rds_info_lengths *lens,
645 int (*visitor)(struct rds_connection *, void *),
646 size_t item_len);
647__printf(2, 3)
648void __rds_conn_error(struct rds_connection *conn, const char *, ...);
649#define rds_conn_error(conn, fmt...) \
650 __rds_conn_error(conn, KERN_WARNING "RDS: " fmt)
651
652static inline int
653rds_conn_transition(struct rds_connection *conn, int old, int new)
654{
655 return atomic_cmpxchg(&conn->c_state, old, new) == old;
656}
657
658static inline int
659rds_conn_state(struct rds_connection *conn)
660{
661 return atomic_read(&conn->c_state);
662}
663
664static inline int
665rds_conn_up(struct rds_connection *conn)
666{
667 return atomic_read(&conn->c_state) == RDS_CONN_UP;
668}
669
670static inline int
671rds_conn_connecting(struct rds_connection *conn)
672{
673 return atomic_read(&conn->c_state) == RDS_CONN_CONNECTING;
674}
675
676
677struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
678struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
679int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from);
680struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
681void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
682 __be16 dport, u64 seq);
683int rds_message_add_extension(struct rds_header *hdr,
684 unsigned int type, const void *data, unsigned int len);
685int rds_message_next_extension(struct rds_header *hdr,
686 unsigned int *pos, void *buf, unsigned int *buflen);
687int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
688int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
689void rds_message_inc_free(struct rds_incoming *inc);
690void rds_message_addref(struct rds_message *rm);
691void rds_message_put(struct rds_message *rm);
692void rds_message_wait(struct rds_message *rm);
693void rds_message_unmapped(struct rds_message *rm);
694
695static inline void rds_message_make_checksum(struct rds_header *hdr)
696{
697 hdr->h_csum = 0;
698 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
699}
700
701static inline int rds_message_verify_checksum(const struct rds_header *hdr)
702{
703 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
704}
705
706
707
708int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
709 gfp_t gfp);
710int rds_page_copy_user(struct page *page, unsigned long offset,
711 void __user *ptr, unsigned long bytes,
712 int to_user);
713#define rds_page_copy_to_user(page, offset, ptr, bytes) \
714 rds_page_copy_user(page, offset, ptr, bytes, 1)
715#define rds_page_copy_from_user(page, offset, ptr, bytes) \
716 rds_page_copy_user(page, offset, ptr, bytes, 0)
717void rds_page_exit(void);
718
719
720void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
721 __be32 saddr);
722void rds_inc_put(struct rds_incoming *inc);
723void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
724 struct rds_incoming *inc, gfp_t gfp);
725int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
726 int msg_flags);
727void rds_clear_recv_queue(struct rds_sock *rs);
728int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
729void rds_inc_info_copy(struct rds_incoming *inc,
730 struct rds_info_iterator *iter,
731 __be32 saddr, __be32 daddr, int flip);
732
733
734int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
735void rds_send_reset(struct rds_connection *conn);
736int rds_send_xmit(struct rds_connection *conn);
737struct sockaddr_in;
738void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest);
739typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
740void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
741 is_acked_func is_acked);
742int rds_send_pong(struct rds_connection *conn, __be16 dport);
743struct rds_message *rds_send_get_message(struct rds_connection *,
744 struct rm_rdma_op *);
745
746
747void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
748int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
749int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
750int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
751void rds_rdma_drop_keys(struct rds_sock *rs);
752int rds_rdma_extra_size(struct rds_rdma_args *args);
753int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
754 struct cmsghdr *cmsg);
755int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
756 struct cmsghdr *cmsg);
757int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
758 struct cmsghdr *cmsg);
759int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
760 struct cmsghdr *cmsg);
761void rds_rdma_free_op(struct rm_rdma_op *ro);
762void rds_atomic_free_op(struct rm_atomic_op *ao);
763void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
764void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
765int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
766 struct cmsghdr *cmsg);
767
768void __rds_put_mr_final(struct rds_mr *mr);
769static inline void rds_mr_put(struct rds_mr *mr)
770{
771 if (atomic_dec_and_test(&mr->r_refcount))
772 __rds_put_mr_final(mr);
773}
774
775
776DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
777#define rds_stats_inc_which(which, member) do { \
778 per_cpu(which, get_cpu()).member++; \
779 put_cpu(); \
780} while (0)
781#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
782#define rds_stats_add_which(which, member, count) do { \
783 per_cpu(which, get_cpu()).member += count; \
784 put_cpu(); \
785} while (0)
786#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
787int rds_stats_init(void);
788void rds_stats_exit(void);
789void rds_stats_info_copy(struct rds_info_iterator *iter,
790 uint64_t *values, const char *const *names,
791 size_t nr);
792
793
794int rds_sysctl_init(void);
795void rds_sysctl_exit(void);
796extern unsigned long rds_sysctl_sndbuf_min;
797extern unsigned long rds_sysctl_sndbuf_default;
798extern unsigned long rds_sysctl_sndbuf_max;
799extern unsigned long rds_sysctl_reconnect_min_jiffies;
800extern unsigned long rds_sysctl_reconnect_max_jiffies;
801extern unsigned int rds_sysctl_max_unacked_packets;
802extern unsigned int rds_sysctl_max_unacked_bytes;
803extern unsigned int rds_sysctl_ping_enable;
804extern unsigned long rds_sysctl_trace_flags;
805extern unsigned int rds_sysctl_trace_level;
806
807
808int rds_threads_init(void);
809void rds_threads_exit(void);
810extern struct workqueue_struct *rds_wq;
811void rds_queue_reconnect(struct rds_connection *conn);
812void rds_connect_worker(struct work_struct *);
813void rds_shutdown_worker(struct work_struct *);
814void rds_send_worker(struct work_struct *);
815void rds_recv_worker(struct work_struct *);
816void rds_connect_complete(struct rds_connection *conn);
817
818
819int rds_trans_register(struct rds_transport *trans);
820void rds_trans_unregister(struct rds_transport *trans);
821struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
822void rds_trans_put(struct rds_transport *trans);
823unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
824 unsigned int avail);
825struct rds_transport *rds_trans_get(int t_type);
826int rds_trans_init(void);
827void rds_trans_exit(void);
828
829#endif
830