1
2#ifndef _RDS_RDS_H
3#define _RDS_RDS_H
4
5#include <net/sock.h>
6#include <linux/scatterlist.h>
7#include <linux/highmem.h>
8#include <rdma/rdma_cm.h>
9#include <linux/mutex.h>
10#include <linux/rds.h>
11#include <linux/rhashtable.h>
12#include <linux/refcount.h>
13
14#include "info.h"
15
16
17
18
19#define RDS_PROTOCOL_3_0 0x0300
20#define RDS_PROTOCOL_3_1 0x0301
21#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
22#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
23#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
24#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
25
26
27
28
29
30
31
32#define RDS_PORT 18634
33
34#ifdef ATOMIC64_INIT
35#define KERNEL_HAS_ATOMIC64
36#endif
37
38#ifdef RDS_DEBUG
39#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
40#else
41
42static inline __printf(1, 2)
43void rdsdebug(char *fmt, ...)
44{
45}
46#endif
47
48
49#define ceil(x, y) \
50 ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
51
52#define RDS_FRAG_SHIFT 12
53#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
54
55
56#define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
57
58#define RDS_CONG_MAP_BYTES (65536 / 8)
59#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
60#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
61
62struct rds_cong_map {
63 struct rb_node m_rb_node;
64 __be32 m_addr;
65 wait_queue_head_t m_waitq;
66 struct list_head m_conn_list;
67 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
68};
69
70
71
72
73
74
75
76
77enum {
78 RDS_CONN_DOWN = 0,
79 RDS_CONN_CONNECTING,
80 RDS_CONN_DISCONNECTING,
81 RDS_CONN_UP,
82 RDS_CONN_RESETTING,
83 RDS_CONN_ERROR,
84};
85
86
87#define RDS_LL_SEND_FULL 0
88#define RDS_RECONNECT_PENDING 1
89#define RDS_IN_XMIT 2
90#define RDS_RECV_REFILL 3
91#define RDS_DESTROY_PENDING 4
92
93
94#define RDS_MPATH_WORKERS 8
95#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
96 (rs)->rs_hash_initval) & ((n) - 1))
97
98#define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
99
100
101struct rds_conn_path {
102 struct rds_connection *cp_conn;
103 struct rds_message *cp_xmit_rm;
104 unsigned long cp_xmit_sg;
105 unsigned int cp_xmit_hdr_off;
106 unsigned int cp_xmit_data_off;
107 unsigned int cp_xmit_atomic_sent;
108 unsigned int cp_xmit_rdma_sent;
109 unsigned int cp_xmit_data_sent;
110
111 spinlock_t cp_lock;
112 u64 cp_next_tx_seq;
113 struct list_head cp_send_queue;
114 struct list_head cp_retrans;
115
116 u64 cp_next_rx_seq;
117
118 void *cp_transport_data;
119
120 atomic_t cp_state;
121 unsigned long cp_send_gen;
122 unsigned long cp_flags;
123 unsigned long cp_reconnect_jiffies;
124 struct delayed_work cp_send_w;
125 struct delayed_work cp_recv_w;
126 struct delayed_work cp_conn_w;
127 struct work_struct cp_down_w;
128 struct mutex cp_cm_lock;
129 wait_queue_head_t cp_waitq;
130
131 unsigned int cp_unacked_packets;
132 unsigned int cp_unacked_bytes;
133 unsigned int cp_index;
134};
135
136
137struct rds_connection {
138 struct hlist_node c_hash_node;
139 __be32 c_laddr;
140 __be32 c_faddr;
141 unsigned int c_loopback:1,
142 c_ping_triggered:1,
143 c_pad_to_32:30;
144 int c_npaths;
145 struct rds_connection *c_passive;
146 struct rds_transport *c_trans;
147
148 struct rds_cong_map *c_lcong;
149 struct rds_cong_map *c_fcong;
150
151
152 unsigned int c_version;
153 possible_net_t c_net;
154
155 struct list_head c_map_item;
156 unsigned long c_map_queued;
157
158 struct rds_conn_path *c_path;
159 wait_queue_head_t c_hs_waitq;
160
161 u32 c_my_gen_num;
162 u32 c_peer_gen_num;
163};
164
165static inline
166struct net *rds_conn_net(struct rds_connection *conn)
167{
168 return read_pnet(&conn->c_net);
169}
170
171static inline
172void rds_conn_net_set(struct rds_connection *conn, struct net *net)
173{
174 write_pnet(&conn->c_net, net);
175}
176
177#define RDS_FLAG_CONG_BITMAP 0x01
178#define RDS_FLAG_ACK_REQUIRED 0x02
179#define RDS_FLAG_RETRANSMITTED 0x04
180#define RDS_MAX_ADV_CREDIT 255
181
182
183
184
185
186
187
188
189#define RDS_FLAG_PROBE_PORT 1
190#define RDS_HS_PROBE(sport, dport) \
191 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
192 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
193
194
195
196#define RDS_HEADER_EXT_SPACE 16
197
198struct rds_header {
199 __be64 h_sequence;
200 __be64 h_ack;
201 __be32 h_len;
202 __be16 h_sport;
203 __be16 h_dport;
204 u8 h_flags;
205 u8 h_credit;
206 u8 h_padding[4];
207 __sum16 h_csum;
208
209 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
210};
211
212
213
214
215#define RDS_EXTHDR_NONE 0
216
217
218
219
220
221
222
223
224
225
226
227#define RDS_EXTHDR_VERSION 1
228struct rds_ext_header_version {
229 __be32 h_version;
230};
231
232
233
234
235
236#define RDS_EXTHDR_RDMA 2
237struct rds_ext_header_rdma {
238 __be32 h_rdma_rkey;
239};
240
241
242
243
244
245
246#define RDS_EXTHDR_RDMA_DEST 3
247struct rds_ext_header_rdma_dest {
248 __be32 h_rdma_rkey;
249 __be32 h_rdma_offset;
250};
251
252
253
254
255#define RDS_EXTHDR_NPATHS 5
256#define RDS_EXTHDR_GEN_NUM 6
257
258#define __RDS_EXTHDR_MAX 16
259#define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
260#define RDS_MSG_RX_HDR 0
261#define RDS_MSG_RX_START 1
262#define RDS_MSG_RX_END 2
263#define RDS_MSG_RX_CMSG 3
264
265struct rds_incoming {
266 refcount_t i_refcount;
267 struct list_head i_item;
268 struct rds_connection *i_conn;
269 struct rds_conn_path *i_conn_path;
270 struct rds_header i_hdr;
271 unsigned long i_rx_jiffies;
272 __be32 i_saddr;
273
274 rds_rdma_cookie_t i_rdma_cookie;
275 struct timeval i_rx_tstamp;
276 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
277};
278
279struct rds_mr {
280 struct rb_node r_rb_node;
281 refcount_t r_refcount;
282 u32 r_key;
283
284
285 unsigned int r_use_once:1;
286 unsigned int r_invalidate:1;
287 unsigned int r_write:1;
288
289
290
291
292
293 unsigned long r_state;
294 struct rds_sock *r_sock;
295 struct rds_transport *r_trans;
296 void *r_trans_private;
297};
298
299
300#define RDS_MR_DEAD 0
301
302static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
303{
304 return r_key | (((u64) offset) << 32);
305}
306
307static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
308{
309 return cookie;
310}
311
312static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
313{
314 return cookie >> 32;
315}
316
317
318#define RDS_ATOMIC_TYPE_CSWP 0
319#define RDS_ATOMIC_TYPE_FADD 1
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350#define RDS_MSG_ON_SOCK 1
351#define RDS_MSG_ON_CONN 2
352#define RDS_MSG_HAS_ACK_SEQ 3
353#define RDS_MSG_ACK_REQUIRED 4
354#define RDS_MSG_RETRANSMITTED 5
355#define RDS_MSG_MAPPED 6
356#define RDS_MSG_PAGEVEC 7
357#define RDS_MSG_FLUSH 8
358
359struct rds_znotifier {
360 struct mmpin z_mmp;
361 u32 z_cookie;
362};
363
364struct rds_msg_zcopy_info {
365 struct list_head rs_zcookie_next;
366 union {
367 struct rds_znotifier znotif;
368 struct rds_zcopy_cookies zcookies;
369 };
370};
371
372struct rds_msg_zcopy_queue {
373 struct list_head zcookie_head;
374 spinlock_t lock;
375};
376
377static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
378{
379 spin_lock_init(&q->lock);
380 INIT_LIST_HEAD(&q->zcookie_head);
381}
382
383struct rds_message {
384 refcount_t m_refcount;
385 struct list_head m_sock_item;
386 struct list_head m_conn_item;
387 struct rds_incoming m_inc;
388 u64 m_ack_seq;
389 __be32 m_daddr;
390 unsigned long m_flags;
391
392
393
394
395
396
397 spinlock_t m_rs_lock;
398 wait_queue_head_t m_flush_wait;
399
400 struct rds_sock *m_rs;
401
402
403 rds_rdma_cookie_t m_rdma_cookie;
404
405 unsigned int m_used_sgs;
406 unsigned int m_total_sgs;
407
408 void *m_final_op;
409
410 struct {
411 struct rm_atomic_op {
412 int op_type;
413 union {
414 struct {
415 uint64_t compare;
416 uint64_t swap;
417 uint64_t compare_mask;
418 uint64_t swap_mask;
419 } op_m_cswp;
420 struct {
421 uint64_t add;
422 uint64_t nocarry_mask;
423 } op_m_fadd;
424 };
425
426 u32 op_rkey;
427 u64 op_remote_addr;
428 unsigned int op_notify:1;
429 unsigned int op_recverr:1;
430 unsigned int op_mapped:1;
431 unsigned int op_silent:1;
432 unsigned int op_active:1;
433 struct scatterlist *op_sg;
434 struct rds_notifier *op_notifier;
435
436 struct rds_mr *op_rdma_mr;
437 } atomic;
438 struct rm_rdma_op {
439 u32 op_rkey;
440 u64 op_remote_addr;
441 unsigned int op_write:1;
442 unsigned int op_fence:1;
443 unsigned int op_notify:1;
444 unsigned int op_recverr:1;
445 unsigned int op_mapped:1;
446 unsigned int op_silent:1;
447 unsigned int op_active:1;
448 unsigned int op_bytes;
449 unsigned int op_nents;
450 unsigned int op_count;
451 struct scatterlist *op_sg;
452 struct rds_notifier *op_notifier;
453
454 struct rds_mr *op_rdma_mr;
455 } rdma;
456 struct rm_data_op {
457 unsigned int op_active:1;
458 unsigned int op_notify:1;
459 unsigned int op_nents;
460 unsigned int op_count;
461 unsigned int op_dmasg;
462 unsigned int op_dmaoff;
463 struct rds_znotifier *op_mmp_znotifier;
464 struct scatterlist *op_sg;
465 } data;
466 };
467
468 struct rds_conn_path *m_conn_path;
469};
470
471
472
473
474
475
476
477
478struct rds_notifier {
479 struct list_head n_list;
480 uint64_t n_user_token;
481 int n_status;
482};
483
484
485
486
487#define RDS_TRANS_LOOP 3
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516struct rds_transport {
517 char t_name[TRANSNAMSIZ];
518 struct list_head t_item;
519 struct module *t_owner;
520 unsigned int t_prefer_loopback:1,
521 t_mp_capable:1;
522 unsigned int t_type;
523
524 int (*laddr_check)(struct net *net, __be32 addr);
525 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
526 void (*conn_free)(void *data);
527 int (*conn_path_connect)(struct rds_conn_path *cp);
528 void (*conn_path_shutdown)(struct rds_conn_path *conn);
529 void (*xmit_path_prepare)(struct rds_conn_path *cp);
530 void (*xmit_path_complete)(struct rds_conn_path *cp);
531 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
532 unsigned int hdr_off, unsigned int sg, unsigned int off);
533 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
534 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
535 int (*recv_path)(struct rds_conn_path *cp);
536 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
537 void (*inc_free)(struct rds_incoming *inc);
538
539 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
540 struct rdma_cm_event *event);
541 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id);
542 void (*cm_connect_complete)(struct rds_connection *conn,
543 struct rdma_cm_event *event);
544
545 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
546 unsigned int avail);
547 void (*exit)(void);
548 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
549 struct rds_sock *rs, u32 *key_ret,
550 struct rds_connection *conn);
551 void (*sync_mr)(void *trans_private, int direction);
552 void (*free_mr)(void *trans_private, int invalidate);
553 void (*flush_mrs)(void);
554 bool (*t_unloading)(struct rds_connection *conn);
555};
556
557struct rds_sock {
558 struct sock rs_sk;
559
560 u64 rs_user_addr;
561 u64 rs_user_bytes;
562
563
564
565
566
567 struct rhash_head rs_bound_node;
568 u64 rs_bound_key;
569 __be32 rs_bound_addr;
570 __be32 rs_conn_addr;
571 __be16 rs_bound_port;
572 __be16 rs_conn_port;
573 struct rds_transport *rs_transport;
574
575
576
577
578
579 struct rds_connection *rs_conn;
580
581
582 int rs_congested;
583
584 int rs_seen_congestion;
585
586
587 spinlock_t rs_lock;
588 struct list_head rs_send_queue;
589 u32 rs_snd_bytes;
590 int rs_rcv_bytes;
591 struct list_head rs_notify_queue;
592
593
594
595
596
597
598 uint64_t rs_cong_mask;
599 uint64_t rs_cong_notify;
600 struct list_head rs_cong_list;
601 unsigned long rs_cong_track;
602
603
604
605
606
607 rwlock_t rs_recv_lock;
608 struct list_head rs_recv_queue;
609
610
611 struct list_head rs_item;
612
613
614 spinlock_t rs_rdma_lock;
615 struct rb_root rs_rdma_keys;
616
617
618 unsigned char rs_recverr,
619 rs_cong_monitor;
620 u32 rs_hash_initval;
621
622
623 u8 rs_rx_traces;
624 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
625 struct rds_msg_zcopy_queue rs_zcookie_queue;
626};
627
628static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
629{
630 return container_of(sk, struct rds_sock, rs_sk);
631}
632static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
633{
634 return &rs->rs_sk;
635}
636
637
638
639
640
641
642static inline int rds_sk_sndbuf(struct rds_sock *rs)
643{
644 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
645}
646static inline int rds_sk_rcvbuf(struct rds_sock *rs)
647{
648 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
649}
650
651struct rds_statistics {
652 uint64_t s_conn_reset;
653 uint64_t s_recv_drop_bad_checksum;
654 uint64_t s_recv_drop_old_seq;
655 uint64_t s_recv_drop_no_sock;
656 uint64_t s_recv_drop_dead_sock;
657 uint64_t s_recv_deliver_raced;
658 uint64_t s_recv_delivered;
659 uint64_t s_recv_queued;
660 uint64_t s_recv_immediate_retry;
661 uint64_t s_recv_delayed_retry;
662 uint64_t s_recv_ack_required;
663 uint64_t s_recv_rdma_bytes;
664 uint64_t s_recv_ping;
665 uint64_t s_send_queue_empty;
666 uint64_t s_send_queue_full;
667 uint64_t s_send_lock_contention;
668 uint64_t s_send_lock_queue_raced;
669 uint64_t s_send_immediate_retry;
670 uint64_t s_send_delayed_retry;
671 uint64_t s_send_drop_acked;
672 uint64_t s_send_ack_required;
673 uint64_t s_send_queued;
674 uint64_t s_send_rdma;
675 uint64_t s_send_rdma_bytes;
676 uint64_t s_send_pong;
677 uint64_t s_page_remainder_hit;
678 uint64_t s_page_remainder_miss;
679 uint64_t s_copy_to_user;
680 uint64_t s_copy_from_user;
681 uint64_t s_cong_update_queued;
682 uint64_t s_cong_update_received;
683 uint64_t s_cong_send_error;
684 uint64_t s_cong_send_blocked;
685 uint64_t s_recv_bytes_added_to_socket;
686 uint64_t s_recv_bytes_removed_from_socket;
687
688};
689
690
691void rds_sock_addref(struct rds_sock *rs);
692void rds_sock_put(struct rds_sock *rs);
693void rds_wake_sk_sleep(struct rds_sock *rs);
694static inline void __rds_wake_sk_sleep(struct sock *sk)
695{
696 wait_queue_head_t *waitq = sk_sleep(sk);
697
698 if (!sock_flag(sk, SOCK_DEAD) && waitq)
699 wake_up(waitq);
700}
701extern wait_queue_head_t rds_poll_waitq;
702
703
704
705int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
706void rds_remove_bound(struct rds_sock *rs);
707struct rds_sock *rds_find_bound(__be32 addr, __be16 port);
708int rds_bind_lock_init(void);
709void rds_bind_lock_destroy(void);
710
711
712int rds_cong_get_maps(struct rds_connection *conn);
713void rds_cong_add_conn(struct rds_connection *conn);
714void rds_cong_remove_conn(struct rds_connection *conn);
715void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
716void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
717int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
718void rds_cong_queue_updates(struct rds_cong_map *map);
719void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
720int rds_cong_updated_since(unsigned long *recent);
721void rds_cong_add_socket(struct rds_sock *);
722void rds_cong_remove_socket(struct rds_sock *);
723void rds_cong_exit(void);
724struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
725
726
727extern u32 rds_gen_num;
728int rds_conn_init(void);
729void rds_conn_exit(void);
730struct rds_connection *rds_conn_create(struct net *net,
731 __be32 laddr, __be32 faddr,
732 struct rds_transport *trans, gfp_t gfp);
733struct rds_connection *rds_conn_create_outgoing(struct net *net,
734 __be32 laddr, __be32 faddr,
735 struct rds_transport *trans, gfp_t gfp);
736void rds_conn_shutdown(struct rds_conn_path *cpath);
737void rds_conn_destroy(struct rds_connection *conn);
738void rds_conn_drop(struct rds_connection *conn);
739void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
740void rds_conn_connect_if_down(struct rds_connection *conn);
741void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
742void rds_for_each_conn_info(struct socket *sock, unsigned int len,
743 struct rds_info_iterator *iter,
744 struct rds_info_lengths *lens,
745 int (*visitor)(struct rds_connection *, void *),
746 u64 *buffer,
747 size_t item_len);
748
749__printf(2, 3)
750void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
751#define rds_conn_path_error(cp, fmt...) \
752 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
753
754static inline int
755rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
756{
757 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
758}
759
760static inline int
761rds_conn_transition(struct rds_connection *conn, int old, int new)
762{
763 WARN_ON(conn->c_trans->t_mp_capable);
764 return rds_conn_path_transition(&conn->c_path[0], old, new);
765}
766
767static inline int
768rds_conn_path_state(struct rds_conn_path *cp)
769{
770 return atomic_read(&cp->cp_state);
771}
772
773static inline int
774rds_conn_state(struct rds_connection *conn)
775{
776 WARN_ON(conn->c_trans->t_mp_capable);
777 return rds_conn_path_state(&conn->c_path[0]);
778}
779
780static inline int
781rds_conn_path_up(struct rds_conn_path *cp)
782{
783 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
784}
785
786static inline int
787rds_conn_up(struct rds_connection *conn)
788{
789 WARN_ON(conn->c_trans->t_mp_capable);
790 return rds_conn_path_up(&conn->c_path[0]);
791}
792
793static inline int
794rds_conn_path_connecting(struct rds_conn_path *cp)
795{
796 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
797}
798
799static inline int
800rds_conn_connecting(struct rds_connection *conn)
801{
802 WARN_ON(conn->c_trans->t_mp_capable);
803 return rds_conn_path_connecting(&conn->c_path[0]);
804}
805
806
807struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
808struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
809int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
810 bool zcopy);
811struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
812void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
813 __be16 dport, u64 seq);
814int rds_message_add_extension(struct rds_header *hdr,
815 unsigned int type, const void *data, unsigned int len);
816int rds_message_next_extension(struct rds_header *hdr,
817 unsigned int *pos, void *buf, unsigned int *buflen);
818int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
819int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
820void rds_message_inc_free(struct rds_incoming *inc);
821void rds_message_addref(struct rds_message *rm);
822void rds_message_put(struct rds_message *rm);
823void rds_message_wait(struct rds_message *rm);
824void rds_message_unmapped(struct rds_message *rm);
825void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
826
827static inline void rds_message_make_checksum(struct rds_header *hdr)
828{
829 hdr->h_csum = 0;
830 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
831}
832
833static inline int rds_message_verify_checksum(const struct rds_header *hdr)
834{
835 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
836}
837
838
839
840int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
841 gfp_t gfp);
842void rds_page_exit(void);
843
844
845void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
846 __be32 saddr);
847void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
848 __be32 saddr);
849void rds_inc_put(struct rds_incoming *inc);
850void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
851 struct rds_incoming *inc, gfp_t gfp);
852int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
853 int msg_flags);
854void rds_clear_recv_queue(struct rds_sock *rs);
855int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
856void rds_inc_info_copy(struct rds_incoming *inc,
857 struct rds_info_iterator *iter,
858 __be32 saddr, __be32 daddr, int flip);
859
860
861int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
862void rds_send_path_reset(struct rds_conn_path *conn);
863int rds_send_xmit(struct rds_conn_path *cp);
864struct sockaddr_in;
865void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest);
866typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
867void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
868 is_acked_func is_acked);
869void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
870 is_acked_func is_acked);
871void rds_send_ping(struct rds_connection *conn, int cp_index);
872int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
873
874
875void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
876int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
877int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
878int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
879void rds_rdma_drop_keys(struct rds_sock *rs);
880int rds_rdma_extra_size(struct rds_rdma_args *args);
881int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
882 struct cmsghdr *cmsg);
883int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
884 struct cmsghdr *cmsg);
885int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
886 struct cmsghdr *cmsg);
887int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
888 struct cmsghdr *cmsg);
889void rds_rdma_free_op(struct rm_rdma_op *ro);
890void rds_atomic_free_op(struct rm_atomic_op *ao);
891void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
892void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
893int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
894 struct cmsghdr *cmsg);
895
896void __rds_put_mr_final(struct rds_mr *mr);
897static inline void rds_mr_put(struct rds_mr *mr)
898{
899 if (refcount_dec_and_test(&mr->r_refcount))
900 __rds_put_mr_final(mr);
901}
902
903static inline bool rds_destroy_pending(struct rds_connection *conn)
904{
905 return !check_net(rds_conn_net(conn)) ||
906 (conn->c_trans->t_unloading && conn->c_trans->t_unloading(conn));
907}
908
909
910DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
911#define rds_stats_inc_which(which, member) do { \
912 per_cpu(which, get_cpu()).member++; \
913 put_cpu(); \
914} while (0)
915#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
916#define rds_stats_add_which(which, member, count) do { \
917 per_cpu(which, get_cpu()).member += count; \
918 put_cpu(); \
919} while (0)
920#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
921int rds_stats_init(void);
922void rds_stats_exit(void);
923void rds_stats_info_copy(struct rds_info_iterator *iter,
924 uint64_t *values, const char *const *names,
925 size_t nr);
926
927
928int rds_sysctl_init(void);
929void rds_sysctl_exit(void);
930extern unsigned long rds_sysctl_sndbuf_min;
931extern unsigned long rds_sysctl_sndbuf_default;
932extern unsigned long rds_sysctl_sndbuf_max;
933extern unsigned long rds_sysctl_reconnect_min_jiffies;
934extern unsigned long rds_sysctl_reconnect_max_jiffies;
935extern unsigned int rds_sysctl_max_unacked_packets;
936extern unsigned int rds_sysctl_max_unacked_bytes;
937extern unsigned int rds_sysctl_ping_enable;
938extern unsigned long rds_sysctl_trace_flags;
939extern unsigned int rds_sysctl_trace_level;
940
941
942int rds_threads_init(void);
943void rds_threads_exit(void);
944extern struct workqueue_struct *rds_wq;
945void rds_queue_reconnect(struct rds_conn_path *cp);
946void rds_connect_worker(struct work_struct *);
947void rds_shutdown_worker(struct work_struct *);
948void rds_send_worker(struct work_struct *);
949void rds_recv_worker(struct work_struct *);
950void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
951void rds_connect_complete(struct rds_connection *conn);
952
953
954void rds_trans_register(struct rds_transport *trans);
955void rds_trans_unregister(struct rds_transport *trans);
956struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
957void rds_trans_put(struct rds_transport *trans);
958unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
959 unsigned int avail);
960struct rds_transport *rds_trans_get(int t_type);
961int rds_trans_init(void);
962void rds_trans_exit(void);
963
964#endif
965