1
2#ifndef _RDS_RDS_H
3#define _RDS_RDS_H
4
5#include <net/sock.h>
6#include <linux/scatterlist.h>
7#include <linux/highmem.h>
8#include <rdma/rdma_cm.h>
9#include <linux/mutex.h>
10#include <linux/rds.h>
11#include <linux/rhashtable.h>
12#include <linux/refcount.h>
13#include <linux/in6.h>
14
15#include "info.h"
16
17
18
19
20#define RDS_PROTOCOL_3_0 0x0300
21#define RDS_PROTOCOL_3_1 0x0301
22#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
23#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
24#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
25#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
26
27
28
29
30
31
32
33
34
35#define RDS_PORT 18634
36
37#ifdef ATOMIC64_INIT
38#define KERNEL_HAS_ATOMIC64
39#endif
40
41#ifdef RDS_DEBUG
42#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
43#else
44
45static inline __printf(1, 2)
46void rdsdebug(char *fmt, ...)
47{
48}
49#endif
50
51#define RDS_FRAG_SHIFT 12
52#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
53
54
55#define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
56
57#define RDS_CONG_MAP_BYTES (65536 / 8)
58#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
59#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
60
61struct rds_cong_map {
62 struct rb_node m_rb_node;
63 struct in6_addr m_addr;
64 wait_queue_head_t m_waitq;
65 struct list_head m_conn_list;
66 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
67};
68
69
70
71
72
73
74
75
76enum {
77 RDS_CONN_DOWN = 0,
78 RDS_CONN_CONNECTING,
79 RDS_CONN_DISCONNECTING,
80 RDS_CONN_UP,
81 RDS_CONN_RESETTING,
82 RDS_CONN_ERROR,
83};
84
85
86#define RDS_LL_SEND_FULL 0
87#define RDS_RECONNECT_PENDING 1
88#define RDS_IN_XMIT 2
89#define RDS_RECV_REFILL 3
90#define RDS_DESTROY_PENDING 4
91
92
93#define RDS_MPATH_WORKERS 8
94#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
95 (rs)->rs_hash_initval) & ((n) - 1))
96
97#define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
98
99
100struct rds_conn_path {
101 struct rds_connection *cp_conn;
102 struct rds_message *cp_xmit_rm;
103 unsigned long cp_xmit_sg;
104 unsigned int cp_xmit_hdr_off;
105 unsigned int cp_xmit_data_off;
106 unsigned int cp_xmit_atomic_sent;
107 unsigned int cp_xmit_rdma_sent;
108 unsigned int cp_xmit_data_sent;
109
110 spinlock_t cp_lock;
111 u64 cp_next_tx_seq;
112 struct list_head cp_send_queue;
113 struct list_head cp_retrans;
114
115 u64 cp_next_rx_seq;
116
117 void *cp_transport_data;
118
119 atomic_t cp_state;
120 unsigned long cp_send_gen;
121 unsigned long cp_flags;
122 unsigned long cp_reconnect_jiffies;
123 struct delayed_work cp_send_w;
124 struct delayed_work cp_recv_w;
125 struct delayed_work cp_conn_w;
126 struct work_struct cp_down_w;
127 struct mutex cp_cm_lock;
128 wait_queue_head_t cp_waitq;
129
130 unsigned int cp_unacked_packets;
131 unsigned int cp_unacked_bytes;
132 unsigned int cp_index;
133};
134
135
136struct rds_connection {
137 struct hlist_node c_hash_node;
138 struct in6_addr c_laddr;
139 struct in6_addr c_faddr;
140 int c_dev_if;
141 int c_bound_if;
142 unsigned int c_loopback:1,
143 c_isv6:1,
144 c_ping_triggered:1,
145 c_pad_to_32:29;
146 int c_npaths;
147 struct rds_connection *c_passive;
148 struct rds_transport *c_trans;
149
150 struct rds_cong_map *c_lcong;
151 struct rds_cong_map *c_fcong;
152
153
154 unsigned int c_version;
155 possible_net_t c_net;
156
157 struct list_head c_map_item;
158 unsigned long c_map_queued;
159
160 struct rds_conn_path *c_path;
161 wait_queue_head_t c_hs_waitq;
162
163 u32 c_my_gen_num;
164 u32 c_peer_gen_num;
165};
166
167static inline
168struct net *rds_conn_net(struct rds_connection *conn)
169{
170 return read_pnet(&conn->c_net);
171}
172
173static inline
174void rds_conn_net_set(struct rds_connection *conn, struct net *net)
175{
176 write_pnet(&conn->c_net, net);
177}
178
179#define RDS_FLAG_CONG_BITMAP 0x01
180#define RDS_FLAG_ACK_REQUIRED 0x02
181#define RDS_FLAG_RETRANSMITTED 0x04
182#define RDS_MAX_ADV_CREDIT 255
183
184
185
186
187
188
189
190
191#define RDS_FLAG_PROBE_PORT 1
192#define RDS_HS_PROBE(sport, dport) \
193 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
194 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
195
196
197
198#define RDS_HEADER_EXT_SPACE 16
199
200struct rds_header {
201 __be64 h_sequence;
202 __be64 h_ack;
203 __be32 h_len;
204 __be16 h_sport;
205 __be16 h_dport;
206 u8 h_flags;
207 u8 h_credit;
208 u8 h_padding[4];
209 __sum16 h_csum;
210
211 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
212};
213
214
215
216
217#define RDS_EXTHDR_NONE 0
218
219
220
221
222
223
224
225
226
227
228
229#define RDS_EXTHDR_VERSION 1
230struct rds_ext_header_version {
231 __be32 h_version;
232};
233
234
235
236
237
238#define RDS_EXTHDR_RDMA 2
239struct rds_ext_header_rdma {
240 __be32 h_rdma_rkey;
241};
242
243
244
245
246
247
248#define RDS_EXTHDR_RDMA_DEST 3
249struct rds_ext_header_rdma_dest {
250 __be32 h_rdma_rkey;
251 __be32 h_rdma_offset;
252};
253
254
255
256
257#define RDS_EXTHDR_NPATHS 5
258#define RDS_EXTHDR_GEN_NUM 6
259
260#define __RDS_EXTHDR_MAX 16
261#define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
262#define RDS_MSG_RX_HDR 0
263#define RDS_MSG_RX_START 1
264#define RDS_MSG_RX_END 2
265#define RDS_MSG_RX_CMSG 3
266
267struct rds_incoming {
268 refcount_t i_refcount;
269 struct list_head i_item;
270 struct rds_connection *i_conn;
271 struct rds_conn_path *i_conn_path;
272 struct rds_header i_hdr;
273 unsigned long i_rx_jiffies;
274 struct in6_addr i_saddr;
275
276 rds_rdma_cookie_t i_rdma_cookie;
277 ktime_t i_rx_tstamp;
278 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
279};
280
281struct rds_mr {
282 struct rb_node r_rb_node;
283 refcount_t r_refcount;
284 u32 r_key;
285
286
287 unsigned int r_use_once:1;
288 unsigned int r_invalidate:1;
289 unsigned int r_write:1;
290
291
292
293
294
295 unsigned long r_state;
296 struct rds_sock *r_sock;
297 struct rds_transport *r_trans;
298 void *r_trans_private;
299};
300
301
302#define RDS_MR_DEAD 0
303
304static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
305{
306 return r_key | (((u64) offset) << 32);
307}
308
309static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
310{
311 return cookie;
312}
313
314static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
315{
316 return cookie >> 32;
317}
318
319
320#define RDS_ATOMIC_TYPE_CSWP 0
321#define RDS_ATOMIC_TYPE_FADD 1
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352#define RDS_MSG_ON_SOCK 1
353#define RDS_MSG_ON_CONN 2
354#define RDS_MSG_HAS_ACK_SEQ 3
355#define RDS_MSG_ACK_REQUIRED 4
356#define RDS_MSG_RETRANSMITTED 5
357#define RDS_MSG_MAPPED 6
358#define RDS_MSG_PAGEVEC 7
359#define RDS_MSG_FLUSH 8
360
361struct rds_znotifier {
362 struct mmpin z_mmp;
363 u32 z_cookie;
364};
365
366struct rds_msg_zcopy_info {
367 struct list_head rs_zcookie_next;
368 union {
369 struct rds_znotifier znotif;
370 struct rds_zcopy_cookies zcookies;
371 };
372};
373
374struct rds_msg_zcopy_queue {
375 struct list_head zcookie_head;
376 spinlock_t lock;
377};
378
379static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
380{
381 spin_lock_init(&q->lock);
382 INIT_LIST_HEAD(&q->zcookie_head);
383}
384
385struct rds_iov_vector {
386 struct rds_iovec *iov;
387 int len;
388};
389
390struct rds_iov_vector_arr {
391 struct rds_iov_vector *vec;
392 int len;
393 int indx;
394 int incr;
395};
396
397struct rds_message {
398 refcount_t m_refcount;
399 struct list_head m_sock_item;
400 struct list_head m_conn_item;
401 struct rds_incoming m_inc;
402 u64 m_ack_seq;
403 struct in6_addr m_daddr;
404 unsigned long m_flags;
405
406
407
408
409
410
411 spinlock_t m_rs_lock;
412 wait_queue_head_t m_flush_wait;
413
414 struct rds_sock *m_rs;
415
416
417 rds_rdma_cookie_t m_rdma_cookie;
418
419 unsigned int m_used_sgs;
420 unsigned int m_total_sgs;
421
422 void *m_final_op;
423
424 struct {
425 struct rm_atomic_op {
426 int op_type;
427 union {
428 struct {
429 uint64_t compare;
430 uint64_t swap;
431 uint64_t compare_mask;
432 uint64_t swap_mask;
433 } op_m_cswp;
434 struct {
435 uint64_t add;
436 uint64_t nocarry_mask;
437 } op_m_fadd;
438 };
439
440 u32 op_rkey;
441 u64 op_remote_addr;
442 unsigned int op_notify:1;
443 unsigned int op_recverr:1;
444 unsigned int op_mapped:1;
445 unsigned int op_silent:1;
446 unsigned int op_active:1;
447 struct scatterlist *op_sg;
448 struct rds_notifier *op_notifier;
449
450 struct rds_mr *op_rdma_mr;
451 } atomic;
452 struct rm_rdma_op {
453 u32 op_rkey;
454 u64 op_remote_addr;
455 unsigned int op_write:1;
456 unsigned int op_fence:1;
457 unsigned int op_notify:1;
458 unsigned int op_recverr:1;
459 unsigned int op_mapped:1;
460 unsigned int op_silent:1;
461 unsigned int op_active:1;
462 unsigned int op_bytes;
463 unsigned int op_nents;
464 unsigned int op_count;
465 struct scatterlist *op_sg;
466 struct rds_notifier *op_notifier;
467
468 struct rds_mr *op_rdma_mr;
469 } rdma;
470 struct rm_data_op {
471 unsigned int op_active:1;
472 unsigned int op_notify:1;
473 unsigned int op_nents;
474 unsigned int op_count;
475 unsigned int op_dmasg;
476 unsigned int op_dmaoff;
477 struct rds_znotifier *op_mmp_znotifier;
478 struct scatterlist *op_sg;
479 } data;
480 };
481
482 struct rds_conn_path *m_conn_path;
483};
484
485
486
487
488
489
490
491
492struct rds_notifier {
493 struct list_head n_list;
494 uint64_t n_user_token;
495 int n_status;
496};
497
498
499
500
501#define RDS_TRANS_LOOP 3
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530struct rds_transport {
531 char t_name[TRANSNAMSIZ];
532 struct list_head t_item;
533 struct module *t_owner;
534 unsigned int t_prefer_loopback:1,
535 t_mp_capable:1;
536 unsigned int t_type;
537
538 int (*laddr_check)(struct net *net, const struct in6_addr *addr,
539 __u32 scope_id);
540 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
541 void (*conn_free)(void *data);
542 int (*conn_path_connect)(struct rds_conn_path *cp);
543 void (*conn_path_shutdown)(struct rds_conn_path *conn);
544 void (*xmit_path_prepare)(struct rds_conn_path *cp);
545 void (*xmit_path_complete)(struct rds_conn_path *cp);
546 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
547 unsigned int hdr_off, unsigned int sg, unsigned int off);
548 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
549 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
550 int (*recv_path)(struct rds_conn_path *cp);
551 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
552 void (*inc_free)(struct rds_incoming *inc);
553
554 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
555 struct rdma_cm_event *event, bool isv6);
556 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id, bool isv6);
557 void (*cm_connect_complete)(struct rds_connection *conn,
558 struct rdma_cm_event *event);
559
560 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
561 unsigned int avail);
562 void (*exit)(void);
563 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
564 struct rds_sock *rs, u32 *key_ret,
565 struct rds_connection *conn);
566 void (*sync_mr)(void *trans_private, int direction);
567 void (*free_mr)(void *trans_private, int invalidate);
568 void (*flush_mrs)(void);
569 bool (*t_unloading)(struct rds_connection *conn);
570};
571
572
573
574
575#define RDS_BOUND_KEY_LEN \
576 (sizeof(struct in6_addr) + sizeof(__u32) + sizeof(__be16))
577
578struct rds_sock {
579 struct sock rs_sk;
580
581 u64 rs_user_addr;
582 u64 rs_user_bytes;
583
584
585
586
587
588 struct rhash_head rs_bound_node;
589 u8 rs_bound_key[RDS_BOUND_KEY_LEN];
590 struct sockaddr_in6 rs_bound_sin6;
591#define rs_bound_addr rs_bound_sin6.sin6_addr
592#define rs_bound_addr_v4 rs_bound_sin6.sin6_addr.s6_addr32[3]
593#define rs_bound_port rs_bound_sin6.sin6_port
594#define rs_bound_scope_id rs_bound_sin6.sin6_scope_id
595 struct in6_addr rs_conn_addr;
596#define rs_conn_addr_v4 rs_conn_addr.s6_addr32[3]
597 __be16 rs_conn_port;
598 struct rds_transport *rs_transport;
599
600
601
602
603
604 struct rds_connection *rs_conn;
605
606
607 int rs_congested;
608
609 int rs_seen_congestion;
610
611
612 spinlock_t rs_lock;
613 struct list_head rs_send_queue;
614 u32 rs_snd_bytes;
615 int rs_rcv_bytes;
616 struct list_head rs_notify_queue;
617
618
619
620
621
622
623 uint64_t rs_cong_mask;
624 uint64_t rs_cong_notify;
625 struct list_head rs_cong_list;
626 unsigned long rs_cong_track;
627
628
629
630
631
632 rwlock_t rs_recv_lock;
633 struct list_head rs_recv_queue;
634
635
636 struct list_head rs_item;
637
638
639 spinlock_t rs_rdma_lock;
640 struct rb_root rs_rdma_keys;
641
642
643 unsigned char rs_recverr,
644 rs_cong_monitor;
645 u32 rs_hash_initval;
646
647
648 u8 rs_rx_traces;
649 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
650 struct rds_msg_zcopy_queue rs_zcookie_queue;
651};
652
653static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
654{
655 return container_of(sk, struct rds_sock, rs_sk);
656}
657static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
658{
659 return &rs->rs_sk;
660}
661
662
663
664
665
666
667static inline int rds_sk_sndbuf(struct rds_sock *rs)
668{
669 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
670}
671static inline int rds_sk_rcvbuf(struct rds_sock *rs)
672{
673 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
674}
675
676struct rds_statistics {
677 uint64_t s_conn_reset;
678 uint64_t s_recv_drop_bad_checksum;
679 uint64_t s_recv_drop_old_seq;
680 uint64_t s_recv_drop_no_sock;
681 uint64_t s_recv_drop_dead_sock;
682 uint64_t s_recv_deliver_raced;
683 uint64_t s_recv_delivered;
684 uint64_t s_recv_queued;
685 uint64_t s_recv_immediate_retry;
686 uint64_t s_recv_delayed_retry;
687 uint64_t s_recv_ack_required;
688 uint64_t s_recv_rdma_bytes;
689 uint64_t s_recv_ping;
690 uint64_t s_send_queue_empty;
691 uint64_t s_send_queue_full;
692 uint64_t s_send_lock_contention;
693 uint64_t s_send_lock_queue_raced;
694 uint64_t s_send_immediate_retry;
695 uint64_t s_send_delayed_retry;
696 uint64_t s_send_drop_acked;
697 uint64_t s_send_ack_required;
698 uint64_t s_send_queued;
699 uint64_t s_send_rdma;
700 uint64_t s_send_rdma_bytes;
701 uint64_t s_send_pong;
702 uint64_t s_page_remainder_hit;
703 uint64_t s_page_remainder_miss;
704 uint64_t s_copy_to_user;
705 uint64_t s_copy_from_user;
706 uint64_t s_cong_update_queued;
707 uint64_t s_cong_update_received;
708 uint64_t s_cong_send_error;
709 uint64_t s_cong_send_blocked;
710 uint64_t s_recv_bytes_added_to_socket;
711 uint64_t s_recv_bytes_removed_from_socket;
712
713};
714
715
716void rds_sock_addref(struct rds_sock *rs);
717void rds_sock_put(struct rds_sock *rs);
718void rds_wake_sk_sleep(struct rds_sock *rs);
719static inline void __rds_wake_sk_sleep(struct sock *sk)
720{
721 wait_queue_head_t *waitq = sk_sleep(sk);
722
723 if (!sock_flag(sk, SOCK_DEAD) && waitq)
724 wake_up(waitq);
725}
726extern wait_queue_head_t rds_poll_waitq;
727
728
729
730int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
731void rds_remove_bound(struct rds_sock *rs);
732struct rds_sock *rds_find_bound(const struct in6_addr *addr, __be16 port,
733 __u32 scope_id);
734int rds_bind_lock_init(void);
735void rds_bind_lock_destroy(void);
736
737
738int rds_cong_get_maps(struct rds_connection *conn);
739void rds_cong_add_conn(struct rds_connection *conn);
740void rds_cong_remove_conn(struct rds_connection *conn);
741void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
742void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
743int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
744void rds_cong_queue_updates(struct rds_cong_map *map);
745void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
746int rds_cong_updated_since(unsigned long *recent);
747void rds_cong_add_socket(struct rds_sock *);
748void rds_cong_remove_socket(struct rds_sock *);
749void rds_cong_exit(void);
750struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
751
752
753extern u32 rds_gen_num;
754int rds_conn_init(void);
755void rds_conn_exit(void);
756struct rds_connection *rds_conn_create(struct net *net,
757 const struct in6_addr *laddr,
758 const struct in6_addr *faddr,
759 struct rds_transport *trans, gfp_t gfp,
760 int dev_if);
761struct rds_connection *rds_conn_create_outgoing(struct net *net,
762 const struct in6_addr *laddr,
763 const struct in6_addr *faddr,
764 struct rds_transport *trans,
765 gfp_t gfp, int dev_if);
766void rds_conn_shutdown(struct rds_conn_path *cpath);
767void rds_conn_destroy(struct rds_connection *conn);
768void rds_conn_drop(struct rds_connection *conn);
769void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
770void rds_conn_connect_if_down(struct rds_connection *conn);
771void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
772void rds_for_each_conn_info(struct socket *sock, unsigned int len,
773 struct rds_info_iterator *iter,
774 struct rds_info_lengths *lens,
775 int (*visitor)(struct rds_connection *, void *),
776 u64 *buffer,
777 size_t item_len);
778
779__printf(2, 3)
780void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
781#define rds_conn_path_error(cp, fmt...) \
782 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
783
784static inline int
785rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
786{
787 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
788}
789
790static inline int
791rds_conn_transition(struct rds_connection *conn, int old, int new)
792{
793 WARN_ON(conn->c_trans->t_mp_capable);
794 return rds_conn_path_transition(&conn->c_path[0], old, new);
795}
796
797static inline int
798rds_conn_path_state(struct rds_conn_path *cp)
799{
800 return atomic_read(&cp->cp_state);
801}
802
803static inline int
804rds_conn_state(struct rds_connection *conn)
805{
806 WARN_ON(conn->c_trans->t_mp_capable);
807 return rds_conn_path_state(&conn->c_path[0]);
808}
809
810static inline int
811rds_conn_path_up(struct rds_conn_path *cp)
812{
813 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
814}
815
816static inline int
817rds_conn_up(struct rds_connection *conn)
818{
819 WARN_ON(conn->c_trans->t_mp_capable);
820 return rds_conn_path_up(&conn->c_path[0]);
821}
822
823static inline int
824rds_conn_path_connecting(struct rds_conn_path *cp)
825{
826 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
827}
828
829static inline int
830rds_conn_connecting(struct rds_connection *conn)
831{
832 WARN_ON(conn->c_trans->t_mp_capable);
833 return rds_conn_path_connecting(&conn->c_path[0]);
834}
835
836
837struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
838struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents,
839 int *ret);
840int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
841 bool zcopy);
842struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
843void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
844 __be16 dport, u64 seq);
845int rds_message_add_extension(struct rds_header *hdr,
846 unsigned int type, const void *data, unsigned int len);
847int rds_message_next_extension(struct rds_header *hdr,
848 unsigned int *pos, void *buf, unsigned int *buflen);
849int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
850int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
851void rds_message_inc_free(struct rds_incoming *inc);
852void rds_message_addref(struct rds_message *rm);
853void rds_message_put(struct rds_message *rm);
854void rds_message_wait(struct rds_message *rm);
855void rds_message_unmapped(struct rds_message *rm);
856void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
857
858static inline void rds_message_make_checksum(struct rds_header *hdr)
859{
860 hdr->h_csum = 0;
861 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
862}
863
864static inline int rds_message_verify_checksum(const struct rds_header *hdr)
865{
866 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
867}
868
869
870
871int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
872 gfp_t gfp);
873void rds_page_exit(void);
874
875
876void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
877 struct in6_addr *saddr);
878void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
879 struct in6_addr *saddr);
880void rds_inc_put(struct rds_incoming *inc);
881void rds_recv_incoming(struct rds_connection *conn, struct in6_addr *saddr,
882 struct in6_addr *daddr,
883 struct rds_incoming *inc, gfp_t gfp);
884int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
885 int msg_flags);
886void rds_clear_recv_queue(struct rds_sock *rs);
887int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
888void rds_inc_info_copy(struct rds_incoming *inc,
889 struct rds_info_iterator *iter,
890 __be32 saddr, __be32 daddr, int flip);
891void rds6_inc_info_copy(struct rds_incoming *inc,
892 struct rds_info_iterator *iter,
893 struct in6_addr *saddr, struct in6_addr *daddr,
894 int flip);
895
896
897int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
898void rds_send_path_reset(struct rds_conn_path *conn);
899int rds_send_xmit(struct rds_conn_path *cp);
900struct sockaddr_in;
901void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest);
902typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
903void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
904 is_acked_func is_acked);
905void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
906 is_acked_func is_acked);
907void rds_send_ping(struct rds_connection *conn, int cp_index);
908int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
909
910
911void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
912int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
913int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
914int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
915void rds_rdma_drop_keys(struct rds_sock *rs);
916int rds_rdma_extra_size(struct rds_rdma_args *args,
917 struct rds_iov_vector *iov);
918int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
919 struct cmsghdr *cmsg);
920int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
921 struct cmsghdr *cmsg,
922 struct rds_iov_vector *vec);
923int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
924 struct cmsghdr *cmsg);
925void rds_rdma_free_op(struct rm_rdma_op *ro);
926void rds_atomic_free_op(struct rm_atomic_op *ao);
927void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
928void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
929int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
930 struct cmsghdr *cmsg);
931
932void __rds_put_mr_final(struct rds_mr *mr);
933static inline void rds_mr_put(struct rds_mr *mr)
934{
935 if (refcount_dec_and_test(&mr->r_refcount))
936 __rds_put_mr_final(mr);
937}
938
939static inline bool rds_destroy_pending(struct rds_connection *conn)
940{
941 return !check_net(rds_conn_net(conn)) ||
942 (conn->c_trans->t_unloading && conn->c_trans->t_unloading(conn));
943}
944
945
946DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
947#define rds_stats_inc_which(which, member) do { \
948 per_cpu(which, get_cpu()).member++; \
949 put_cpu(); \
950} while (0)
951#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
952#define rds_stats_add_which(which, member, count) do { \
953 per_cpu(which, get_cpu()).member += count; \
954 put_cpu(); \
955} while (0)
956#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
957int rds_stats_init(void);
958void rds_stats_exit(void);
959void rds_stats_info_copy(struct rds_info_iterator *iter,
960 uint64_t *values, const char *const *names,
961 size_t nr);
962
963
964int rds_sysctl_init(void);
965void rds_sysctl_exit(void);
966extern unsigned long rds_sysctl_sndbuf_min;
967extern unsigned long rds_sysctl_sndbuf_default;
968extern unsigned long rds_sysctl_sndbuf_max;
969extern unsigned long rds_sysctl_reconnect_min_jiffies;
970extern unsigned long rds_sysctl_reconnect_max_jiffies;
971extern unsigned int rds_sysctl_max_unacked_packets;
972extern unsigned int rds_sysctl_max_unacked_bytes;
973extern unsigned int rds_sysctl_ping_enable;
974extern unsigned long rds_sysctl_trace_flags;
975extern unsigned int rds_sysctl_trace_level;
976
977
978int rds_threads_init(void);
979void rds_threads_exit(void);
980extern struct workqueue_struct *rds_wq;
981void rds_queue_reconnect(struct rds_conn_path *cp);
982void rds_connect_worker(struct work_struct *);
983void rds_shutdown_worker(struct work_struct *);
984void rds_send_worker(struct work_struct *);
985void rds_recv_worker(struct work_struct *);
986void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
987void rds_connect_complete(struct rds_connection *conn);
988int rds_addr_cmp(const struct in6_addr *a1, const struct in6_addr *a2);
989
990
991void rds_trans_register(struct rds_transport *trans);
992void rds_trans_unregister(struct rds_transport *trans);
993struct rds_transport *rds_trans_get_preferred(struct net *net,
994 const struct in6_addr *addr,
995 __u32 scope_id);
996void rds_trans_put(struct rds_transport *trans);
997unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
998 unsigned int avail);
999struct rds_transport *rds_trans_get(int t_type);
1000int rds_trans_init(void);
1001void rds_trans_exit(void);
1002
1003#endif
1004