1
2#ifndef _RDS_RDS_H
3#define _RDS_RDS_H
4
5#include <net/sock.h>
6#include <linux/scatterlist.h>
7#include <linux/highmem.h>
8#include <rdma/rdma_cm.h>
9#include <linux/mutex.h>
10#include <linux/rds.h>
11#include <linux/rhashtable.h>
12#include <linux/refcount.h>
13#include <linux/in6.h>
14
15#include "info.h"
16
17
18
19
20#define RDS_PROTOCOL_3_0 0x0300
21#define RDS_PROTOCOL_3_1 0x0301
22#define RDS_PROTOCOL_4_0 0x0400
23#define RDS_PROTOCOL_4_1 0x0401
24#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
25#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
26#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
27#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
28#define RDS_PROTOCOL_COMPAT_VERSION RDS_PROTOCOL_3_1
29
30
31
32
33
34
35
36
37
38#define RDS_PORT 18634
39
40#ifdef ATOMIC64_INIT
41#define KERNEL_HAS_ATOMIC64
42#endif
43
44#ifdef RDS_DEBUG
45#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
46#else
47
48static inline __printf(1, 2)
49void rdsdebug(char *fmt, ...)
50{
51}
52#endif
53
54#define RDS_FRAG_SHIFT 12
55#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
56
57
58#define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
59
60#define RDS_CONG_MAP_BYTES (65536 / 8)
61#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
62#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
63
64struct rds_cong_map {
65 struct rb_node m_rb_node;
66 struct in6_addr m_addr;
67 wait_queue_head_t m_waitq;
68 struct list_head m_conn_list;
69 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
70};
71
72
73
74
75
76
77
78
79enum {
80 RDS_CONN_DOWN = 0,
81 RDS_CONN_CONNECTING,
82 RDS_CONN_DISCONNECTING,
83 RDS_CONN_UP,
84 RDS_CONN_RESETTING,
85 RDS_CONN_ERROR,
86};
87
88
89#define RDS_LL_SEND_FULL 0
90#define RDS_RECONNECT_PENDING 1
91#define RDS_IN_XMIT 2
92#define RDS_RECV_REFILL 3
93#define RDS_DESTROY_PENDING 4
94
95
96#define RDS_MPATH_WORKERS 8
97#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
98 (rs)->rs_hash_initval) & ((n) - 1))
99
100#define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
101
102
103struct rds_conn_path {
104 struct rds_connection *cp_conn;
105 struct rds_message *cp_xmit_rm;
106 unsigned long cp_xmit_sg;
107 unsigned int cp_xmit_hdr_off;
108 unsigned int cp_xmit_data_off;
109 unsigned int cp_xmit_atomic_sent;
110 unsigned int cp_xmit_rdma_sent;
111 unsigned int cp_xmit_data_sent;
112
113 spinlock_t cp_lock;
114 u64 cp_next_tx_seq;
115 struct list_head cp_send_queue;
116 struct list_head cp_retrans;
117
118 u64 cp_next_rx_seq;
119
120 void *cp_transport_data;
121
122 atomic_t cp_state;
123 unsigned long cp_send_gen;
124 unsigned long cp_flags;
125 unsigned long cp_reconnect_jiffies;
126 struct delayed_work cp_send_w;
127 struct delayed_work cp_recv_w;
128 struct delayed_work cp_conn_w;
129 struct work_struct cp_down_w;
130 struct mutex cp_cm_lock;
131 wait_queue_head_t cp_waitq;
132
133 unsigned int cp_unacked_packets;
134 unsigned int cp_unacked_bytes;
135 unsigned int cp_index;
136};
137
138
139struct rds_connection {
140 struct hlist_node c_hash_node;
141 struct in6_addr c_laddr;
142 struct in6_addr c_faddr;
143 int c_dev_if;
144 int c_bound_if;
145 unsigned int c_loopback:1,
146 c_isv6:1,
147 c_ping_triggered:1,
148 c_pad_to_32:29;
149 int c_npaths;
150 struct rds_connection *c_passive;
151 struct rds_transport *c_trans;
152
153 struct rds_cong_map *c_lcong;
154 struct rds_cong_map *c_fcong;
155
156
157 unsigned int c_proposed_version;
158 unsigned int c_version;
159 possible_net_t c_net;
160
161
162 u8 c_tos;
163
164 struct list_head c_map_item;
165 unsigned long c_map_queued;
166
167 struct rds_conn_path *c_path;
168 wait_queue_head_t c_hs_waitq;
169
170 u32 c_my_gen_num;
171 u32 c_peer_gen_num;
172};
173
174static inline
175struct net *rds_conn_net(struct rds_connection *conn)
176{
177 return read_pnet(&conn->c_net);
178}
179
180static inline
181void rds_conn_net_set(struct rds_connection *conn, struct net *net)
182{
183 write_pnet(&conn->c_net, net);
184}
185
186#define RDS_FLAG_CONG_BITMAP 0x01
187#define RDS_FLAG_ACK_REQUIRED 0x02
188#define RDS_FLAG_RETRANSMITTED 0x04
189#define RDS_MAX_ADV_CREDIT 255
190
191
192
193
194
195
196
197
198#define RDS_FLAG_PROBE_PORT 1
199#define RDS_HS_PROBE(sport, dport) \
200 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
201 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
202
203
204
205#define RDS_HEADER_EXT_SPACE 16
206
207struct rds_header {
208 __be64 h_sequence;
209 __be64 h_ack;
210 __be32 h_len;
211 __be16 h_sport;
212 __be16 h_dport;
213 u8 h_flags;
214 u8 h_credit;
215 u8 h_padding[4];
216 __sum16 h_csum;
217
218 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
219};
220
221
222
223
224#define RDS_EXTHDR_NONE 0
225
226
227
228
229
230
231
232
233
234
235
236#define RDS_EXTHDR_VERSION 1
237struct rds_ext_header_version {
238 __be32 h_version;
239};
240
241
242
243
244
245#define RDS_EXTHDR_RDMA 2
246struct rds_ext_header_rdma {
247 __be32 h_rdma_rkey;
248};
249
250
251
252
253
254
255#define RDS_EXTHDR_RDMA_DEST 3
256struct rds_ext_header_rdma_dest {
257 __be32 h_rdma_rkey;
258 __be32 h_rdma_offset;
259};
260
261
262
263
264#define RDS_EXTHDR_NPATHS 5
265#define RDS_EXTHDR_GEN_NUM 6
266
267#define __RDS_EXTHDR_MAX 16
268#define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
269#define RDS_MSG_RX_HDR 0
270#define RDS_MSG_RX_START 1
271#define RDS_MSG_RX_END 2
272#define RDS_MSG_RX_CMSG 3
273
274struct rds_incoming {
275 refcount_t i_refcount;
276 struct list_head i_item;
277 struct rds_connection *i_conn;
278 struct rds_conn_path *i_conn_path;
279 struct rds_header i_hdr;
280 unsigned long i_rx_jiffies;
281 struct in6_addr i_saddr;
282
283 rds_rdma_cookie_t i_rdma_cookie;
284 ktime_t i_rx_tstamp;
285 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
286};
287
288struct rds_mr {
289 struct rb_node r_rb_node;
290 refcount_t r_refcount;
291 u32 r_key;
292
293
294 unsigned int r_use_once:1;
295 unsigned int r_invalidate:1;
296 unsigned int r_write:1;
297
298
299
300
301
302 unsigned long r_state;
303 struct rds_sock *r_sock;
304 struct rds_transport *r_trans;
305 void *r_trans_private;
306};
307
308
309#define RDS_MR_DEAD 0
310
311static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
312{
313 return r_key | (((u64) offset) << 32);
314}
315
316static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
317{
318 return cookie;
319}
320
321static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
322{
323 return cookie >> 32;
324}
325
326
327#define RDS_ATOMIC_TYPE_CSWP 0
328#define RDS_ATOMIC_TYPE_FADD 1
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359#define RDS_MSG_ON_SOCK 1
360#define RDS_MSG_ON_CONN 2
361#define RDS_MSG_HAS_ACK_SEQ 3
362#define RDS_MSG_ACK_REQUIRED 4
363#define RDS_MSG_RETRANSMITTED 5
364#define RDS_MSG_MAPPED 6
365#define RDS_MSG_PAGEVEC 7
366#define RDS_MSG_FLUSH 8
367
368struct rds_znotifier {
369 struct mmpin z_mmp;
370 u32 z_cookie;
371};
372
373struct rds_msg_zcopy_info {
374 struct list_head rs_zcookie_next;
375 union {
376 struct rds_znotifier znotif;
377 struct rds_zcopy_cookies zcookies;
378 };
379};
380
381struct rds_msg_zcopy_queue {
382 struct list_head zcookie_head;
383 spinlock_t lock;
384};
385
386static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
387{
388 spin_lock_init(&q->lock);
389 INIT_LIST_HEAD(&q->zcookie_head);
390}
391
392struct rds_iov_vector {
393 struct rds_iovec *iov;
394 int len;
395};
396
397struct rds_iov_vector_arr {
398 struct rds_iov_vector *vec;
399 int len;
400 int indx;
401 int incr;
402};
403
404struct rds_message {
405 refcount_t m_refcount;
406 struct list_head m_sock_item;
407 struct list_head m_conn_item;
408 struct rds_incoming m_inc;
409 u64 m_ack_seq;
410 struct in6_addr m_daddr;
411 unsigned long m_flags;
412
413
414
415
416
417
418 spinlock_t m_rs_lock;
419 wait_queue_head_t m_flush_wait;
420
421 struct rds_sock *m_rs;
422
423
424 rds_rdma_cookie_t m_rdma_cookie;
425
426 unsigned int m_used_sgs;
427 unsigned int m_total_sgs;
428
429 void *m_final_op;
430
431 struct {
432 struct rm_atomic_op {
433 int op_type;
434 union {
435 struct {
436 uint64_t compare;
437 uint64_t swap;
438 uint64_t compare_mask;
439 uint64_t swap_mask;
440 } op_m_cswp;
441 struct {
442 uint64_t add;
443 uint64_t nocarry_mask;
444 } op_m_fadd;
445 };
446
447 u32 op_rkey;
448 u64 op_remote_addr;
449 unsigned int op_notify:1;
450 unsigned int op_recverr:1;
451 unsigned int op_mapped:1;
452 unsigned int op_silent:1;
453 unsigned int op_active:1;
454 struct scatterlist *op_sg;
455 struct rds_notifier *op_notifier;
456
457 struct rds_mr *op_rdma_mr;
458 } atomic;
459 struct rm_rdma_op {
460 u32 op_rkey;
461 u64 op_remote_addr;
462 unsigned int op_write:1;
463 unsigned int op_fence:1;
464 unsigned int op_notify:1;
465 unsigned int op_recverr:1;
466 unsigned int op_mapped:1;
467 unsigned int op_silent:1;
468 unsigned int op_active:1;
469 unsigned int op_bytes;
470 unsigned int op_nents;
471 unsigned int op_count;
472 struct scatterlist *op_sg;
473 struct rds_notifier *op_notifier;
474
475 struct rds_mr *op_rdma_mr;
476 } rdma;
477 struct rm_data_op {
478 unsigned int op_active:1;
479 unsigned int op_notify:1;
480 unsigned int op_nents;
481 unsigned int op_count;
482 unsigned int op_dmasg;
483 unsigned int op_dmaoff;
484 struct rds_znotifier *op_mmp_znotifier;
485 struct scatterlist *op_sg;
486 } data;
487 };
488
489 struct rds_conn_path *m_conn_path;
490};
491
492
493
494
495
496
497
498
499struct rds_notifier {
500 struct list_head n_list;
501 uint64_t n_user_token;
502 int n_status;
503};
504
505
506
507
508#define RDS_TRANS_LOOP 3
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537struct rds_transport {
538 char t_name[TRANSNAMSIZ];
539 struct list_head t_item;
540 struct module *t_owner;
541 unsigned int t_prefer_loopback:1,
542 t_mp_capable:1;
543 unsigned int t_type;
544
545 int (*laddr_check)(struct net *net, const struct in6_addr *addr,
546 __u32 scope_id);
547 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
548 void (*conn_free)(void *data);
549 int (*conn_path_connect)(struct rds_conn_path *cp);
550 void (*conn_path_shutdown)(struct rds_conn_path *conn);
551 void (*xmit_path_prepare)(struct rds_conn_path *cp);
552 void (*xmit_path_complete)(struct rds_conn_path *cp);
553 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
554 unsigned int hdr_off, unsigned int sg, unsigned int off);
555 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
556 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
557 int (*recv_path)(struct rds_conn_path *cp);
558 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
559 void (*inc_free)(struct rds_incoming *inc);
560
561 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
562 struct rdma_cm_event *event, bool isv6);
563 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id, bool isv6);
564 void (*cm_connect_complete)(struct rds_connection *conn,
565 struct rdma_cm_event *event);
566
567 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
568 unsigned int avail);
569 void (*exit)(void);
570 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
571 struct rds_sock *rs, u32 *key_ret,
572 struct rds_connection *conn);
573 void (*sync_mr)(void *trans_private, int direction);
574 void (*free_mr)(void *trans_private, int invalidate);
575 void (*flush_mrs)(void);
576 bool (*t_unloading)(struct rds_connection *conn);
577 u8 (*get_tos_map)(u8 tos);
578};
579
580
581
582
583#define RDS_BOUND_KEY_LEN \
584 (sizeof(struct in6_addr) + sizeof(__u32) + sizeof(__be16))
585
586struct rds_sock {
587 struct sock rs_sk;
588
589 u64 rs_user_addr;
590 u64 rs_user_bytes;
591
592
593
594
595
596 struct rhash_head rs_bound_node;
597 u8 rs_bound_key[RDS_BOUND_KEY_LEN];
598 struct sockaddr_in6 rs_bound_sin6;
599#define rs_bound_addr rs_bound_sin6.sin6_addr
600#define rs_bound_addr_v4 rs_bound_sin6.sin6_addr.s6_addr32[3]
601#define rs_bound_port rs_bound_sin6.sin6_port
602#define rs_bound_scope_id rs_bound_sin6.sin6_scope_id
603 struct in6_addr rs_conn_addr;
604#define rs_conn_addr_v4 rs_conn_addr.s6_addr32[3]
605 __be16 rs_conn_port;
606 struct rds_transport *rs_transport;
607
608
609
610
611
612 struct rds_connection *rs_conn;
613
614
615 int rs_congested;
616
617 int rs_seen_congestion;
618
619
620 spinlock_t rs_lock;
621 struct list_head rs_send_queue;
622 u32 rs_snd_bytes;
623 int rs_rcv_bytes;
624 struct list_head rs_notify_queue;
625
626
627
628
629
630
631 uint64_t rs_cong_mask;
632 uint64_t rs_cong_notify;
633 struct list_head rs_cong_list;
634 unsigned long rs_cong_track;
635
636
637
638
639
640 rwlock_t rs_recv_lock;
641 struct list_head rs_recv_queue;
642
643
644 struct list_head rs_item;
645
646
647 spinlock_t rs_rdma_lock;
648 struct rb_root rs_rdma_keys;
649
650
651 unsigned char rs_recverr,
652 rs_cong_monitor;
653 u32 rs_hash_initval;
654
655
656 u8 rs_rx_traces;
657 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
658 struct rds_msg_zcopy_queue rs_zcookie_queue;
659 u8 rs_tos;
660};
661
662static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
663{
664 return container_of(sk, struct rds_sock, rs_sk);
665}
666static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
667{
668 return &rs->rs_sk;
669}
670
671
672
673
674
675
676static inline int rds_sk_sndbuf(struct rds_sock *rs)
677{
678 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
679}
680static inline int rds_sk_rcvbuf(struct rds_sock *rs)
681{
682 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
683}
684
685struct rds_statistics {
686 uint64_t s_conn_reset;
687 uint64_t s_recv_drop_bad_checksum;
688 uint64_t s_recv_drop_old_seq;
689 uint64_t s_recv_drop_no_sock;
690 uint64_t s_recv_drop_dead_sock;
691 uint64_t s_recv_deliver_raced;
692 uint64_t s_recv_delivered;
693 uint64_t s_recv_queued;
694 uint64_t s_recv_immediate_retry;
695 uint64_t s_recv_delayed_retry;
696 uint64_t s_recv_ack_required;
697 uint64_t s_recv_rdma_bytes;
698 uint64_t s_recv_ping;
699 uint64_t s_send_queue_empty;
700 uint64_t s_send_queue_full;
701 uint64_t s_send_lock_contention;
702 uint64_t s_send_lock_queue_raced;
703 uint64_t s_send_immediate_retry;
704 uint64_t s_send_delayed_retry;
705 uint64_t s_send_drop_acked;
706 uint64_t s_send_ack_required;
707 uint64_t s_send_queued;
708 uint64_t s_send_rdma;
709 uint64_t s_send_rdma_bytes;
710 uint64_t s_send_pong;
711 uint64_t s_page_remainder_hit;
712 uint64_t s_page_remainder_miss;
713 uint64_t s_copy_to_user;
714 uint64_t s_copy_from_user;
715 uint64_t s_cong_update_queued;
716 uint64_t s_cong_update_received;
717 uint64_t s_cong_send_error;
718 uint64_t s_cong_send_blocked;
719 uint64_t s_recv_bytes_added_to_socket;
720 uint64_t s_recv_bytes_removed_from_socket;
721
722};
723
724
725void rds_sock_addref(struct rds_sock *rs);
726void rds_sock_put(struct rds_sock *rs);
727void rds_wake_sk_sleep(struct rds_sock *rs);
728static inline void __rds_wake_sk_sleep(struct sock *sk)
729{
730 wait_queue_head_t *waitq = sk_sleep(sk);
731
732 if (!sock_flag(sk, SOCK_DEAD) && waitq)
733 wake_up(waitq);
734}
735extern wait_queue_head_t rds_poll_waitq;
736
737
738
739int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
740void rds_remove_bound(struct rds_sock *rs);
741struct rds_sock *rds_find_bound(const struct in6_addr *addr, __be16 port,
742 __u32 scope_id);
743int rds_bind_lock_init(void);
744void rds_bind_lock_destroy(void);
745
746
747int rds_cong_get_maps(struct rds_connection *conn);
748void rds_cong_add_conn(struct rds_connection *conn);
749void rds_cong_remove_conn(struct rds_connection *conn);
750void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
751void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
752int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
753void rds_cong_queue_updates(struct rds_cong_map *map);
754void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
755int rds_cong_updated_since(unsigned long *recent);
756void rds_cong_add_socket(struct rds_sock *);
757void rds_cong_remove_socket(struct rds_sock *);
758void rds_cong_exit(void);
759struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
760
761
762extern u32 rds_gen_num;
763int rds_conn_init(void);
764void rds_conn_exit(void);
765struct rds_connection *rds_conn_create(struct net *net,
766 const struct in6_addr *laddr,
767 const struct in6_addr *faddr,
768 struct rds_transport *trans,
769 u8 tos, gfp_t gfp,
770 int dev_if);
771struct rds_connection *rds_conn_create_outgoing(struct net *net,
772 const struct in6_addr *laddr,
773 const struct in6_addr *faddr,
774 struct rds_transport *trans,
775 u8 tos, gfp_t gfp, int dev_if);
776void rds_conn_shutdown(struct rds_conn_path *cpath);
777void rds_conn_destroy(struct rds_connection *conn);
778void rds_conn_drop(struct rds_connection *conn);
779void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
780void rds_conn_connect_if_down(struct rds_connection *conn);
781void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
782void rds_for_each_conn_info(struct socket *sock, unsigned int len,
783 struct rds_info_iterator *iter,
784 struct rds_info_lengths *lens,
785 int (*visitor)(struct rds_connection *, void *),
786 u64 *buffer,
787 size_t item_len);
788
789__printf(2, 3)
790void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
791#define rds_conn_path_error(cp, fmt...) \
792 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
793
794static inline int
795rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
796{
797 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
798}
799
800static inline int
801rds_conn_transition(struct rds_connection *conn, int old, int new)
802{
803 WARN_ON(conn->c_trans->t_mp_capable);
804 return rds_conn_path_transition(&conn->c_path[0], old, new);
805}
806
807static inline int
808rds_conn_path_state(struct rds_conn_path *cp)
809{
810 return atomic_read(&cp->cp_state);
811}
812
813static inline int
814rds_conn_state(struct rds_connection *conn)
815{
816 WARN_ON(conn->c_trans->t_mp_capable);
817 return rds_conn_path_state(&conn->c_path[0]);
818}
819
820static inline int
821rds_conn_path_up(struct rds_conn_path *cp)
822{
823 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
824}
825
826static inline int
827rds_conn_up(struct rds_connection *conn)
828{
829 WARN_ON(conn->c_trans->t_mp_capable);
830 return rds_conn_path_up(&conn->c_path[0]);
831}
832
833static inline int
834rds_conn_path_connecting(struct rds_conn_path *cp)
835{
836 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
837}
838
839static inline int
840rds_conn_connecting(struct rds_connection *conn)
841{
842 WARN_ON(conn->c_trans->t_mp_capable);
843 return rds_conn_path_connecting(&conn->c_path[0]);
844}
845
846
847struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
848struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents,
849 int *ret);
850int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
851 bool zcopy);
852struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
853void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
854 __be16 dport, u64 seq);
855int rds_message_add_extension(struct rds_header *hdr,
856 unsigned int type, const void *data, unsigned int len);
857int rds_message_next_extension(struct rds_header *hdr,
858 unsigned int *pos, void *buf, unsigned int *buflen);
859int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
860int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
861void rds_message_inc_free(struct rds_incoming *inc);
862void rds_message_addref(struct rds_message *rm);
863void rds_message_put(struct rds_message *rm);
864void rds_message_wait(struct rds_message *rm);
865void rds_message_unmapped(struct rds_message *rm);
866void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
867
868static inline void rds_message_make_checksum(struct rds_header *hdr)
869{
870 hdr->h_csum = 0;
871 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
872}
873
874static inline int rds_message_verify_checksum(const struct rds_header *hdr)
875{
876 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
877}
878
879
880
881int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
882 gfp_t gfp);
883void rds_page_exit(void);
884
885
886void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
887 struct in6_addr *saddr);
888void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
889 struct in6_addr *saddr);
890void rds_inc_put(struct rds_incoming *inc);
891void rds_recv_incoming(struct rds_connection *conn, struct in6_addr *saddr,
892 struct in6_addr *daddr,
893 struct rds_incoming *inc, gfp_t gfp);
894int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
895 int msg_flags);
896void rds_clear_recv_queue(struct rds_sock *rs);
897int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
898void rds_inc_info_copy(struct rds_incoming *inc,
899 struct rds_info_iterator *iter,
900 __be32 saddr, __be32 daddr, int flip);
901void rds6_inc_info_copy(struct rds_incoming *inc,
902 struct rds_info_iterator *iter,
903 struct in6_addr *saddr, struct in6_addr *daddr,
904 int flip);
905
906
907int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
908void rds_send_path_reset(struct rds_conn_path *conn);
909int rds_send_xmit(struct rds_conn_path *cp);
910struct sockaddr_in;
911void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest);
912typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
913void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
914 is_acked_func is_acked);
915void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
916 is_acked_func is_acked);
917void rds_send_ping(struct rds_connection *conn, int cp_index);
918int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
919
920
921void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
922int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
923int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
924int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
925void rds_rdma_drop_keys(struct rds_sock *rs);
926int rds_rdma_extra_size(struct rds_rdma_args *args,
927 struct rds_iov_vector *iov);
928int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
929 struct cmsghdr *cmsg);
930int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
931 struct cmsghdr *cmsg,
932 struct rds_iov_vector *vec);
933int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
934 struct cmsghdr *cmsg);
935void rds_rdma_free_op(struct rm_rdma_op *ro);
936void rds_atomic_free_op(struct rm_atomic_op *ao);
937void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
938void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
939int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
940 struct cmsghdr *cmsg);
941
942void __rds_put_mr_final(struct rds_mr *mr);
943static inline void rds_mr_put(struct rds_mr *mr)
944{
945 if (refcount_dec_and_test(&mr->r_refcount))
946 __rds_put_mr_final(mr);
947}
948
949static inline bool rds_destroy_pending(struct rds_connection *conn)
950{
951 return !check_net(rds_conn_net(conn)) ||
952 (conn->c_trans->t_unloading && conn->c_trans->t_unloading(conn));
953}
954
955
956DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
957#define rds_stats_inc_which(which, member) do { \
958 per_cpu(which, get_cpu()).member++; \
959 put_cpu(); \
960} while (0)
961#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
962#define rds_stats_add_which(which, member, count) do { \
963 per_cpu(which, get_cpu()).member += count; \
964 put_cpu(); \
965} while (0)
966#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
967int rds_stats_init(void);
968void rds_stats_exit(void);
969void rds_stats_info_copy(struct rds_info_iterator *iter,
970 uint64_t *values, const char *const *names,
971 size_t nr);
972
973
974int rds_sysctl_init(void);
975void rds_sysctl_exit(void);
976extern unsigned long rds_sysctl_sndbuf_min;
977extern unsigned long rds_sysctl_sndbuf_default;
978extern unsigned long rds_sysctl_sndbuf_max;
979extern unsigned long rds_sysctl_reconnect_min_jiffies;
980extern unsigned long rds_sysctl_reconnect_max_jiffies;
981extern unsigned int rds_sysctl_max_unacked_packets;
982extern unsigned int rds_sysctl_max_unacked_bytes;
983extern unsigned int rds_sysctl_ping_enable;
984extern unsigned long rds_sysctl_trace_flags;
985extern unsigned int rds_sysctl_trace_level;
986
987
988int rds_threads_init(void);
989void rds_threads_exit(void);
990extern struct workqueue_struct *rds_wq;
991void rds_queue_reconnect(struct rds_conn_path *cp);
992void rds_connect_worker(struct work_struct *);
993void rds_shutdown_worker(struct work_struct *);
994void rds_send_worker(struct work_struct *);
995void rds_recv_worker(struct work_struct *);
996void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
997void rds_connect_complete(struct rds_connection *conn);
998int rds_addr_cmp(const struct in6_addr *a1, const struct in6_addr *a2);
999
1000
1001void rds_trans_register(struct rds_transport *trans);
1002void rds_trans_unregister(struct rds_transport *trans);
1003struct rds_transport *rds_trans_get_preferred(struct net *net,
1004 const struct in6_addr *addr,
1005 __u32 scope_id);
1006void rds_trans_put(struct rds_transport *trans);
1007unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
1008 unsigned int avail);
1009struct rds_transport *rds_trans_get(int t_type);
1010int rds_trans_init(void);
1011void rds_trans_exit(void);
1012
1013#endif
1014