1
2#ifndef _RDS_RDS_H
3#define _RDS_RDS_H
4
5#include <net/sock.h>
6#include <linux/scatterlist.h>
7#include <linux/highmem.h>
8#include <rdma/rdma_cm.h>
9#include <linux/mutex.h>
10#include <linux/rds.h>
11#include <linux/rhashtable.h>
12#include <linux/refcount.h>
13#include <linux/in6.h>
14
15#include "info.h"
16
17
18
19
20#define RDS_PROTOCOL_3_0 0x0300
21#define RDS_PROTOCOL_3_1 0x0301
22#define RDS_PROTOCOL_4_0 0x0400
23#define RDS_PROTOCOL_4_1 0x0401
24#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
25#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
26#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
27#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
28#define RDS_PROTOCOL_COMPAT_VERSION RDS_PROTOCOL_3_1
29
30
31
32
33
34
35
36
37
38#define RDS_PORT 18634
39
40#ifdef ATOMIC64_INIT
41#define KERNEL_HAS_ATOMIC64
42#endif
43
44#ifdef RDS_DEBUG
45#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
46#else
47
48static inline __printf(1, 2)
49void rdsdebug(char *fmt, ...)
50{
51}
52#endif
53
54#define RDS_FRAG_SHIFT 12
55#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
56
57
58#define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
59
60#define RDS_CONG_MAP_BYTES (65536 / 8)
61#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
62#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
63
64struct rds_cong_map {
65 struct rb_node m_rb_node;
66 struct in6_addr m_addr;
67 wait_queue_head_t m_waitq;
68 struct list_head m_conn_list;
69 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
70};
71
72
73
74
75
76
77
78
79enum {
80 RDS_CONN_DOWN = 0,
81 RDS_CONN_CONNECTING,
82 RDS_CONN_DISCONNECTING,
83 RDS_CONN_UP,
84 RDS_CONN_RESETTING,
85 RDS_CONN_ERROR,
86};
87
88
89#define RDS_LL_SEND_FULL 0
90#define RDS_RECONNECT_PENDING 1
91#define RDS_IN_XMIT 2
92#define RDS_RECV_REFILL 3
93#define RDS_DESTROY_PENDING 4
94
95
96#define RDS_MPATH_WORKERS 8
97#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
98 (rs)->rs_hash_initval) & ((n) - 1))
99
100#define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
101
102
103struct rds_conn_path {
104 struct rds_connection *cp_conn;
105 struct rds_message *cp_xmit_rm;
106 unsigned long cp_xmit_sg;
107 unsigned int cp_xmit_hdr_off;
108 unsigned int cp_xmit_data_off;
109 unsigned int cp_xmit_atomic_sent;
110 unsigned int cp_xmit_rdma_sent;
111 unsigned int cp_xmit_data_sent;
112
113 spinlock_t cp_lock;
114 u64 cp_next_tx_seq;
115 struct list_head cp_send_queue;
116 struct list_head cp_retrans;
117
118 u64 cp_next_rx_seq;
119
120 void *cp_transport_data;
121
122 atomic_t cp_state;
123 unsigned long cp_send_gen;
124 unsigned long cp_flags;
125 unsigned long cp_reconnect_jiffies;
126 struct delayed_work cp_send_w;
127 struct delayed_work cp_recv_w;
128 struct delayed_work cp_conn_w;
129 struct work_struct cp_down_w;
130 struct mutex cp_cm_lock;
131 wait_queue_head_t cp_waitq;
132
133 unsigned int cp_unacked_packets;
134 unsigned int cp_unacked_bytes;
135 unsigned int cp_index;
136};
137
138
139struct rds_connection {
140 struct hlist_node c_hash_node;
141 struct in6_addr c_laddr;
142 struct in6_addr c_faddr;
143 int c_dev_if;
144 int c_bound_if;
145 unsigned int c_loopback:1,
146 c_isv6:1,
147 c_ping_triggered:1,
148 c_pad_to_32:29;
149 int c_npaths;
150 struct rds_connection *c_passive;
151 struct rds_transport *c_trans;
152
153 struct rds_cong_map *c_lcong;
154 struct rds_cong_map *c_fcong;
155
156
157 unsigned int c_proposed_version;
158 unsigned int c_version;
159 possible_net_t c_net;
160
161
162 u8 c_tos;
163
164 struct list_head c_map_item;
165 unsigned long c_map_queued;
166
167 struct rds_conn_path *c_path;
168 wait_queue_head_t c_hs_waitq;
169
170 u32 c_my_gen_num;
171 u32 c_peer_gen_num;
172};
173
174static inline
175struct net *rds_conn_net(struct rds_connection *conn)
176{
177 return read_pnet(&conn->c_net);
178}
179
180static inline
181void rds_conn_net_set(struct rds_connection *conn, struct net *net)
182{
183 write_pnet(&conn->c_net, net);
184}
185
186#define RDS_FLAG_CONG_BITMAP 0x01
187#define RDS_FLAG_ACK_REQUIRED 0x02
188#define RDS_FLAG_RETRANSMITTED 0x04
189#define RDS_MAX_ADV_CREDIT 255
190
191
192
193
194
195
196
197
198#define RDS_FLAG_PROBE_PORT 1
199#define RDS_HS_PROBE(sport, dport) \
200 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
201 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
202
203
204
205#define RDS_HEADER_EXT_SPACE 16
206
207struct rds_header {
208 __be64 h_sequence;
209 __be64 h_ack;
210 __be32 h_len;
211 __be16 h_sport;
212 __be16 h_dport;
213 u8 h_flags;
214 u8 h_credit;
215 u8 h_padding[4];
216 __sum16 h_csum;
217
218 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
219};
220
221
222
223
224#define RDS_EXTHDR_NONE 0
225
226
227
228
229
230
231
232
233
234
235
236#define RDS_EXTHDR_VERSION 1
237struct rds_ext_header_version {
238 __be32 h_version;
239};
240
241
242
243
244
245#define RDS_EXTHDR_RDMA 2
246struct rds_ext_header_rdma {
247 __be32 h_rdma_rkey;
248};
249
250
251
252
253
254
255#define RDS_EXTHDR_RDMA_DEST 3
256struct rds_ext_header_rdma_dest {
257 __be32 h_rdma_rkey;
258 __be32 h_rdma_offset;
259};
260
261
262
263
264#define RDS_EXTHDR_NPATHS 5
265#define RDS_EXTHDR_GEN_NUM 6
266
267#define __RDS_EXTHDR_MAX 16
268#define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
269#define RDS_MSG_RX_HDR 0
270#define RDS_MSG_RX_START 1
271#define RDS_MSG_RX_END 2
272#define RDS_MSG_RX_CMSG 3
273
274struct rds_incoming {
275 refcount_t i_refcount;
276 struct list_head i_item;
277 struct rds_connection *i_conn;
278 struct rds_conn_path *i_conn_path;
279 struct rds_header i_hdr;
280 unsigned long i_rx_jiffies;
281 struct in6_addr i_saddr;
282
283 rds_rdma_cookie_t i_rdma_cookie;
284 ktime_t i_rx_tstamp;
285 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
286};
287
288struct rds_mr {
289 struct rb_node r_rb_node;
290 refcount_t r_refcount;
291 u32 r_key;
292
293
294 unsigned int r_use_once:1;
295 unsigned int r_invalidate:1;
296 unsigned int r_write:1;
297
298
299
300
301
302 unsigned long r_state;
303 struct rds_sock *r_sock;
304 struct rds_transport *r_trans;
305 void *r_trans_private;
306};
307
308
309#define RDS_MR_DEAD 0
310
311static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
312{
313 return r_key | (((u64) offset) << 32);
314}
315
316static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
317{
318 return cookie;
319}
320
321static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
322{
323 return cookie >> 32;
324}
325
326
327#define RDS_ATOMIC_TYPE_CSWP 0
328#define RDS_ATOMIC_TYPE_FADD 1
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359#define RDS_MSG_ON_SOCK 1
360#define RDS_MSG_ON_CONN 2
361#define RDS_MSG_HAS_ACK_SEQ 3
362#define RDS_MSG_ACK_REQUIRED 4
363#define RDS_MSG_RETRANSMITTED 5
364#define RDS_MSG_MAPPED 6
365#define RDS_MSG_PAGEVEC 7
366#define RDS_MSG_FLUSH 8
367
368struct rds_znotifier {
369 struct mmpin z_mmp;
370 u32 z_cookie;
371};
372
373struct rds_msg_zcopy_info {
374 struct list_head rs_zcookie_next;
375 union {
376 struct rds_znotifier znotif;
377 struct rds_zcopy_cookies zcookies;
378 };
379};
380
381struct rds_msg_zcopy_queue {
382 struct list_head zcookie_head;
383 spinlock_t lock;
384};
385
386static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
387{
388 spin_lock_init(&q->lock);
389 INIT_LIST_HEAD(&q->zcookie_head);
390}
391
392struct rds_iov_vector {
393 struct rds_iovec *iov;
394 int len;
395};
396
397struct rds_iov_vector_arr {
398 struct rds_iov_vector *vec;
399 int len;
400 int indx;
401 int incr;
402};
403
404struct rds_message {
405 refcount_t m_refcount;
406 struct list_head m_sock_item;
407 struct list_head m_conn_item;
408 struct rds_incoming m_inc;
409 u64 m_ack_seq;
410 struct in6_addr m_daddr;
411 unsigned long m_flags;
412
413
414
415
416
417
418 spinlock_t m_rs_lock;
419 wait_queue_head_t m_flush_wait;
420
421 struct rds_sock *m_rs;
422
423
424 rds_rdma_cookie_t m_rdma_cookie;
425
426 unsigned int m_used_sgs;
427 unsigned int m_total_sgs;
428
429 void *m_final_op;
430
431 struct {
432 struct rm_atomic_op {
433 int op_type;
434 union {
435 struct {
436 uint64_t compare;
437 uint64_t swap;
438 uint64_t compare_mask;
439 uint64_t swap_mask;
440 } op_m_cswp;
441 struct {
442 uint64_t add;
443 uint64_t nocarry_mask;
444 } op_m_fadd;
445 };
446
447 u32 op_rkey;
448 u64 op_remote_addr;
449 unsigned int op_notify:1;
450 unsigned int op_recverr:1;
451 unsigned int op_mapped:1;
452 unsigned int op_silent:1;
453 unsigned int op_active:1;
454 struct scatterlist *op_sg;
455 struct rds_notifier *op_notifier;
456
457 struct rds_mr *op_rdma_mr;
458 } atomic;
459 struct rm_rdma_op {
460 u32 op_rkey;
461 u64 op_remote_addr;
462 unsigned int op_write:1;
463 unsigned int op_fence:1;
464 unsigned int op_notify:1;
465 unsigned int op_recverr:1;
466 unsigned int op_mapped:1;
467 unsigned int op_silent:1;
468 unsigned int op_active:1;
469 unsigned int op_bytes;
470 unsigned int op_nents;
471 unsigned int op_count;
472 struct scatterlist *op_sg;
473 struct rds_notifier *op_notifier;
474
475 struct rds_mr *op_rdma_mr;
476 } rdma;
477 struct rm_data_op {
478 unsigned int op_active:1;
479 unsigned int op_nents;
480 unsigned int op_count;
481 unsigned int op_dmasg;
482 unsigned int op_dmaoff;
483 struct rds_znotifier *op_mmp_znotifier;
484 struct scatterlist *op_sg;
485 } data;
486 };
487
488 struct rds_conn_path *m_conn_path;
489};
490
491
492
493
494
495
496
497
498struct rds_notifier {
499 struct list_head n_list;
500 uint64_t n_user_token;
501 int n_status;
502};
503
504
505
506
507#define RDS_TRANS_LOOP 3
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536struct rds_transport {
537 char t_name[TRANSNAMSIZ];
538 struct list_head t_item;
539 struct module *t_owner;
540 unsigned int t_prefer_loopback:1,
541 t_mp_capable:1;
542 unsigned int t_type;
543
544 int (*laddr_check)(struct net *net, const struct in6_addr *addr,
545 __u32 scope_id);
546 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
547 void (*conn_free)(void *data);
548 int (*conn_path_connect)(struct rds_conn_path *cp);
549 void (*conn_path_shutdown)(struct rds_conn_path *conn);
550 void (*xmit_path_prepare)(struct rds_conn_path *cp);
551 void (*xmit_path_complete)(struct rds_conn_path *cp);
552 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
553 unsigned int hdr_off, unsigned int sg, unsigned int off);
554 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
555 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
556 int (*recv_path)(struct rds_conn_path *cp);
557 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
558 void (*inc_free)(struct rds_incoming *inc);
559
560 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
561 struct rdma_cm_event *event, bool isv6);
562 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id, bool isv6);
563 void (*cm_connect_complete)(struct rds_connection *conn,
564 struct rdma_cm_event *event);
565
566 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
567 unsigned int avail);
568 void (*exit)(void);
569 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
570 struct rds_sock *rs, u32 *key_ret,
571 struct rds_connection *conn);
572 void (*sync_mr)(void *trans_private, int direction);
573 void (*free_mr)(void *trans_private, int invalidate);
574 void (*flush_mrs)(void);
575 bool (*t_unloading)(struct rds_connection *conn);
576 u8 (*get_tos_map)(u8 tos);
577};
578
579
580
581
582#define RDS_BOUND_KEY_LEN \
583 (sizeof(struct in6_addr) + sizeof(__u32) + sizeof(__be16))
584
585struct rds_sock {
586 struct sock rs_sk;
587
588 u64 rs_user_addr;
589 u64 rs_user_bytes;
590
591
592
593
594
595 struct rhash_head rs_bound_node;
596 u8 rs_bound_key[RDS_BOUND_KEY_LEN];
597 struct sockaddr_in6 rs_bound_sin6;
598#define rs_bound_addr rs_bound_sin6.sin6_addr
599#define rs_bound_addr_v4 rs_bound_sin6.sin6_addr.s6_addr32[3]
600#define rs_bound_port rs_bound_sin6.sin6_port
601#define rs_bound_scope_id rs_bound_sin6.sin6_scope_id
602 struct in6_addr rs_conn_addr;
603#define rs_conn_addr_v4 rs_conn_addr.s6_addr32[3]
604 __be16 rs_conn_port;
605 struct rds_transport *rs_transport;
606
607
608
609
610
611 struct rds_connection *rs_conn;
612
613
614 int rs_congested;
615
616 int rs_seen_congestion;
617
618
619 spinlock_t rs_lock;
620 struct list_head rs_send_queue;
621 u32 rs_snd_bytes;
622 int rs_rcv_bytes;
623 struct list_head rs_notify_queue;
624
625
626
627
628
629
630 uint64_t rs_cong_mask;
631 uint64_t rs_cong_notify;
632 struct list_head rs_cong_list;
633 unsigned long rs_cong_track;
634
635
636
637
638
639 rwlock_t rs_recv_lock;
640 struct list_head rs_recv_queue;
641
642
643 struct list_head rs_item;
644
645
646 spinlock_t rs_rdma_lock;
647 struct rb_root rs_rdma_keys;
648
649
650 unsigned char rs_recverr,
651 rs_cong_monitor;
652 u32 rs_hash_initval;
653
654
655 u8 rs_rx_traces;
656 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
657 struct rds_msg_zcopy_queue rs_zcookie_queue;
658 u8 rs_tos;
659};
660
661static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
662{
663 return container_of(sk, struct rds_sock, rs_sk);
664}
665static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
666{
667 return &rs->rs_sk;
668}
669
670
671
672
673
674
675static inline int rds_sk_sndbuf(struct rds_sock *rs)
676{
677 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
678}
679static inline int rds_sk_rcvbuf(struct rds_sock *rs)
680{
681 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
682}
683
684struct rds_statistics {
685 uint64_t s_conn_reset;
686 uint64_t s_recv_drop_bad_checksum;
687 uint64_t s_recv_drop_old_seq;
688 uint64_t s_recv_drop_no_sock;
689 uint64_t s_recv_drop_dead_sock;
690 uint64_t s_recv_deliver_raced;
691 uint64_t s_recv_delivered;
692 uint64_t s_recv_queued;
693 uint64_t s_recv_immediate_retry;
694 uint64_t s_recv_delayed_retry;
695 uint64_t s_recv_ack_required;
696 uint64_t s_recv_rdma_bytes;
697 uint64_t s_recv_ping;
698 uint64_t s_send_queue_empty;
699 uint64_t s_send_queue_full;
700 uint64_t s_send_lock_contention;
701 uint64_t s_send_lock_queue_raced;
702 uint64_t s_send_immediate_retry;
703 uint64_t s_send_delayed_retry;
704 uint64_t s_send_drop_acked;
705 uint64_t s_send_ack_required;
706 uint64_t s_send_queued;
707 uint64_t s_send_rdma;
708 uint64_t s_send_rdma_bytes;
709 uint64_t s_send_pong;
710 uint64_t s_page_remainder_hit;
711 uint64_t s_page_remainder_miss;
712 uint64_t s_copy_to_user;
713 uint64_t s_copy_from_user;
714 uint64_t s_cong_update_queued;
715 uint64_t s_cong_update_received;
716 uint64_t s_cong_send_error;
717 uint64_t s_cong_send_blocked;
718 uint64_t s_recv_bytes_added_to_socket;
719 uint64_t s_recv_bytes_removed_from_socket;
720
721};
722
723
724void rds_sock_addref(struct rds_sock *rs);
725void rds_sock_put(struct rds_sock *rs);
726void rds_wake_sk_sleep(struct rds_sock *rs);
727static inline void __rds_wake_sk_sleep(struct sock *sk)
728{
729 wait_queue_head_t *waitq = sk_sleep(sk);
730
731 if (!sock_flag(sk, SOCK_DEAD) && waitq)
732 wake_up(waitq);
733}
734extern wait_queue_head_t rds_poll_waitq;
735
736
737
738int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
739void rds_remove_bound(struct rds_sock *rs);
740struct rds_sock *rds_find_bound(const struct in6_addr *addr, __be16 port,
741 __u32 scope_id);
742int rds_bind_lock_init(void);
743void rds_bind_lock_destroy(void);
744
745
746int rds_cong_get_maps(struct rds_connection *conn);
747void rds_cong_add_conn(struct rds_connection *conn);
748void rds_cong_remove_conn(struct rds_connection *conn);
749void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
750void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
751int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
752void rds_cong_queue_updates(struct rds_cong_map *map);
753void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
754int rds_cong_updated_since(unsigned long *recent);
755void rds_cong_add_socket(struct rds_sock *);
756void rds_cong_remove_socket(struct rds_sock *);
757void rds_cong_exit(void);
758struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
759
760
761extern u32 rds_gen_num;
762int rds_conn_init(void);
763void rds_conn_exit(void);
764struct rds_connection *rds_conn_create(struct net *net,
765 const struct in6_addr *laddr,
766 const struct in6_addr *faddr,
767 struct rds_transport *trans,
768 u8 tos, gfp_t gfp,
769 int dev_if);
770struct rds_connection *rds_conn_create_outgoing(struct net *net,
771 const struct in6_addr *laddr,
772 const struct in6_addr *faddr,
773 struct rds_transport *trans,
774 u8 tos, gfp_t gfp, int dev_if);
775void rds_conn_shutdown(struct rds_conn_path *cpath);
776void rds_conn_destroy(struct rds_connection *conn);
777void rds_conn_drop(struct rds_connection *conn);
778void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
779void rds_conn_connect_if_down(struct rds_connection *conn);
780void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
781void rds_for_each_conn_info(struct socket *sock, unsigned int len,
782 struct rds_info_iterator *iter,
783 struct rds_info_lengths *lens,
784 int (*visitor)(struct rds_connection *, void *),
785 u64 *buffer,
786 size_t item_len);
787
788__printf(2, 3)
789void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
790#define rds_conn_path_error(cp, fmt...) \
791 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
792
793static inline int
794rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
795{
796 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
797}
798
799static inline int
800rds_conn_transition(struct rds_connection *conn, int old, int new)
801{
802 WARN_ON(conn->c_trans->t_mp_capable);
803 return rds_conn_path_transition(&conn->c_path[0], old, new);
804}
805
806static inline int
807rds_conn_path_state(struct rds_conn_path *cp)
808{
809 return atomic_read(&cp->cp_state);
810}
811
812static inline int
813rds_conn_state(struct rds_connection *conn)
814{
815 WARN_ON(conn->c_trans->t_mp_capable);
816 return rds_conn_path_state(&conn->c_path[0]);
817}
818
819static inline int
820rds_conn_path_up(struct rds_conn_path *cp)
821{
822 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
823}
824
825static inline int
826rds_conn_up(struct rds_connection *conn)
827{
828 WARN_ON(conn->c_trans->t_mp_capable);
829 return rds_conn_path_up(&conn->c_path[0]);
830}
831
832static inline int
833rds_conn_path_connecting(struct rds_conn_path *cp)
834{
835 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
836}
837
838static inline int
839rds_conn_connecting(struct rds_connection *conn)
840{
841 WARN_ON(conn->c_trans->t_mp_capable);
842 return rds_conn_path_connecting(&conn->c_path[0]);
843}
844
845
846struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
847struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents,
848 int *ret);
849int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
850 bool zcopy);
851struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
852void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
853 __be16 dport, u64 seq);
854int rds_message_add_extension(struct rds_header *hdr,
855 unsigned int type, const void *data, unsigned int len);
856int rds_message_next_extension(struct rds_header *hdr,
857 unsigned int *pos, void *buf, unsigned int *buflen);
858int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
859int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
860void rds_message_inc_free(struct rds_incoming *inc);
861void rds_message_addref(struct rds_message *rm);
862void rds_message_put(struct rds_message *rm);
863void rds_message_wait(struct rds_message *rm);
864void rds_message_unmapped(struct rds_message *rm);
865void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
866
867static inline void rds_message_make_checksum(struct rds_header *hdr)
868{
869 hdr->h_csum = 0;
870 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
871}
872
873static inline int rds_message_verify_checksum(const struct rds_header *hdr)
874{
875 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
876}
877
878
879
880int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
881 gfp_t gfp);
882void rds_page_exit(void);
883
884
885void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
886 struct in6_addr *saddr);
887void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
888 struct in6_addr *saddr);
889void rds_inc_put(struct rds_incoming *inc);
890void rds_recv_incoming(struct rds_connection *conn, struct in6_addr *saddr,
891 struct in6_addr *daddr,
892 struct rds_incoming *inc, gfp_t gfp);
893int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
894 int msg_flags);
895void rds_clear_recv_queue(struct rds_sock *rs);
896int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
897void rds_inc_info_copy(struct rds_incoming *inc,
898 struct rds_info_iterator *iter,
899 __be32 saddr, __be32 daddr, int flip);
900void rds6_inc_info_copy(struct rds_incoming *inc,
901 struct rds_info_iterator *iter,
902 struct in6_addr *saddr, struct in6_addr *daddr,
903 int flip);
904
905
906int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
907void rds_send_path_reset(struct rds_conn_path *conn);
908int rds_send_xmit(struct rds_conn_path *cp);
909struct sockaddr_in;
910void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest);
911typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
912void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
913 is_acked_func is_acked);
914void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
915 is_acked_func is_acked);
916void rds_send_ping(struct rds_connection *conn, int cp_index);
917int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
918
919
920void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
921int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
922int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
923int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
924void rds_rdma_drop_keys(struct rds_sock *rs);
925int rds_rdma_extra_size(struct rds_rdma_args *args,
926 struct rds_iov_vector *iov);
927int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
928 struct cmsghdr *cmsg);
929int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
930 struct cmsghdr *cmsg,
931 struct rds_iov_vector *vec);
932int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
933 struct cmsghdr *cmsg);
934void rds_rdma_free_op(struct rm_rdma_op *ro);
935void rds_atomic_free_op(struct rm_atomic_op *ao);
936void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
937void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
938int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
939 struct cmsghdr *cmsg);
940
941void __rds_put_mr_final(struct rds_mr *mr);
942static inline void rds_mr_put(struct rds_mr *mr)
943{
944 if (refcount_dec_and_test(&mr->r_refcount))
945 __rds_put_mr_final(mr);
946}
947
948static inline bool rds_destroy_pending(struct rds_connection *conn)
949{
950 return !check_net(rds_conn_net(conn)) ||
951 (conn->c_trans->t_unloading && conn->c_trans->t_unloading(conn));
952}
953
954
955DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
956#define rds_stats_inc_which(which, member) do { \
957 per_cpu(which, get_cpu()).member++; \
958 put_cpu(); \
959} while (0)
960#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
961#define rds_stats_add_which(which, member, count) do { \
962 per_cpu(which, get_cpu()).member += count; \
963 put_cpu(); \
964} while (0)
965#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
966int rds_stats_init(void);
967void rds_stats_exit(void);
968void rds_stats_info_copy(struct rds_info_iterator *iter,
969 uint64_t *values, const char *const *names,
970 size_t nr);
971
972
973int rds_sysctl_init(void);
974void rds_sysctl_exit(void);
975extern unsigned long rds_sysctl_sndbuf_min;
976extern unsigned long rds_sysctl_sndbuf_default;
977extern unsigned long rds_sysctl_sndbuf_max;
978extern unsigned long rds_sysctl_reconnect_min_jiffies;
979extern unsigned long rds_sysctl_reconnect_max_jiffies;
980extern unsigned int rds_sysctl_max_unacked_packets;
981extern unsigned int rds_sysctl_max_unacked_bytes;
982extern unsigned int rds_sysctl_ping_enable;
983extern unsigned long rds_sysctl_trace_flags;
984extern unsigned int rds_sysctl_trace_level;
985
986
987int rds_threads_init(void);
988void rds_threads_exit(void);
989extern struct workqueue_struct *rds_wq;
990void rds_queue_reconnect(struct rds_conn_path *cp);
991void rds_connect_worker(struct work_struct *);
992void rds_shutdown_worker(struct work_struct *);
993void rds_send_worker(struct work_struct *);
994void rds_recv_worker(struct work_struct *);
995void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
996void rds_connect_complete(struct rds_connection *conn);
997int rds_addr_cmp(const struct in6_addr *a1, const struct in6_addr *a2);
998
999
1000void rds_trans_register(struct rds_transport *trans);
1001void rds_trans_unregister(struct rds_transport *trans);
1002struct rds_transport *rds_trans_get_preferred(struct net *net,
1003 const struct in6_addr *addr,
1004 __u32 scope_id);
1005void rds_trans_put(struct rds_transport *trans);
1006unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
1007 unsigned int avail);
1008struct rds_transport *rds_trans_get(int t_type);
1009int rds_trans_init(void);
1010void rds_trans_exit(void);
1011
1012#endif
1013