1
2#ifndef _RDS_RDS_H
3#define _RDS_RDS_H
4
5#include <net/sock.h>
6#include <linux/scatterlist.h>
7#include <linux/highmem.h>
8#include <rdma/rdma_cm.h>
9#include <linux/mutex.h>
10#include <linux/rds.h>
11#include <linux/rhashtable.h>
12#include <linux/refcount.h>
13
14#include "info.h"
15
16
17
18
19#define RDS_PROTOCOL_3_0 0x0300
20#define RDS_PROTOCOL_3_1 0x0301
21#define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
22#define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
23#define RDS_PROTOCOL_MINOR(v) ((v) & 255)
24#define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
25
26
27
28
29
30
31
32#define RDS_PORT 18634
33
34#ifdef ATOMIC64_INIT
35#define KERNEL_HAS_ATOMIC64
36#endif
37
38#ifdef RDS_DEBUG
39#define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
40#else
41
42static inline __printf(1, 2)
43void rdsdebug(char *fmt, ...)
44{
45}
46#endif
47
48
49#define ceil(x, y) \
50 ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
51
52#define RDS_FRAG_SHIFT 12
53#define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
54
55
56#define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
57
58#define RDS_CONG_MAP_BYTES (65536 / 8)
59#define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
60#define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
61
62struct rds_cong_map {
63 struct rb_node m_rb_node;
64 __be32 m_addr;
65 wait_queue_head_t m_waitq;
66 struct list_head m_conn_list;
67 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
68};
69
70
71
72
73
74
75
76
77enum {
78 RDS_CONN_DOWN = 0,
79 RDS_CONN_CONNECTING,
80 RDS_CONN_DISCONNECTING,
81 RDS_CONN_UP,
82 RDS_CONN_RESETTING,
83 RDS_CONN_ERROR,
84};
85
86
87#define RDS_LL_SEND_FULL 0
88#define RDS_RECONNECT_PENDING 1
89#define RDS_IN_XMIT 2
90#define RDS_RECV_REFILL 3
91#define RDS_DESTROY_PENDING 4
92
93
94#define RDS_MPATH_WORKERS 8
95#define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
96 (rs)->rs_hash_initval) & ((n) - 1))
97
98#define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
99
100
101struct rds_conn_path {
102 struct rds_connection *cp_conn;
103 struct rds_message *cp_xmit_rm;
104 unsigned long cp_xmit_sg;
105 unsigned int cp_xmit_hdr_off;
106 unsigned int cp_xmit_data_off;
107 unsigned int cp_xmit_atomic_sent;
108 unsigned int cp_xmit_rdma_sent;
109 unsigned int cp_xmit_data_sent;
110
111 spinlock_t cp_lock;
112 u64 cp_next_tx_seq;
113 struct list_head cp_send_queue;
114 struct list_head cp_retrans;
115
116 u64 cp_next_rx_seq;
117
118 void *cp_transport_data;
119
120 atomic_t cp_state;
121 unsigned long cp_send_gen;
122 unsigned long cp_flags;
123 unsigned long cp_reconnect_jiffies;
124 struct delayed_work cp_send_w;
125 struct delayed_work cp_recv_w;
126 struct delayed_work cp_conn_w;
127 struct work_struct cp_down_w;
128 struct mutex cp_cm_lock;
129 wait_queue_head_t cp_waitq;
130
131 unsigned int cp_unacked_packets;
132 unsigned int cp_unacked_bytes;
133 unsigned int cp_index;
134};
135
136
137struct rds_connection {
138 struct hlist_node c_hash_node;
139 __be32 c_laddr;
140 __be32 c_faddr;
141 unsigned int c_loopback:1,
142 c_ping_triggered:1,
143 c_pad_to_32:30;
144 int c_npaths;
145 struct rds_connection *c_passive;
146 struct rds_transport *c_trans;
147
148 struct rds_cong_map *c_lcong;
149 struct rds_cong_map *c_fcong;
150
151
152 unsigned int c_version;
153 possible_net_t c_net;
154
155 struct list_head c_map_item;
156 unsigned long c_map_queued;
157
158 struct rds_conn_path *c_path;
159 wait_queue_head_t c_hs_waitq;
160
161 u32 c_my_gen_num;
162 u32 c_peer_gen_num;
163};
164
165static inline
166struct net *rds_conn_net(struct rds_connection *conn)
167{
168 return read_pnet(&conn->c_net);
169}
170
171static inline
172void rds_conn_net_set(struct rds_connection *conn, struct net *net)
173{
174 write_pnet(&conn->c_net, net);
175}
176
177#define RDS_FLAG_CONG_BITMAP 0x01
178#define RDS_FLAG_ACK_REQUIRED 0x02
179#define RDS_FLAG_RETRANSMITTED 0x04
180#define RDS_MAX_ADV_CREDIT 255
181
182
183
184
185
186
187
188
189#define RDS_FLAG_PROBE_PORT 1
190#define RDS_HS_PROBE(sport, dport) \
191 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
192 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
193
194
195
196#define RDS_HEADER_EXT_SPACE 16
197
198struct rds_header {
199 __be64 h_sequence;
200 __be64 h_ack;
201 __be32 h_len;
202 __be16 h_sport;
203 __be16 h_dport;
204 u8 h_flags;
205 u8 h_credit;
206 u8 h_padding[4];
207 __sum16 h_csum;
208
209 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
210};
211
212
213
214
215#define RDS_EXTHDR_NONE 0
216
217
218
219
220
221
222
223
224
225
226
227#define RDS_EXTHDR_VERSION 1
228struct rds_ext_header_version {
229 __be32 h_version;
230};
231
232
233
234
235
236#define RDS_EXTHDR_RDMA 2
237struct rds_ext_header_rdma {
238 __be32 h_rdma_rkey;
239};
240
241
242
243
244
245
246#define RDS_EXTHDR_RDMA_DEST 3
247struct rds_ext_header_rdma_dest {
248 __be32 h_rdma_rkey;
249 __be32 h_rdma_offset;
250};
251
252
253
254
255#define RDS_EXTHDR_NPATHS 5
256#define RDS_EXTHDR_GEN_NUM 6
257
258#define __RDS_EXTHDR_MAX 16
259#define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
260#define RDS_MSG_RX_HDR 0
261#define RDS_MSG_RX_START 1
262#define RDS_MSG_RX_END 2
263#define RDS_MSG_RX_CMSG 3
264
265struct rds_incoming {
266 refcount_t i_refcount;
267 struct list_head i_item;
268 struct rds_connection *i_conn;
269 struct rds_conn_path *i_conn_path;
270 struct rds_header i_hdr;
271 unsigned long i_rx_jiffies;
272 __be32 i_saddr;
273
274 rds_rdma_cookie_t i_rdma_cookie;
275 struct timeval i_rx_tstamp;
276 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
277};
278
279struct rds_mr {
280 struct rb_node r_rb_node;
281 refcount_t r_refcount;
282 u32 r_key;
283
284
285 unsigned int r_use_once:1;
286 unsigned int r_invalidate:1;
287 unsigned int r_write:1;
288
289
290
291
292
293 unsigned long r_state;
294 struct rds_sock *r_sock;
295 struct rds_transport *r_trans;
296 void *r_trans_private;
297};
298
299
300#define RDS_MR_DEAD 0
301
302static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
303{
304 return r_key | (((u64) offset) << 32);
305}
306
307static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
308{
309 return cookie;
310}
311
312static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
313{
314 return cookie >> 32;
315}
316
317
318#define RDS_ATOMIC_TYPE_CSWP 0
319#define RDS_ATOMIC_TYPE_FADD 1
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350#define RDS_MSG_ON_SOCK 1
351#define RDS_MSG_ON_CONN 2
352#define RDS_MSG_HAS_ACK_SEQ 3
353#define RDS_MSG_ACK_REQUIRED 4
354#define RDS_MSG_RETRANSMITTED 5
355#define RDS_MSG_MAPPED 6
356#define RDS_MSG_PAGEVEC 7
357#define RDS_MSG_FLUSH 8
358
359struct rds_znotifier {
360 struct mmpin z_mmp;
361 u32 z_cookie;
362};
363
364struct rds_msg_zcopy_info {
365 struct list_head rs_zcookie_next;
366 union {
367 struct rds_znotifier znotif;
368 struct rds_zcopy_cookies zcookies;
369 };
370};
371
372struct rds_msg_zcopy_queue {
373 struct list_head zcookie_head;
374 spinlock_t lock;
375};
376
377static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
378{
379 spin_lock_init(&q->lock);
380 INIT_LIST_HEAD(&q->zcookie_head);
381}
382
383struct rds_message {
384 refcount_t m_refcount;
385 struct list_head m_sock_item;
386 struct list_head m_conn_item;
387 struct rds_incoming m_inc;
388 u64 m_ack_seq;
389 __be32 m_daddr;
390 unsigned long m_flags;
391
392
393
394
395
396
397 spinlock_t m_rs_lock;
398 wait_queue_head_t m_flush_wait;
399
400 struct rds_sock *m_rs;
401
402
403 rds_rdma_cookie_t m_rdma_cookie;
404
405 unsigned int m_used_sgs;
406 unsigned int m_total_sgs;
407
408 void *m_final_op;
409
410 struct {
411 struct rm_atomic_op {
412 int op_type;
413 union {
414 struct {
415 uint64_t compare;
416 uint64_t swap;
417 uint64_t compare_mask;
418 uint64_t swap_mask;
419 } op_m_cswp;
420 struct {
421 uint64_t add;
422 uint64_t nocarry_mask;
423 } op_m_fadd;
424 };
425
426 u32 op_rkey;
427 u64 op_remote_addr;
428 unsigned int op_notify:1;
429 unsigned int op_recverr:1;
430 unsigned int op_mapped:1;
431 unsigned int op_silent:1;
432 unsigned int op_active:1;
433 struct scatterlist *op_sg;
434 struct rds_notifier *op_notifier;
435
436 struct rds_mr *op_rdma_mr;
437 } atomic;
438 struct rm_rdma_op {
439 u32 op_rkey;
440 u64 op_remote_addr;
441 unsigned int op_write:1;
442 unsigned int op_fence:1;
443 unsigned int op_notify:1;
444 unsigned int op_recverr:1;
445 unsigned int op_mapped:1;
446 unsigned int op_silent:1;
447 unsigned int op_active:1;
448 unsigned int op_bytes;
449 unsigned int op_nents;
450 unsigned int op_count;
451 struct scatterlist *op_sg;
452 struct rds_notifier *op_notifier;
453
454 struct rds_mr *op_rdma_mr;
455 } rdma;
456 struct rm_data_op {
457 unsigned int op_active:1;
458 unsigned int op_notify:1;
459 unsigned int op_nents;
460 unsigned int op_count;
461 unsigned int op_dmasg;
462 unsigned int op_dmaoff;
463 struct rds_znotifier *op_mmp_znotifier;
464 struct scatterlist *op_sg;
465 } data;
466 };
467};
468
469
470
471
472
473
474
475
476struct rds_notifier {
477 struct list_head n_list;
478 uint64_t n_user_token;
479 int n_status;
480};
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509struct rds_transport {
510 char t_name[TRANSNAMSIZ];
511 struct list_head t_item;
512 struct module *t_owner;
513 unsigned int t_prefer_loopback:1,
514 t_mp_capable:1;
515 unsigned int t_type;
516
517 int (*laddr_check)(struct net *net, __be32 addr);
518 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
519 void (*conn_free)(void *data);
520 int (*conn_path_connect)(struct rds_conn_path *cp);
521 void (*conn_path_shutdown)(struct rds_conn_path *conn);
522 void (*xmit_path_prepare)(struct rds_conn_path *cp);
523 void (*xmit_path_complete)(struct rds_conn_path *cp);
524 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
525 unsigned int hdr_off, unsigned int sg, unsigned int off);
526 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
527 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
528 int (*recv_path)(struct rds_conn_path *cp);
529 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
530 void (*inc_free)(struct rds_incoming *inc);
531
532 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
533 struct rdma_cm_event *event);
534 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id);
535 void (*cm_connect_complete)(struct rds_connection *conn,
536 struct rdma_cm_event *event);
537
538 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
539 unsigned int avail);
540 void (*exit)(void);
541 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
542 struct rds_sock *rs, u32 *key_ret);
543 void (*sync_mr)(void *trans_private, int direction);
544 void (*free_mr)(void *trans_private, int invalidate);
545 void (*flush_mrs)(void);
546 bool (*t_unloading)(struct rds_connection *conn);
547};
548
549struct rds_sock {
550 struct sock rs_sk;
551
552 u64 rs_user_addr;
553 u64 rs_user_bytes;
554
555
556
557
558
559 struct rhash_head rs_bound_node;
560 u64 rs_bound_key;
561 __be32 rs_bound_addr;
562 __be32 rs_conn_addr;
563 __be16 rs_bound_port;
564 __be16 rs_conn_port;
565 struct rds_transport *rs_transport;
566
567
568
569
570
571 struct rds_connection *rs_conn;
572
573
574 int rs_congested;
575
576 int rs_seen_congestion;
577
578
579 spinlock_t rs_lock;
580 struct list_head rs_send_queue;
581 u32 rs_snd_bytes;
582 int rs_rcv_bytes;
583 struct list_head rs_notify_queue;
584
585
586
587
588
589
590 uint64_t rs_cong_mask;
591 uint64_t rs_cong_notify;
592 struct list_head rs_cong_list;
593 unsigned long rs_cong_track;
594
595
596
597
598
599 rwlock_t rs_recv_lock;
600 struct list_head rs_recv_queue;
601
602
603 struct list_head rs_item;
604
605
606 spinlock_t rs_rdma_lock;
607 struct rb_root rs_rdma_keys;
608
609
610 unsigned char rs_recverr,
611 rs_cong_monitor;
612 u32 rs_hash_initval;
613
614
615 u8 rs_rx_traces;
616 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
617 struct rds_msg_zcopy_queue rs_zcookie_queue;
618};
619
620static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
621{
622 return container_of(sk, struct rds_sock, rs_sk);
623}
624static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
625{
626 return &rs->rs_sk;
627}
628
629
630
631
632
633
634static inline int rds_sk_sndbuf(struct rds_sock *rs)
635{
636 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
637}
638static inline int rds_sk_rcvbuf(struct rds_sock *rs)
639{
640 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
641}
642
643struct rds_statistics {
644 uint64_t s_conn_reset;
645 uint64_t s_recv_drop_bad_checksum;
646 uint64_t s_recv_drop_old_seq;
647 uint64_t s_recv_drop_no_sock;
648 uint64_t s_recv_drop_dead_sock;
649 uint64_t s_recv_deliver_raced;
650 uint64_t s_recv_delivered;
651 uint64_t s_recv_queued;
652 uint64_t s_recv_immediate_retry;
653 uint64_t s_recv_delayed_retry;
654 uint64_t s_recv_ack_required;
655 uint64_t s_recv_rdma_bytes;
656 uint64_t s_recv_ping;
657 uint64_t s_send_queue_empty;
658 uint64_t s_send_queue_full;
659 uint64_t s_send_lock_contention;
660 uint64_t s_send_lock_queue_raced;
661 uint64_t s_send_immediate_retry;
662 uint64_t s_send_delayed_retry;
663 uint64_t s_send_drop_acked;
664 uint64_t s_send_ack_required;
665 uint64_t s_send_queued;
666 uint64_t s_send_rdma;
667 uint64_t s_send_rdma_bytes;
668 uint64_t s_send_pong;
669 uint64_t s_page_remainder_hit;
670 uint64_t s_page_remainder_miss;
671 uint64_t s_copy_to_user;
672 uint64_t s_copy_from_user;
673 uint64_t s_cong_update_queued;
674 uint64_t s_cong_update_received;
675 uint64_t s_cong_send_error;
676 uint64_t s_cong_send_blocked;
677 uint64_t s_recv_bytes_added_to_socket;
678 uint64_t s_recv_bytes_removed_from_socket;
679
680};
681
682
683void rds_sock_addref(struct rds_sock *rs);
684void rds_sock_put(struct rds_sock *rs);
685void rds_wake_sk_sleep(struct rds_sock *rs);
686static inline void __rds_wake_sk_sleep(struct sock *sk)
687{
688 wait_queue_head_t *waitq = sk_sleep(sk);
689
690 if (!sock_flag(sk, SOCK_DEAD) && waitq)
691 wake_up(waitq);
692}
693extern wait_queue_head_t rds_poll_waitq;
694
695
696
697int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
698void rds_remove_bound(struct rds_sock *rs);
699struct rds_sock *rds_find_bound(__be32 addr, __be16 port);
700int rds_bind_lock_init(void);
701void rds_bind_lock_destroy(void);
702
703
704int rds_cong_get_maps(struct rds_connection *conn);
705void rds_cong_add_conn(struct rds_connection *conn);
706void rds_cong_remove_conn(struct rds_connection *conn);
707void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
708void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
709int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
710void rds_cong_queue_updates(struct rds_cong_map *map);
711void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
712int rds_cong_updated_since(unsigned long *recent);
713void rds_cong_add_socket(struct rds_sock *);
714void rds_cong_remove_socket(struct rds_sock *);
715void rds_cong_exit(void);
716struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
717
718
719extern u32 rds_gen_num;
720int rds_conn_init(void);
721void rds_conn_exit(void);
722struct rds_connection *rds_conn_create(struct net *net,
723 __be32 laddr, __be32 faddr,
724 struct rds_transport *trans, gfp_t gfp);
725struct rds_connection *rds_conn_create_outgoing(struct net *net,
726 __be32 laddr, __be32 faddr,
727 struct rds_transport *trans, gfp_t gfp);
728void rds_conn_shutdown(struct rds_conn_path *cpath);
729void rds_conn_destroy(struct rds_connection *conn);
730void rds_conn_drop(struct rds_connection *conn);
731void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
732void rds_conn_connect_if_down(struct rds_connection *conn);
733void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
734void rds_for_each_conn_info(struct socket *sock, unsigned int len,
735 struct rds_info_iterator *iter,
736 struct rds_info_lengths *lens,
737 int (*visitor)(struct rds_connection *, void *),
738 u64 *buffer,
739 size_t item_len);
740
741__printf(2, 3)
742void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
743#define rds_conn_path_error(cp, fmt...) \
744 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
745
746static inline int
747rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
748{
749 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
750}
751
752static inline int
753rds_conn_transition(struct rds_connection *conn, int old, int new)
754{
755 WARN_ON(conn->c_trans->t_mp_capable);
756 return rds_conn_path_transition(&conn->c_path[0], old, new);
757}
758
759static inline int
760rds_conn_path_state(struct rds_conn_path *cp)
761{
762 return atomic_read(&cp->cp_state);
763}
764
765static inline int
766rds_conn_state(struct rds_connection *conn)
767{
768 WARN_ON(conn->c_trans->t_mp_capable);
769 return rds_conn_path_state(&conn->c_path[0]);
770}
771
772static inline int
773rds_conn_path_up(struct rds_conn_path *cp)
774{
775 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
776}
777
778static inline int
779rds_conn_up(struct rds_connection *conn)
780{
781 WARN_ON(conn->c_trans->t_mp_capable);
782 return rds_conn_path_up(&conn->c_path[0]);
783}
784
785static inline int
786rds_conn_path_connecting(struct rds_conn_path *cp)
787{
788 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
789}
790
791static inline int
792rds_conn_connecting(struct rds_connection *conn)
793{
794 WARN_ON(conn->c_trans->t_mp_capable);
795 return rds_conn_path_connecting(&conn->c_path[0]);
796}
797
798
799struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
800struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
801int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
802 bool zcopy);
803struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
804void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
805 __be16 dport, u64 seq);
806int rds_message_add_extension(struct rds_header *hdr,
807 unsigned int type, const void *data, unsigned int len);
808int rds_message_next_extension(struct rds_header *hdr,
809 unsigned int *pos, void *buf, unsigned int *buflen);
810int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
811int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
812void rds_message_inc_free(struct rds_incoming *inc);
813void rds_message_addref(struct rds_message *rm);
814void rds_message_put(struct rds_message *rm);
815void rds_message_wait(struct rds_message *rm);
816void rds_message_unmapped(struct rds_message *rm);
817void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
818
819static inline void rds_message_make_checksum(struct rds_header *hdr)
820{
821 hdr->h_csum = 0;
822 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
823}
824
825static inline int rds_message_verify_checksum(const struct rds_header *hdr)
826{
827 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
828}
829
830
831
832int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
833 gfp_t gfp);
834void rds_page_exit(void);
835
836
837void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
838 __be32 saddr);
839void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
840 __be32 saddr);
841void rds_inc_put(struct rds_incoming *inc);
842void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
843 struct rds_incoming *inc, gfp_t gfp);
844int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
845 int msg_flags);
846void rds_clear_recv_queue(struct rds_sock *rs);
847int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
848void rds_inc_info_copy(struct rds_incoming *inc,
849 struct rds_info_iterator *iter,
850 __be32 saddr, __be32 daddr, int flip);
851
852
853int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
854void rds_send_path_reset(struct rds_conn_path *conn);
855int rds_send_xmit(struct rds_conn_path *cp);
856struct sockaddr_in;
857void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest);
858typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
859void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
860 is_acked_func is_acked);
861void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
862 is_acked_func is_acked);
863void rds_send_ping(struct rds_connection *conn, int cp_index);
864int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
865
866
867void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
868int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
869int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
870int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
871void rds_rdma_drop_keys(struct rds_sock *rs);
872int rds_rdma_extra_size(struct rds_rdma_args *args);
873int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
874 struct cmsghdr *cmsg);
875int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
876 struct cmsghdr *cmsg);
877int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
878 struct cmsghdr *cmsg);
879int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
880 struct cmsghdr *cmsg);
881void rds_rdma_free_op(struct rm_rdma_op *ro);
882void rds_atomic_free_op(struct rm_atomic_op *ao);
883void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
884void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
885int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
886 struct cmsghdr *cmsg);
887
888void __rds_put_mr_final(struct rds_mr *mr);
889static inline void rds_mr_put(struct rds_mr *mr)
890{
891 if (refcount_dec_and_test(&mr->r_refcount))
892 __rds_put_mr_final(mr);
893}
894
895static inline bool rds_destroy_pending(struct rds_connection *conn)
896{
897 return !check_net(rds_conn_net(conn)) ||
898 (conn->c_trans->t_unloading && conn->c_trans->t_unloading(conn));
899}
900
901
902DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
903#define rds_stats_inc_which(which, member) do { \
904 per_cpu(which, get_cpu()).member++; \
905 put_cpu(); \
906} while (0)
907#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
908#define rds_stats_add_which(which, member, count) do { \
909 per_cpu(which, get_cpu()).member += count; \
910 put_cpu(); \
911} while (0)
912#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
913int rds_stats_init(void);
914void rds_stats_exit(void);
915void rds_stats_info_copy(struct rds_info_iterator *iter,
916 uint64_t *values, const char *const *names,
917 size_t nr);
918
919
920int rds_sysctl_init(void);
921void rds_sysctl_exit(void);
922extern unsigned long rds_sysctl_sndbuf_min;
923extern unsigned long rds_sysctl_sndbuf_default;
924extern unsigned long rds_sysctl_sndbuf_max;
925extern unsigned long rds_sysctl_reconnect_min_jiffies;
926extern unsigned long rds_sysctl_reconnect_max_jiffies;
927extern unsigned int rds_sysctl_max_unacked_packets;
928extern unsigned int rds_sysctl_max_unacked_bytes;
929extern unsigned int rds_sysctl_ping_enable;
930extern unsigned long rds_sysctl_trace_flags;
931extern unsigned int rds_sysctl_trace_level;
932
933
934int rds_threads_init(void);
935void rds_threads_exit(void);
936extern struct workqueue_struct *rds_wq;
937void rds_queue_reconnect(struct rds_conn_path *cp);
938void rds_connect_worker(struct work_struct *);
939void rds_shutdown_worker(struct work_struct *);
940void rds_send_worker(struct work_struct *);
941void rds_recv_worker(struct work_struct *);
942void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
943void rds_connect_complete(struct rds_connection *conn);
944
945
946void rds_trans_register(struct rds_transport *trans);
947void rds_trans_unregister(struct rds_transport *trans);
948struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
949void rds_trans_put(struct rds_transport *trans);
950unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
951 unsigned int avail);
952struct rds_transport *rds_trans_get(int t_type);
953int rds_trans_init(void);
954void rds_trans_exit(void);
955
956#endif
957