1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43#ifndef SVC_RDMA_H
44#define SVC_RDMA_H
45#include <linux/llist.h>
46#include <linux/sunrpc/xdr.h>
47#include <linux/sunrpc/svcsock.h>
48#include <linux/sunrpc/rpc_rdma.h>
49#include <linux/sunrpc/rpc_rdma_cid.h>
50#include <linux/sunrpc/svc_rdma_pcl.h>
51
52#include <linux/percpu_counter.h>
53#include <rdma/ib_verbs.h>
54#include <rdma/rdma_cm.h>
55
56
57enum {
58 RPCRDMA_PULLUP_THRESH = RPCRDMA_V1_DEF_INLINE_SIZE >> 1,
59 RPCRDMA_DEF_INLINE_THRESH = 4096,
60 RPCRDMA_MAX_INLINE_THRESH = 65536
61};
62
63
64extern unsigned int svcrdma_ord;
65extern unsigned int svcrdma_max_requests;
66extern unsigned int svcrdma_max_bc_requests;
67extern unsigned int svcrdma_max_req_size;
68
69extern struct percpu_counter svcrdma_stat_read;
70extern struct percpu_counter svcrdma_stat_recv;
71extern struct percpu_counter svcrdma_stat_sq_starve;
72extern struct percpu_counter svcrdma_stat_write;
73
74struct svcxprt_rdma {
75 struct svc_xprt sc_xprt;
76 struct rdma_cm_id *sc_cm_id;
77 struct list_head sc_accept_q;
78 int sc_ord;
79 int sc_max_send_sges;
80 bool sc_snd_w_inv;
81
82 atomic_t sc_sq_avail;
83 unsigned int sc_sq_depth;
84 __be32 sc_fc_credits;
85 u32 sc_max_requests;
86 u32 sc_max_bc_requests;
87 int sc_max_req_size;
88 u8 sc_port_num;
89
90 struct ib_pd *sc_pd;
91
92 spinlock_t sc_send_lock;
93 struct list_head sc_send_ctxts;
94 spinlock_t sc_rw_ctxt_lock;
95 struct list_head sc_rw_ctxts;
96
97 u32 sc_pending_recvs;
98 u32 sc_recv_batch;
99 struct list_head sc_rq_dto_q;
100 spinlock_t sc_rq_dto_lock;
101 struct ib_qp *sc_qp;
102 struct ib_cq *sc_rq_cq;
103 struct ib_cq *sc_sq_cq;
104
105 spinlock_t sc_lock;
106
107 wait_queue_head_t sc_send_wait;
108 unsigned long sc_flags;
109 struct work_struct sc_work;
110
111 struct llist_head sc_recv_ctxts;
112
113 atomic_t sc_completion_ids;
114};
115
116#define RDMAXPRT_CONN_PENDING 3
117
118
119
120
121enum {
122 RPCRDMA_LISTEN_BACKLOG = 10,
123 RPCRDMA_MAX_REQUESTS = 64,
124 RPCRDMA_MAX_BC_REQUESTS = 2,
125};
126
127#define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
128
129struct svc_rdma_recv_ctxt {
130 struct llist_node rc_node;
131 struct list_head rc_list;
132 struct ib_recv_wr rc_recv_wr;
133 struct ib_cqe rc_cqe;
134 struct rpc_rdma_cid rc_cid;
135 struct ib_sge rc_recv_sge;
136 void *rc_recv_buf;
137 struct xdr_stream rc_stream;
138 bool rc_temp;
139 u32 rc_byte_len;
140 unsigned int rc_page_count;
141 u32 rc_inv_rkey;
142 __be32 rc_msgtype;
143
144 struct svc_rdma_pcl rc_call_pcl;
145
146 struct svc_rdma_pcl rc_read_pcl;
147 struct svc_rdma_chunk *rc_cur_result_payload;
148 struct svc_rdma_pcl rc_write_pcl;
149 struct svc_rdma_pcl rc_reply_pcl;
150};
151
152struct svc_rdma_send_ctxt {
153 struct list_head sc_list;
154 struct rpc_rdma_cid sc_cid;
155
156 struct ib_send_wr sc_send_wr;
157 struct ib_cqe sc_cqe;
158 struct completion sc_done;
159 struct xdr_buf sc_hdrbuf;
160 struct xdr_stream sc_stream;
161 void *sc_xprt_buf;
162 int sc_cur_sge_no;
163
164 struct ib_sge sc_sges[];
165};
166
167
168extern void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp,
169 struct svc_rdma_recv_ctxt *rctxt);
170
171
172extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma);
173extern bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma);
174extern struct svc_rdma_recv_ctxt *
175 svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma);
176extern void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,
177 struct svc_rdma_recv_ctxt *ctxt);
178extern void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma);
179extern void svc_rdma_release_rqst(struct svc_rqst *rqstp);
180extern int svc_rdma_recvfrom(struct svc_rqst *);
181
182
183extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
184extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
185 const struct svc_rdma_chunk *chunk,
186 const struct xdr_buf *xdr);
187extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
188 const struct svc_rdma_recv_ctxt *rctxt,
189 const struct xdr_buf *xdr);
190extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
191 struct svc_rqst *rqstp,
192 struct svc_rdma_recv_ctxt *head);
193
194
195extern void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma);
196extern struct svc_rdma_send_ctxt *
197 svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma);
198extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma,
199 struct svc_rdma_send_ctxt *ctxt);
200extern int svc_rdma_send(struct svcxprt_rdma *rdma,
201 struct svc_rdma_send_ctxt *ctxt);
202extern int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma,
203 struct svc_rdma_send_ctxt *sctxt,
204 const struct svc_rdma_recv_ctxt *rctxt,
205 const struct xdr_buf *xdr);
206extern void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma,
207 struct svc_rdma_send_ctxt *sctxt,
208 struct svc_rdma_recv_ctxt *rctxt,
209 int status);
210extern int svc_rdma_sendto(struct svc_rqst *);
211extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset,
212 unsigned int length);
213
214
215extern struct svc_xprt_class svc_rdma_class;
216#ifdef CONFIG_SUNRPC_BACKCHANNEL
217extern struct svc_xprt_class svc_rdma_bc_class;
218#endif
219
220
221extern int svc_rdma_init(void);
222extern void svc_rdma_cleanup(void);
223
224#endif
225