1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _LINUX_SUNRPC_XPRT_RDMA_H
41#define _LINUX_SUNRPC_XPRT_RDMA_H
42
43#include <linux/wait.h>
44#include <linux/spinlock.h>
45#include <asm/atomic.h>
46
47#include <rdma/rdma_cm.h>
48#include <rdma/ib_verbs.h>
49
50#include <linux/sunrpc/clnt.h>
51#include <linux/sunrpc/rpc_rdma.h>
52#include <linux/sunrpc/xprtrdma.h>
53
54#define RDMA_RESOLVE_TIMEOUT (5000)
55#define RDMA_CONNECT_RETRY_MAX (2)
56
57
58
59
60struct rpcrdma_ia {
61 struct rdma_cm_id *ri_id;
62 struct ib_pd *ri_pd;
63 struct ib_mr *ri_bind_mem;
64 u32 ri_dma_lkey;
65 int ri_have_dma_lkey;
66 struct completion ri_done;
67 int ri_async_rc;
68 enum rpcrdma_memreg ri_memreg_strategy;
69};
70
71
72
73
74
75struct rpcrdma_ep {
76 atomic_t rep_cqcount;
77 int rep_cqinit;
78 int rep_connected;
79 struct rpcrdma_ia *rep_ia;
80 struct ib_cq *rep_cq;
81 struct ib_qp_init_attr rep_attr;
82 wait_queue_head_t rep_connect_wait;
83 struct ib_sge rep_pad;
84 struct ib_mr *rep_pad_mr;
85 void (*rep_func)(struct rpcrdma_ep *);
86 struct rpc_xprt *rep_xprt;
87 struct rdma_conn_param rep_remote_cma;
88 struct sockaddr_storage rep_remote_addr;
89};
90
91#define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit)
92#define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount)
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112#define RPCRDMA_MAX_DATA_SEGS (8)
113#define RPCRDMA_MAX_SEGS (RPCRDMA_MAX_DATA_SEGS + 2)
114#define MAX_RPCRDMAHDR (\
115 \
116 sizeof(struct rpcrdma_msg) + (2 * sizeof(u32)) + \
117 (sizeof(struct rpcrdma_read_chunk) * RPCRDMA_MAX_SEGS) + sizeof(u32))
118
119struct rpcrdma_buffer;
120
121struct rpcrdma_rep {
122 unsigned int rr_len;
123 struct rpcrdma_buffer *rr_buffer;
124 struct rpc_xprt *rr_xprt;
125 void (*rr_func)(struct rpcrdma_rep *);
126 struct list_head rr_list;
127 wait_queue_head_t rr_unbind;
128 struct ib_sge rr_iov;
129 struct ib_mr *rr_handle;
130 char rr_base[MAX_RPCRDMAHDR];
131};
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157struct rpcrdma_mr_seg {
158 union {
159 struct ib_mr *rl_mr;
160 struct rpcrdma_mw {
161 union {
162 struct ib_mw *mw;
163 struct ib_fmr *fmr;
164 struct {
165 struct ib_fast_reg_page_list *fr_pgl;
166 struct ib_mr *fr_mr;
167 } frmr;
168 } r;
169 struct list_head mw_list;
170 } *rl_mw;
171 } mr_chunk;
172 u64 mr_base;
173 u32 mr_rkey;
174 u32 mr_len;
175 int mr_nsegs;
176 enum dma_data_direction mr_dir;
177 dma_addr_t mr_dma;
178 size_t mr_dmalen;
179 struct page *mr_page;
180 char *mr_offset;
181};
182
183struct rpcrdma_req {
184 size_t rl_size;
185 unsigned int rl_niovs;
186 unsigned int rl_nchunks;
187 unsigned int rl_connect_cookie;
188 struct rpcrdma_buffer *rl_buffer;
189 struct rpcrdma_rep *rl_reply;
190 struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS];
191 struct ib_sge rl_send_iov[4];
192 struct ib_sge rl_iov;
193 struct ib_mr *rl_handle;
194 char rl_base[MAX_RPCRDMAHDR];
195 __u32 rl_xdr_buf[0];
196};
197#define rpcr_to_rdmar(r) \
198 container_of((r)->rq_buffer, struct rpcrdma_req, rl_xdr_buf[0])
199
200
201
202
203
204
205
206struct rpcrdma_buffer {
207 spinlock_t rb_lock;
208 atomic_t rb_credits;
209 unsigned long rb_cwndscale;
210 int rb_max_requests;
211 struct list_head rb_mws;
212 int rb_send_index;
213 struct rpcrdma_req **rb_send_bufs;
214 int rb_recv_index;
215 struct rpcrdma_rep **rb_recv_bufs;
216 char *rb_pool;
217};
218#define rdmab_to_ia(b) (&container_of((b), struct rpcrdma_xprt, rx_buf)->rx_ia)
219
220
221
222
223
224
225
226struct rpcrdma_create_data_internal {
227 struct sockaddr_storage addr;
228 unsigned int max_requests;
229 unsigned int rsize;
230 unsigned int wsize;
231 unsigned int inline_rsize;
232 unsigned int inline_wsize;
233 unsigned int padding;
234};
235
236#define RPCRDMA_INLINE_READ_THRESHOLD(rq) \
237 (rpcx_to_rdmad(rq->rq_task->tk_xprt).inline_rsize)
238
239#define RPCRDMA_INLINE_WRITE_THRESHOLD(rq)\
240 (rpcx_to_rdmad(rq->rq_task->tk_xprt).inline_wsize)
241
242#define RPCRDMA_INLINE_PAD_VALUE(rq)\
243 rpcx_to_rdmad(rq->rq_task->tk_xprt).padding
244
245
246
247
248struct rpcrdma_stats {
249 unsigned long read_chunk_count;
250 unsigned long write_chunk_count;
251 unsigned long reply_chunk_count;
252
253 unsigned long long total_rdma_request;
254 unsigned long long total_rdma_reply;
255
256 unsigned long long pullup_copy_count;
257 unsigned long long fixup_copy_count;
258 unsigned long hardway_register_count;
259 unsigned long failed_marshal_count;
260 unsigned long bad_reply_count;
261};
262
263
264
265
266
267
268
269
270
271
272
273struct rpcrdma_xprt {
274 struct rpc_xprt xprt;
275 struct rpcrdma_ia rx_ia;
276 struct rpcrdma_ep rx_ep;
277 struct rpcrdma_buffer rx_buf;
278 struct rpcrdma_create_data_internal rx_data;
279 struct delayed_work rdma_connect;
280 struct rpcrdma_stats rx_stats;
281};
282
283#define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, xprt)
284#define rpcx_to_rdmad(x) (rpcx_to_rdmax(x)->rx_data)
285
286
287
288
289extern int xprt_rdma_pad_optimize;
290
291
292
293
294int rpcrdma_ia_open(struct rpcrdma_xprt *, struct sockaddr *, int);
295void rpcrdma_ia_close(struct rpcrdma_ia *);
296
297
298
299
300int rpcrdma_ep_create(struct rpcrdma_ep *, struct rpcrdma_ia *,
301 struct rpcrdma_create_data_internal *);
302int rpcrdma_ep_destroy(struct rpcrdma_ep *, struct rpcrdma_ia *);
303int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *);
304int rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *);
305
306int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *,
307 struct rpcrdma_req *);
308int rpcrdma_ep_post_recv(struct rpcrdma_ia *, struct rpcrdma_ep *,
309 struct rpcrdma_rep *);
310
311
312
313
314int rpcrdma_buffer_create(struct rpcrdma_buffer *, struct rpcrdma_ep *,
315 struct rpcrdma_ia *,
316 struct rpcrdma_create_data_internal *);
317void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
318
319struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
320void rpcrdma_buffer_put(struct rpcrdma_req *);
321void rpcrdma_recv_buffer_get(struct rpcrdma_req *);
322void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
323
324int rpcrdma_register_internal(struct rpcrdma_ia *, void *, int,
325 struct ib_mr **, struct ib_sge *);
326int rpcrdma_deregister_internal(struct rpcrdma_ia *,
327 struct ib_mr *, struct ib_sge *);
328
329int rpcrdma_register_external(struct rpcrdma_mr_seg *,
330 int, int, struct rpcrdma_xprt *);
331int rpcrdma_deregister_external(struct rpcrdma_mr_seg *,
332 struct rpcrdma_xprt *, void *);
333
334
335
336
337void rpcrdma_conn_func(struct rpcrdma_ep *);
338void rpcrdma_reply_handler(struct rpcrdma_rep *);
339
340
341
342
343int rpcrdma_marshal_req(struct rpc_rqst *);
344
345#endif
346