1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43#ifndef SVC_RDMA_H
44#define SVC_RDMA_H
45#include <linux/llist.h>
46#include <linux/sunrpc/xdr.h>
47#include <linux/sunrpc/svcsock.h>
48#include <linux/sunrpc/rpc_rdma.h>
49#include <linux/sunrpc/rpc_rdma_cid.h>
50#include <linux/sunrpc/svc_rdma_pcl.h>
51
52#include <linux/percpu_counter.h>
53#include <rdma/ib_verbs.h>
54#include <rdma/rdma_cm.h>
55
56
57enum {
58 RPCRDMA_PULLUP_THRESH = RPCRDMA_V1_DEF_INLINE_SIZE >> 1,
59 RPCRDMA_DEF_INLINE_THRESH = 4096,
60 RPCRDMA_MAX_INLINE_THRESH = 65536
61};
62
63
64extern unsigned int svcrdma_ord;
65extern unsigned int svcrdma_max_requests;
66extern unsigned int svcrdma_max_bc_requests;
67extern unsigned int svcrdma_max_req_size;
68
69extern struct percpu_counter svcrdma_stat_read;
70extern struct percpu_counter svcrdma_stat_recv;
71extern struct percpu_counter svcrdma_stat_sq_starve;
72extern struct percpu_counter svcrdma_stat_write;
73extern atomic_t rdma_stat_rq_starve;
74extern atomic_t rdma_stat_rq_poll;
75extern atomic_t rdma_stat_rq_prod;
76extern atomic_t rdma_stat_sq_poll;
77extern atomic_t rdma_stat_sq_prod;
78
79struct svcxprt_rdma {
80 struct svc_xprt sc_xprt;
81 struct rdma_cm_id *sc_cm_id;
82 struct list_head sc_accept_q;
83 int sc_ord;
84 int sc_max_send_sges;
85 bool sc_snd_w_inv;
86
87 atomic_t sc_sq_avail;
88 unsigned int sc_sq_depth;
89 __be32 sc_fc_credits;
90 u32 sc_max_requests;
91 u32 sc_max_bc_requests;
92 int sc_max_req_size;
93 u8 sc_port_num;
94
95 struct ib_pd *sc_pd;
96
97 spinlock_t sc_send_lock;
98 struct list_head sc_send_ctxts;
99 spinlock_t sc_rw_ctxt_lock;
100 struct list_head sc_rw_ctxts;
101
102 u32 sc_pending_recvs;
103 u32 sc_recv_batch;
104 struct list_head sc_rq_dto_q;
105 spinlock_t sc_rq_dto_lock;
106 struct ib_qp *sc_qp;
107 struct ib_cq *sc_rq_cq;
108 struct ib_cq *sc_sq_cq;
109
110 spinlock_t sc_lock;
111
112 wait_queue_head_t sc_send_wait;
113 unsigned long sc_flags;
114 struct work_struct sc_work;
115
116 struct llist_head sc_recv_ctxts;
117
118 atomic_t sc_completion_ids;
119};
120
121#define RDMAXPRT_CONN_PENDING 3
122
123
124
125
126enum {
127 RPCRDMA_LISTEN_BACKLOG = 10,
128 RPCRDMA_MAX_REQUESTS = 64,
129 RPCRDMA_MAX_BC_REQUESTS = 2,
130};
131
132#define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
133
134struct svc_rdma_recv_ctxt {
135 struct llist_node rc_node;
136 struct list_head rc_list;
137 struct ib_recv_wr rc_recv_wr;
138 struct ib_cqe rc_cqe;
139 struct rpc_rdma_cid rc_cid;
140 struct ib_sge rc_recv_sge;
141 void *rc_recv_buf;
142 struct xdr_stream rc_stream;
143 bool rc_temp;
144 u32 rc_byte_len;
145 unsigned int rc_page_count;
146 u32 rc_inv_rkey;
147 __be32 rc_msgtype;
148
149 struct svc_rdma_pcl rc_call_pcl;
150
151 struct svc_rdma_pcl rc_read_pcl;
152 struct svc_rdma_chunk *rc_cur_result_payload;
153 struct svc_rdma_pcl rc_write_pcl;
154 struct svc_rdma_pcl rc_reply_pcl;
155};
156
157struct svc_rdma_send_ctxt {
158 struct list_head sc_list;
159 struct rpc_rdma_cid sc_cid;
160
161 struct ib_send_wr sc_send_wr;
162 struct ib_cqe sc_cqe;
163 struct completion sc_done;
164 struct xdr_buf sc_hdrbuf;
165 struct xdr_stream sc_stream;
166 void *sc_xprt_buf;
167 int sc_cur_sge_no;
168
169 struct ib_sge sc_sges[];
170};
171
172
173extern void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp,
174 struct svc_rdma_recv_ctxt *rctxt);
175
176
177extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma);
178extern bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma);
179extern struct svc_rdma_recv_ctxt *
180 svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma);
181extern void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,
182 struct svc_rdma_recv_ctxt *ctxt);
183extern void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma);
184extern void svc_rdma_release_rqst(struct svc_rqst *rqstp);
185extern int svc_rdma_recvfrom(struct svc_rqst *);
186
187
188extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
189extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
190 const struct svc_rdma_chunk *chunk,
191 const struct xdr_buf *xdr);
192extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
193 const struct svc_rdma_recv_ctxt *rctxt,
194 const struct xdr_buf *xdr);
195extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
196 struct svc_rqst *rqstp,
197 struct svc_rdma_recv_ctxt *head);
198
199
200extern void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma);
201extern struct svc_rdma_send_ctxt *
202 svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma);
203extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma,
204 struct svc_rdma_send_ctxt *ctxt);
205extern int svc_rdma_send(struct svcxprt_rdma *rdma,
206 struct svc_rdma_send_ctxt *ctxt);
207extern int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma,
208 struct svc_rdma_send_ctxt *sctxt,
209 const struct svc_rdma_recv_ctxt *rctxt,
210 const struct xdr_buf *xdr);
211extern void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma,
212 struct svc_rdma_send_ctxt *sctxt,
213 struct svc_rdma_recv_ctxt *rctxt,
214 int status);
215extern int svc_rdma_sendto(struct svc_rqst *);
216extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset,
217 unsigned int length);
218
219
220extern struct svc_xprt_class svc_rdma_class;
221#ifdef CONFIG_SUNRPC_BACKCHANNEL
222extern struct svc_xprt_class svc_rdma_bc_class;
223#endif
224
225
226extern int svc_rdma_init(void);
227extern void svc_rdma_cleanup(void);
228
229#endif
230