1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#ifndef RXE_VERBS_H
35#define RXE_VERBS_H
36
37#include <linux/interrupt.h>
38#include <linux/workqueue.h>
39#include <rdma/rdma_user_rxe.h>
40#include "rxe_pool.h"
41#include "rxe_task.h"
42#include "rxe_hw_counters.h"
43
44static inline int pkey_match(u16 key1, u16 key2)
45{
46 return (((key1 & 0x7fff) != 0) &&
47 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
48 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
49}
50
51
52
53
54
55static inline int psn_compare(u32 psn_a, u32 psn_b)
56{
57 s32 diff;
58
59 diff = (psn_a - psn_b) << 8;
60 return diff;
61}
62
63struct rxe_ucontext {
64 struct ib_ucontext ibuc;
65 struct rxe_pool_entry pelem;
66};
67
68struct rxe_pd {
69 struct ib_pd ibpd;
70 struct rxe_pool_entry pelem;
71};
72
73struct rxe_ah {
74 struct ib_ah ibah;
75 struct rxe_pool_entry pelem;
76 struct rxe_pd *pd;
77 struct rxe_av av;
78};
79
80struct rxe_cqe {
81 union {
82 struct ib_wc ibwc;
83 struct ib_uverbs_wc uibwc;
84 };
85};
86
87struct rxe_cq {
88 struct ib_cq ibcq;
89 struct rxe_pool_entry pelem;
90 struct rxe_queue *queue;
91 spinlock_t cq_lock;
92 u8 notify;
93 bool is_dying;
94 int is_user;
95 struct tasklet_struct comp_task;
96};
97
98enum wqe_state {
99 wqe_state_posted,
100 wqe_state_processing,
101 wqe_state_pending,
102 wqe_state_done,
103 wqe_state_error,
104};
105
106struct rxe_sq {
107 int max_wr;
108 int max_sge;
109 int max_inline;
110 spinlock_t sq_lock;
111 struct rxe_queue *queue;
112};
113
114struct rxe_rq {
115 int max_wr;
116 int max_sge;
117 spinlock_t producer_lock;
118 spinlock_t consumer_lock;
119 struct rxe_queue *queue;
120};
121
122struct rxe_srq {
123 struct ib_srq ibsrq;
124 struct rxe_pool_entry pelem;
125 struct rxe_pd *pd;
126 struct rxe_rq rq;
127 u32 srq_num;
128
129 int limit;
130 int error;
131};
132
133enum rxe_qp_state {
134 QP_STATE_RESET,
135 QP_STATE_INIT,
136 QP_STATE_READY,
137 QP_STATE_DRAIN,
138 QP_STATE_DRAINED,
139 QP_STATE_ERROR
140};
141
142struct rxe_req_info {
143 enum rxe_qp_state state;
144 int wqe_index;
145 u32 psn;
146 int opcode;
147 atomic_t rd_atomic;
148 int wait_fence;
149 int need_rd_atomic;
150 int wait_psn;
151 int need_retry;
152 int noack_pkts;
153 struct rxe_task task;
154};
155
156struct rxe_comp_info {
157 u32 psn;
158 int opcode;
159 int timeout;
160 int timeout_retry;
161 int started_retry;
162 u32 retry_cnt;
163 u32 rnr_retry;
164 struct rxe_task task;
165};
166
167enum rdatm_res_state {
168 rdatm_res_state_next,
169 rdatm_res_state_new,
170 rdatm_res_state_replay,
171};
172
173struct resp_res {
174 int type;
175 int replay;
176 u32 first_psn;
177 u32 last_psn;
178 u32 cur_psn;
179 enum rdatm_res_state state;
180
181 union {
182 struct {
183 struct sk_buff *skb;
184 } atomic;
185 struct {
186 struct rxe_mem *mr;
187 u64 va_org;
188 u32 rkey;
189 u32 length;
190 u64 va;
191 u32 resid;
192 } read;
193 };
194};
195
196struct rxe_resp_info {
197 enum rxe_qp_state state;
198 u32 msn;
199 u32 psn;
200 u32 ack_psn;
201 int opcode;
202 int drop_msg;
203 int goto_error;
204 int sent_psn_nak;
205 enum ib_wc_status status;
206 u8 aeth_syndrome;
207
208
209 struct rxe_recv_wqe *wqe;
210
211
212 u64 va;
213 struct rxe_mem *mr;
214 u32 resid;
215 u32 rkey;
216 u32 length;
217 u64 atomic_orig;
218
219
220 struct {
221 struct rxe_recv_wqe wqe;
222 struct ib_sge sge[RXE_MAX_SGE];
223 } srq_wqe;
224
225
226
227
228 struct resp_res *resources;
229 unsigned int res_head;
230 unsigned int res_tail;
231 struct resp_res *res;
232 struct rxe_task task;
233};
234
235struct rxe_qp {
236 struct rxe_pool_entry pelem;
237 struct ib_qp ibqp;
238 struct ib_qp_attr attr;
239 unsigned int valid;
240 unsigned int mtu;
241 int is_user;
242
243 struct rxe_pd *pd;
244 struct rxe_srq *srq;
245 struct rxe_cq *scq;
246 struct rxe_cq *rcq;
247
248 enum ib_sig_type sq_sig_type;
249
250 struct rxe_sq sq;
251 struct rxe_rq rq;
252
253 struct socket *sk;
254 u32 dst_cookie;
255 u16 src_port;
256
257 struct rxe_av pri_av;
258 struct rxe_av alt_av;
259
260
261 struct list_head grp_list;
262 spinlock_t grp_lock;
263
264 struct sk_buff_head req_pkts;
265 struct sk_buff_head resp_pkts;
266 struct sk_buff_head send_pkts;
267
268 struct rxe_req_info req;
269 struct rxe_comp_info comp;
270 struct rxe_resp_info resp;
271
272 atomic_t ssn;
273 atomic_t skb_out;
274 int need_req_skb;
275
276
277
278
279
280
281 struct timer_list retrans_timer;
282 u64 qp_timeout_jiffies;
283
284
285 struct timer_list rnr_nak_timer;
286
287 spinlock_t state_lock;
288
289 struct execute_work cleanup_work;
290};
291
292enum rxe_mem_state {
293 RXE_MEM_STATE_ZOMBIE,
294 RXE_MEM_STATE_INVALID,
295 RXE_MEM_STATE_FREE,
296 RXE_MEM_STATE_VALID,
297};
298
299enum rxe_mem_type {
300 RXE_MEM_TYPE_NONE,
301 RXE_MEM_TYPE_DMA,
302 RXE_MEM_TYPE_MR,
303 RXE_MEM_TYPE_FMR,
304 RXE_MEM_TYPE_MW,
305};
306
307#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
308
309struct rxe_phys_buf {
310 u64 addr;
311 u64 size;
312};
313
314struct rxe_map {
315 struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
316};
317
318struct rxe_mem {
319 struct rxe_pool_entry pelem;
320 union {
321 struct ib_mr ibmr;
322 struct ib_mw ibmw;
323 };
324
325 struct rxe_pd *pd;
326 struct ib_umem *umem;
327
328 u32 lkey;
329 u32 rkey;
330
331 enum rxe_mem_state state;
332 enum rxe_mem_type type;
333 u64 va;
334 u64 iova;
335 size_t length;
336 u32 offset;
337 int access;
338
339 int page_shift;
340 int page_mask;
341 int map_shift;
342 int map_mask;
343
344 u32 num_buf;
345 u32 nbuf;
346
347 u32 max_buf;
348 u32 num_map;
349
350 struct rxe_map **map;
351};
352
353struct rxe_mc_grp {
354 struct rxe_pool_entry pelem;
355 spinlock_t mcg_lock;
356 struct rxe_dev *rxe;
357 struct list_head qp_list;
358 union ib_gid mgid;
359 int num_qp;
360 u32 qkey;
361 u16 pkey;
362};
363
364struct rxe_mc_elem {
365 struct rxe_pool_entry pelem;
366 struct list_head qp_list;
367 struct list_head grp_list;
368 struct rxe_qp *qp;
369 struct rxe_mc_grp *grp;
370};
371
372struct rxe_port {
373 struct ib_port_attr attr;
374 u16 *pkey_tbl;
375 __be64 port_guid;
376 __be64 subnet_prefix;
377 spinlock_t port_lock;
378 unsigned int mtu_cap;
379
380 u32 qp_smi_index;
381 u32 qp_gsi_index;
382};
383
384struct rxe_dev {
385 struct ib_device ib_dev;
386 struct ib_device_attr attr;
387 int max_ucontext;
388 int max_inline_data;
389 struct mutex usdev_lock;
390
391 struct net_device *ndev;
392
393 int xmit_errors;
394
395 struct rxe_pool uc_pool;
396 struct rxe_pool pd_pool;
397 struct rxe_pool ah_pool;
398 struct rxe_pool srq_pool;
399 struct rxe_pool qp_pool;
400 struct rxe_pool cq_pool;
401 struct rxe_pool mr_pool;
402 struct rxe_pool mw_pool;
403 struct rxe_pool mc_grp_pool;
404 struct rxe_pool mc_elem_pool;
405
406 spinlock_t pending_lock;
407 struct list_head pending_mmaps;
408
409 spinlock_t mmap_offset_lock;
410 int mmap_offset;
411
412 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
413
414 struct rxe_port port;
415 struct crypto_shash *tfm;
416};
417
418static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
419{
420 atomic64_inc(&rxe->stats_counters[index]);
421}
422
423static inline struct rxe_dev *to_rdev(struct ib_device *dev)
424{
425 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
426}
427
428static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
429{
430 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
431}
432
433static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
434{
435 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
436}
437
438static inline struct rxe_ah *to_rah(struct ib_ah *ah)
439{
440 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
441}
442
443static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
444{
445 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
446}
447
448static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
449{
450 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
451}
452
453static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
454{
455 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
456}
457
458static inline struct rxe_mem *to_rmr(struct ib_mr *mr)
459{
460 return mr ? container_of(mr, struct rxe_mem, ibmr) : NULL;
461}
462
463static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
464{
465 return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL;
466}
467
468int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
469
470void rxe_mc_cleanup(struct rxe_pool_entry *arg);
471
472#endif
473
