1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#ifndef MLX4_IB_H
35#define MLX4_IB_H
36
37#include <linux/compiler.h>
38#include <linux/list.h>
39#include <linux/mutex.h>
40
41#include <rdma/ib_verbs.h>
42#include <rdma/ib_umem.h>
43
44#include <linux/mlx4/device.h>
45#include <linux/mlx4/doorbell.h>
46
47struct mlx4_ib_ucontext {
48 struct ib_ucontext ibucontext;
49 struct mlx4_uar uar;
50 struct list_head db_page_list;
51 struct mutex db_page_mutex;
52};
53
54struct mlx4_ib_pd {
55 struct ib_pd ibpd;
56 u32 pdn;
57};
58
59struct mlx4_ib_cq_buf {
60 struct mlx4_buf buf;
61 struct mlx4_mtt mtt;
62};
63
64struct mlx4_ib_cq_resize {
65 struct mlx4_ib_cq_buf buf;
66 int cqe;
67};
68
69struct mlx4_ib_cq {
70 struct ib_cq ibcq;
71 struct mlx4_cq mcq;
72 struct mlx4_ib_cq_buf buf;
73 struct mlx4_ib_cq_resize *resize_buf;
74 struct mlx4_db db;
75 spinlock_t lock;
76 struct mutex resize_mutex;
77 struct ib_umem *umem;
78 struct ib_umem *resize_umem;
79};
80
81struct mlx4_ib_mr {
82 struct ib_mr ibmr;
83 struct mlx4_mr mmr;
84 struct ib_umem *umem;
85};
86
87struct mlx4_ib_fast_reg_page_list {
88 struct ib_fast_reg_page_list ibfrpl;
89 __be64 *mapped_page_list;
90 dma_addr_t map;
91};
92
93struct mlx4_ib_fmr {
94 struct ib_fmr ibfmr;
95 struct mlx4_fmr mfmr;
96};
97
98struct mlx4_ib_wq {
99 u64 *wrid;
100 spinlock_t lock;
101 int wqe_cnt;
102 int max_post;
103 int max_gs;
104 int offset;
105 int wqe_shift;
106 unsigned head;
107 unsigned tail;
108};
109
110enum mlx4_ib_qp_flags {
111 MLX4_IB_QP_LSO = 1 << 0,
112 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK = 1 << 1,
113};
114
115struct mlx4_ib_qp {
116 struct ib_qp ibqp;
117 struct mlx4_qp mqp;
118 struct mlx4_buf buf;
119
120 struct mlx4_db db;
121 struct mlx4_ib_wq rq;
122
123 u32 doorbell_qpn;
124 __be32 sq_signal_bits;
125 unsigned sq_next_wqe;
126 int sq_max_wqes_per_wr;
127 int sq_spare_wqes;
128 struct mlx4_ib_wq sq;
129
130 struct ib_umem *umem;
131 struct mlx4_mtt mtt;
132 int buf_size;
133 struct mutex mutex;
134 u32 flags;
135 u8 port;
136 u8 alt_port;
137 u8 atomic_rd_en;
138 u8 resp_depth;
139 u8 sq_no_prefetch;
140 u8 state;
141};
142
143struct mlx4_ib_srq {
144 struct ib_srq ibsrq;
145 struct mlx4_srq msrq;
146 struct mlx4_buf buf;
147 struct mlx4_db db;
148 u64 *wrid;
149 spinlock_t lock;
150 int head;
151 int tail;
152 u16 wqe_ctr;
153 struct ib_umem *umem;
154 struct mlx4_mtt mtt;
155 struct mutex mutex;
156};
157
158struct mlx4_ib_ah {
159 struct ib_ah ibah;
160 struct mlx4_av av;
161};
162
163struct mlx4_ib_dev {
164 struct ib_device ib_dev;
165 struct mlx4_dev *dev;
166 int num_ports;
167 void __iomem *uar_map;
168
169 struct mlx4_uar priv_uar;
170 u32 priv_pdn;
171 MLX4_DECLARE_DOORBELL_LOCK(uar_lock);
172
173 struct ib_mad_agent *send_agent[MLX4_MAX_PORTS][2];
174 struct ib_ah *sm_ah[MLX4_MAX_PORTS];
175 spinlock_t sm_lock;
176
177 struct mutex cap_mask_mutex;
178 bool ib_active;
179};
180
181static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev)
182{
183 return container_of(ibdev, struct mlx4_ib_dev, ib_dev);
184}
185
186static inline struct mlx4_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
187{
188 return container_of(ibucontext, struct mlx4_ib_ucontext, ibucontext);
189}
190
191static inline struct mlx4_ib_pd *to_mpd(struct ib_pd *ibpd)
192{
193 return container_of(ibpd, struct mlx4_ib_pd, ibpd);
194}
195
196static inline struct mlx4_ib_cq *to_mcq(struct ib_cq *ibcq)
197{
198 return container_of(ibcq, struct mlx4_ib_cq, ibcq);
199}
200
201static inline struct mlx4_ib_cq *to_mibcq(struct mlx4_cq *mcq)
202{
203 return container_of(mcq, struct mlx4_ib_cq, mcq);
204}
205
206static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr)
207{
208 return container_of(ibmr, struct mlx4_ib_mr, ibmr);
209}
210
211static inline struct mlx4_ib_fast_reg_page_list *to_mfrpl(struct ib_fast_reg_page_list *ibfrpl)
212{
213 return container_of(ibfrpl, struct mlx4_ib_fast_reg_page_list, ibfrpl);
214}
215
216static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)
217{
218 return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr);
219}
220static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp)
221{
222 return container_of(ibqp, struct mlx4_ib_qp, ibqp);
223}
224
225static inline struct mlx4_ib_qp *to_mibqp(struct mlx4_qp *mqp)
226{
227 return container_of(mqp, struct mlx4_ib_qp, mqp);
228}
229
230static inline struct mlx4_ib_srq *to_msrq(struct ib_srq *ibsrq)
231{
232 return container_of(ibsrq, struct mlx4_ib_srq, ibsrq);
233}
234
235static inline struct mlx4_ib_srq *to_mibsrq(struct mlx4_srq *msrq)
236{
237 return container_of(msrq, struct mlx4_ib_srq, msrq);
238}
239
240static inline struct mlx4_ib_ah *to_mah(struct ib_ah *ibah)
241{
242 return container_of(ibah, struct mlx4_ib_ah, ibah);
243}
244
245int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
246 struct mlx4_db *db);
247void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db);
248
249struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc);
250int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
251 struct ib_umem *umem);
252struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
253 u64 virt_addr, int access_flags,
254 struct ib_udata *udata);
255int mlx4_ib_dereg_mr(struct ib_mr *mr);
256struct ib_mr *mlx4_ib_alloc_fast_reg_mr(struct ib_pd *pd,
257 int max_page_list_len);
258struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
259 int page_list_len);
260void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list);
261
262int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
263int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
264struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,
265 struct ib_ucontext *context,
266 struct ib_udata *udata);
267int mlx4_ib_destroy_cq(struct ib_cq *cq);
268int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
269int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
270void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
271void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
272
273struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
274int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
275int mlx4_ib_destroy_ah(struct ib_ah *ah);
276
277struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
278 struct ib_srq_init_attr *init_attr,
279 struct ib_udata *udata);
280int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
281 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
282int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
283int mlx4_ib_destroy_srq(struct ib_srq *srq);
284void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index);
285int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
286 struct ib_recv_wr **bad_wr);
287
288struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
289 struct ib_qp_init_attr *init_attr,
290 struct ib_udata *udata);
291int mlx4_ib_destroy_qp(struct ib_qp *qp);
292int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
293 int attr_mask, struct ib_udata *udata);
294int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
295 struct ib_qp_init_attr *qp_init_attr);
296int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
297 struct ib_send_wr **bad_wr);
298int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
299 struct ib_recv_wr **bad_wr);
300
301int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int ignore_mkey, int ignore_bkey,
302 int port, struct ib_wc *in_wc, struct ib_grh *in_grh,
303 void *in_mad, void *response_mad);
304int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
305 struct ib_wc *in_wc, struct ib_grh *in_grh,
306 struct ib_mad *in_mad, struct ib_mad *out_mad);
307int mlx4_ib_mad_init(struct mlx4_ib_dev *dev);
308void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev);
309
310struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags,
311 struct ib_fmr_attr *fmr_attr);
312int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages,
313 u64 iova);
314int mlx4_ib_unmap_fmr(struct list_head *fmr_list);
315int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr);
316
317static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah)
318{
319 return !!(ah->av.g_slid & 0x80);
320}
321
322#endif
323
