1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#ifndef __QEDR_H__
33#define __QEDR_H__
34
35#include <linux/pci.h>
36#include <linux/idr.h>
37#include <rdma/ib_addr.h>
38#include <linux/qed/qed_if.h>
39#include <linux/qed/qed_chain.h>
40#include <linux/qed/qed_rdma_if.h>
41#include <linux/qed/qede_rdma.h>
42#include <linux/qed/roce_common.h>
43#include "qedr_hsi_rdma.h"
44
45#define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
46#define DP_NAME(dev) ((dev)->ibdev.name)
47#define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
48#define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
49
50#define DP_DEBUG(dev, module, fmt, ...) \
51 pr_debug("(%s) " module ": " fmt, \
52 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
53
54#define QEDR_MSG_INIT "INIT"
55#define QEDR_MSG_MISC "MISC"
56#define QEDR_MSG_CQ " CQ"
57#define QEDR_MSG_MR " MR"
58#define QEDR_MSG_RQ " RQ"
59#define QEDR_MSG_SQ " SQ"
60#define QEDR_MSG_QP " QP"
61#define QEDR_MSG_GSI " GSI"
62#define QEDR_MSG_IWARP " IW"
63
64#define QEDR_CQ_MAGIC_NUMBER (0x11223344)
65
66#define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE)
67#define FW_PAGE_SHIFT (12)
68
69struct qedr_dev;
70
71struct qedr_cnq {
72 struct qedr_dev *dev;
73 struct qed_chain pbl;
74 struct qed_sb_info *sb;
75 char name[32];
76 u64 n_comp;
77 __le16 *hw_cons_ptr;
78 u8 index;
79};
80
81#define QEDR_MAX_SGID 128
82
83struct qedr_device_attr {
84 u32 vendor_id;
85 u32 vendor_part_id;
86 u32 hw_ver;
87 u64 fw_ver;
88 u64 node_guid;
89 u64 sys_image_guid;
90 u8 max_cnq;
91 u8 max_sge;
92 u16 max_inline;
93 u32 max_sqe;
94 u32 max_rqe;
95 u8 max_qp_resp_rd_atomic_resc;
96 u8 max_qp_req_rd_atomic_resc;
97 u64 max_dev_resp_rd_atomic_resc;
98 u32 max_cq;
99 u32 max_qp;
100 u32 max_mr;
101 u64 max_mr_size;
102 u32 max_cqe;
103 u32 max_mw;
104 u32 max_fmr;
105 u32 max_mr_mw_fmr_pbl;
106 u64 max_mr_mw_fmr_size;
107 u32 max_pd;
108 u32 max_ah;
109 u8 max_pkey;
110 u32 max_srq;
111 u32 max_srq_wr;
112 u8 max_srq_sge;
113 u8 max_stats_queues;
114 u32 dev_caps;
115
116 u64 page_size_caps;
117 u8 dev_ack_delay;
118 u32 reserved_lkey;
119 u32 bad_pkey_counter;
120 struct qed_rdma_events events;
121};
122
123#define QEDR_ENET_STATE_BIT (0)
124
125struct qedr_dev {
126 struct ib_device ibdev;
127 struct qed_dev *cdev;
128 struct pci_dev *pdev;
129 struct net_device *ndev;
130
131 enum ib_atomic_cap atomic_cap;
132
133 void *rdma_ctx;
134 struct qedr_device_attr attr;
135
136 const struct qed_rdma_ops *ops;
137 struct qed_int_info int_info;
138
139 struct qed_sb_info *sb_array;
140 struct qedr_cnq *cnq_array;
141 int num_cnq;
142 int sb_start;
143
144 void __iomem *db_addr;
145 u64 db_phys_addr;
146 u32 db_size;
147 u16 dpi;
148
149 union ib_gid *sgid_tbl;
150
151
152 spinlock_t sgid_lock;
153
154 u64 guid;
155
156 u32 dp_module;
157 u8 dp_level;
158 u8 num_hwfns;
159 u8 gsi_ll2_handle;
160
161 uint wq_multiplier;
162 u8 gsi_ll2_mac_address[ETH_ALEN];
163 int gsi_qp_created;
164 struct qedr_cq *gsi_sqcq;
165 struct qedr_cq *gsi_rqcq;
166 struct qedr_qp *gsi_qp;
167 enum qed_rdma_type rdma_type;
168 spinlock_t idr_lock;
169 struct idr qpidr;
170 struct workqueue_struct *iwarp_wq;
171 u16 iwarp_max_mtu;
172
173 unsigned long enet_state;
174
175 u8 user_dpm_enabled;
176};
177
178#define QEDR_MAX_SQ_PBL (0x8000)
179#define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *))
180#define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
181#define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
182 QEDR_SQE_ELEMENT_SIZE)
183#define QEDR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
184 QEDR_SQE_ELEMENT_SIZE)
185#define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\
186 (RDMA_RING_PAGE_SIZE) / \
187 (QEDR_SQE_ELEMENT_SIZE) /\
188 (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
189
190#define QEDR_MAX_RQ_PBL (0x2000)
191#define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *))
192#define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
193#define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE)
194#define QEDR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
195 QEDR_RQE_ELEMENT_SIZE)
196#define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\
197 (RDMA_RING_PAGE_SIZE) / \
198 (QEDR_RQE_ELEMENT_SIZE) /\
199 (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
200
201#define QEDR_CQE_SIZE (sizeof(union rdma_cqe))
202#define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
203#define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
204 sizeof(u64)) - 1)
205#define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
206 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
207
208#define QEDR_ROCE_MAX_CNQ_SIZE (0x4000)
209
210#define QEDR_MAX_PORT (1)
211#define QEDR_PORT (1)
212
213#define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
214
215#define QEDR_ROCE_PKEY_MAX 1
216#define QEDR_ROCE_PKEY_TABLE_LEN 1
217#define QEDR_ROCE_PKEY_DEFAULT 0xffff
218
219struct qedr_pbl {
220 struct list_head list_entry;
221 void *va;
222 dma_addr_t pa;
223};
224
225struct qedr_ucontext {
226 struct ib_ucontext ibucontext;
227 struct qedr_dev *dev;
228 struct qedr_pd *pd;
229 u64 dpi_addr;
230 u64 dpi_phys_addr;
231 u32 dpi_size;
232 u16 dpi;
233
234 struct list_head mm_head;
235
236
237 struct mutex mm_list_lock;
238};
239
240union db_prod64 {
241 struct rdma_pwm_val32_data data;
242 u64 raw;
243};
244
245enum qedr_cq_type {
246 QEDR_CQ_TYPE_GSI,
247 QEDR_CQ_TYPE_KERNEL,
248 QEDR_CQ_TYPE_USER,
249};
250
251struct qedr_pbl_info {
252 u32 num_pbls;
253 u32 num_pbes;
254 u32 pbl_size;
255 u32 pbe_size;
256 bool two_layered;
257};
258
259struct qedr_userq {
260 struct ib_umem *umem;
261 struct qedr_pbl_info pbl_info;
262 struct qedr_pbl *pbl_tbl;
263 u64 buf_addr;
264 size_t buf_len;
265};
266
267struct qedr_cq {
268 struct ib_cq ibcq;
269
270 enum qedr_cq_type cq_type;
271 u32 sig;
272
273 u16 icid;
274
275
276 spinlock_t cq_lock;
277 u8 arm_flags;
278 struct qed_chain pbl;
279
280 void __iomem *db_addr;
281 union db_prod64 db;
282
283 u8 pbl_toggle;
284 union rdma_cqe *latest_cqe;
285 union rdma_cqe *toggle_cqe;
286
287 u32 cq_cons;
288
289 struct qedr_userq q;
290 u8 destroyed;
291 u16 cnq_notif;
292};
293
294struct qedr_pd {
295 struct ib_pd ibpd;
296 u32 pd_id;
297 struct qedr_ucontext *uctx;
298};
299
300struct qedr_mm {
301 struct {
302 u64 phy_addr;
303 unsigned long len;
304 } key;
305 struct list_head entry;
306};
307
308union db_prod32 {
309 struct rdma_pwm_val16_data data;
310 u32 raw;
311};
312
313struct qedr_qp_hwq_info {
314
315 struct qed_chain pbl;
316 u64 p_phys_addr_tbl;
317 u32 max_sges;
318
319
320 u16 prod;
321 u16 cons;
322 u16 wqe_cons;
323 u16 gsi_cons;
324 u16 max_wr;
325
326
327 void __iomem *db;
328 union db_prod32 db_data;
329
330 void __iomem *iwarp_db2;
331 union db_prod32 iwarp_db2_data;
332};
333
334#define QEDR_INC_SW_IDX(p_info, index) \
335 do { \
336 p_info->index = (p_info->index + 1) & \
337 qed_chain_get_capacity(p_info->pbl) \
338 } while (0)
339
340enum qedr_qp_err_bitmap {
341 QEDR_QP_ERR_SQ_FULL = 1,
342 QEDR_QP_ERR_RQ_FULL = 2,
343 QEDR_QP_ERR_BAD_SR = 4,
344 QEDR_QP_ERR_BAD_RR = 8,
345 QEDR_QP_ERR_SQ_PBL_FULL = 16,
346 QEDR_QP_ERR_RQ_PBL_FULL = 32,
347};
348
349struct qedr_qp {
350 struct ib_qp ibqp;
351 struct qedr_dev *dev;
352 struct qedr_iw_ep *ep;
353 struct qedr_qp_hwq_info sq;
354 struct qedr_qp_hwq_info rq;
355
356 u32 max_inline_data;
357
358
359 spinlock_t q_lock;
360 struct qedr_cq *sq_cq;
361 struct qedr_cq *rq_cq;
362 struct qedr_srq *srq;
363 enum qed_roce_qp_state state;
364 u32 id;
365 struct qedr_pd *pd;
366 enum ib_qp_type qp_type;
367 struct qed_rdma_qp *qed_qp;
368 u32 qp_id;
369 u16 icid;
370 u16 mtu;
371 int sgid_idx;
372 u32 rq_psn;
373 u32 sq_psn;
374 u32 qkey;
375 u32 dest_qp_num;
376
377
378 u8 prev_wqe_size;
379 u16 wqe_cons;
380 u32 err_bitmap;
381 bool signaled;
382
383
384 struct {
385 u64 wr_id;
386 enum ib_wc_opcode opcode;
387 u32 bytes_len;
388 u8 wqe_size;
389 bool signaled;
390 dma_addr_t icrc_mapping;
391 u32 *icrc;
392 struct qedr_mr *mr;
393 } *wqe_wr_id;
394
395
396 struct {
397 u64 wr_id;
398 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
399 u8 wqe_size;
400
401 u8 smac[ETH_ALEN];
402 u16 vlan;
403 int rc;
404 } *rqe_wr_id;
405
406
407 struct qedr_userq usq;
408 struct qedr_userq urq;
409 atomic_t refcnt;
410 bool destroyed;
411};
412
413struct qedr_ah {
414 struct ib_ah ibah;
415 struct rdma_ah_attr attr;
416};
417
418enum qedr_mr_type {
419 QEDR_MR_USER,
420 QEDR_MR_KERNEL,
421 QEDR_MR_DMA,
422 QEDR_MR_FRMR,
423};
424
425struct mr_info {
426 struct qedr_pbl *pbl_table;
427 struct qedr_pbl_info pbl_info;
428 struct list_head free_pbl_list;
429 struct list_head inuse_pbl_list;
430 u32 completed;
431 u32 completed_handled;
432};
433
434struct qedr_mr {
435 struct ib_mr ibmr;
436 struct ib_umem *umem;
437
438 struct qed_rdma_register_tid_in_params hw_mr;
439 enum qedr_mr_type type;
440
441 struct qedr_dev *dev;
442 struct mr_info info;
443
444 u64 *pages;
445 u32 npages;
446};
447
448#define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
449
450#define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
451 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
452#define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
453 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
454#define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
455 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
456
457static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
458{
459 info->cons = (info->cons + 1) % info->max_wr;
460 info->wqe_cons++;
461}
462
463static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
464{
465 info->prod = (info->prod + 1) % info->max_wr;
466}
467
468static inline int qedr_get_dmac(struct qedr_dev *dev,
469 struct rdma_ah_attr *ah_attr, u8 *mac_addr)
470{
471 union ib_gid zero_sgid = { { 0 } };
472 struct in6_addr in6;
473 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
474 u8 *dmac;
475
476 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
477 DP_ERR(dev, "Local port GID not supported\n");
478 eth_zero_addr(mac_addr);
479 return -EINVAL;
480 }
481
482 memcpy(&in6, grh->dgid.raw, sizeof(in6));
483 dmac = rdma_ah_retrieve_dmac(ah_attr);
484 if (!dmac)
485 return -EINVAL;
486 ether_addr_copy(mac_addr, dmac);
487
488 return 0;
489}
490
491struct qedr_iw_listener {
492 struct qedr_dev *dev;
493 struct iw_cm_id *cm_id;
494 int backlog;
495 void *qed_handle;
496};
497
498struct qedr_iw_ep {
499 struct qedr_dev *dev;
500 struct iw_cm_id *cm_id;
501 struct qedr_qp *qp;
502 void *qed_context;
503 u8 during_connect;
504};
505
506static inline
507struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
508{
509 return container_of(ibucontext, struct qedr_ucontext, ibucontext);
510}
511
512static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
513{
514 return container_of(ibdev, struct qedr_dev, ibdev);
515}
516
517static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
518{
519 return container_of(ibpd, struct qedr_pd, ibpd);
520}
521
522static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
523{
524 return container_of(ibcq, struct qedr_cq, ibcq);
525}
526
527static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
528{
529 return container_of(ibqp, struct qedr_qp, ibqp);
530}
531
532static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
533{
534 return container_of(ibah, struct qedr_ah, ibah);
535}
536
537static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
538{
539 return container_of(ibmr, struct qedr_mr, ibmr);
540}
541#endif
542