1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#ifndef __QEDR_H__
33#define __QEDR_H__
34
35#include <linux/pci.h>
36#include <linux/idr.h>
37#include <rdma/ib_addr.h>
38#include <linux/qed/qed_if.h>
39#include <linux/qed/qed_chain.h>
40#include <linux/qed/qed_rdma_if.h>
41#include <linux/qed/qede_rdma.h>
42#include <linux/qed/roce_common.h>
43#include "qedr_hsi_rdma.h"
44
45#define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
46#define DP_NAME(dev) ((dev)->ibdev.name)
47#define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
48#define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
49
50#define DP_DEBUG(dev, module, fmt, ...) \
51 pr_debug("(%s) " module ": " fmt, \
52 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
53
54#define QEDR_MSG_INIT "INIT"
55#define QEDR_MSG_MISC "MISC"
56#define QEDR_MSG_CQ " CQ"
57#define QEDR_MSG_MR " MR"
58#define QEDR_MSG_RQ " RQ"
59#define QEDR_MSG_SQ " SQ"
60#define QEDR_MSG_QP " QP"
61#define QEDR_MSG_SRQ " SRQ"
62#define QEDR_MSG_GSI " GSI"
63#define QEDR_MSG_IWARP " IW"
64
65#define QEDR_CQ_MAGIC_NUMBER (0x11223344)
66
67#define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE)
68#define FW_PAGE_SHIFT (12)
69
70struct qedr_dev;
71
72struct qedr_cnq {
73 struct qedr_dev *dev;
74 struct qed_chain pbl;
75 struct qed_sb_info *sb;
76 char name[32];
77 u64 n_comp;
78 __le16 *hw_cons_ptr;
79 u8 index;
80};
81
82#define QEDR_MAX_SGID 128
83
84struct qedr_device_attr {
85 u32 vendor_id;
86 u32 vendor_part_id;
87 u32 hw_ver;
88 u64 fw_ver;
89 u64 node_guid;
90 u64 sys_image_guid;
91 u8 max_cnq;
92 u8 max_sge;
93 u16 max_inline;
94 u32 max_sqe;
95 u32 max_rqe;
96 u8 max_qp_resp_rd_atomic_resc;
97 u8 max_qp_req_rd_atomic_resc;
98 u64 max_dev_resp_rd_atomic_resc;
99 u32 max_cq;
100 u32 max_qp;
101 u32 max_mr;
102 u64 max_mr_size;
103 u32 max_cqe;
104 u32 max_mw;
105 u32 max_fmr;
106 u32 max_mr_mw_fmr_pbl;
107 u64 max_mr_mw_fmr_size;
108 u32 max_pd;
109 u32 max_ah;
110 u8 max_pkey;
111 u32 max_srq;
112 u32 max_srq_wr;
113 u8 max_srq_sge;
114 u8 max_stats_queues;
115 u32 dev_caps;
116
117 u64 page_size_caps;
118 u8 dev_ack_delay;
119 u32 reserved_lkey;
120 u32 bad_pkey_counter;
121 struct qed_rdma_events events;
122};
123
124#define QEDR_ENET_STATE_BIT (0)
125
126struct qedr_idr {
127 spinlock_t idr_lock;
128 struct idr idr;
129};
130
131struct qedr_dev {
132 struct ib_device ibdev;
133 struct qed_dev *cdev;
134 struct pci_dev *pdev;
135 struct net_device *ndev;
136
137 enum ib_atomic_cap atomic_cap;
138
139 void *rdma_ctx;
140 struct qedr_device_attr attr;
141
142 const struct qed_rdma_ops *ops;
143 struct qed_int_info int_info;
144
145 struct qed_sb_info *sb_array;
146 struct qedr_cnq *cnq_array;
147 int num_cnq;
148 int sb_start;
149
150 void __iomem *db_addr;
151 u64 db_phys_addr;
152 u32 db_size;
153 u16 dpi;
154
155 union ib_gid *sgid_tbl;
156
157
158 spinlock_t sgid_lock;
159
160 u64 guid;
161
162 u32 dp_module;
163 u8 dp_level;
164 u8 num_hwfns;
165 u8 gsi_ll2_handle;
166
167 uint wq_multiplier;
168 u8 gsi_ll2_mac_address[ETH_ALEN];
169 int gsi_qp_created;
170 struct qedr_cq *gsi_sqcq;
171 struct qedr_cq *gsi_rqcq;
172 struct qedr_qp *gsi_qp;
173 enum qed_rdma_type rdma_type;
174 struct qedr_idr qpidr;
175 struct qedr_idr srqidr;
176 struct workqueue_struct *iwarp_wq;
177 u16 iwarp_max_mtu;
178
179 unsigned long enet_state;
180
181 u8 user_dpm_enabled;
182};
183
184#define QEDR_MAX_SQ_PBL (0x8000)
185#define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *))
186#define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
187#define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
188 QEDR_SQE_ELEMENT_SIZE)
189#define QEDR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
190 QEDR_SQE_ELEMENT_SIZE)
191#define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\
192 (RDMA_RING_PAGE_SIZE) / \
193 (QEDR_SQE_ELEMENT_SIZE) /\
194 (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
195
196#define QEDR_MAX_RQ_PBL (0x2000)
197#define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *))
198#define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
199#define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE)
200#define QEDR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
201 QEDR_RQE_ELEMENT_SIZE)
202#define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\
203 (RDMA_RING_PAGE_SIZE) / \
204 (QEDR_RQE_ELEMENT_SIZE) /\
205 (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
206
207#define QEDR_CQE_SIZE (sizeof(union rdma_cqe))
208#define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
209#define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
210 sizeof(u64)) - 1)
211#define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
212 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
213
214#define QEDR_ROCE_MAX_CNQ_SIZE (0x4000)
215
216#define QEDR_MAX_PORT (1)
217#define QEDR_PORT (1)
218
219#define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
220
221#define QEDR_ROCE_PKEY_MAX 1
222#define QEDR_ROCE_PKEY_TABLE_LEN 1
223#define QEDR_ROCE_PKEY_DEFAULT 0xffff
224
225struct qedr_pbl {
226 struct list_head list_entry;
227 void *va;
228 dma_addr_t pa;
229};
230
231struct qedr_ucontext {
232 struct ib_ucontext ibucontext;
233 struct qedr_dev *dev;
234 struct qedr_pd *pd;
235 u64 dpi_addr;
236 u64 dpi_phys_addr;
237 u32 dpi_size;
238 u16 dpi;
239
240 struct list_head mm_head;
241
242
243 struct mutex mm_list_lock;
244};
245
246union db_prod64 {
247 struct rdma_pwm_val32_data data;
248 u64 raw;
249};
250
251enum qedr_cq_type {
252 QEDR_CQ_TYPE_GSI,
253 QEDR_CQ_TYPE_KERNEL,
254 QEDR_CQ_TYPE_USER,
255};
256
257struct qedr_pbl_info {
258 u32 num_pbls;
259 u32 num_pbes;
260 u32 pbl_size;
261 u32 pbe_size;
262 bool two_layered;
263};
264
265struct qedr_userq {
266 struct ib_umem *umem;
267 struct qedr_pbl_info pbl_info;
268 struct qedr_pbl *pbl_tbl;
269 u64 buf_addr;
270 size_t buf_len;
271};
272
273struct qedr_cq {
274 struct ib_cq ibcq;
275
276 enum qedr_cq_type cq_type;
277 u32 sig;
278
279 u16 icid;
280
281
282 spinlock_t cq_lock;
283 u8 arm_flags;
284 struct qed_chain pbl;
285
286 void __iomem *db_addr;
287 union db_prod64 db;
288
289 u8 pbl_toggle;
290 union rdma_cqe *latest_cqe;
291 union rdma_cqe *toggle_cqe;
292
293 u32 cq_cons;
294
295 struct qedr_userq q;
296 u8 destroyed;
297 u16 cnq_notif;
298};
299
300struct qedr_pd {
301 struct ib_pd ibpd;
302 u32 pd_id;
303 struct qedr_ucontext *uctx;
304};
305
306struct qedr_mm {
307 struct {
308 u64 phy_addr;
309 unsigned long len;
310 } key;
311 struct list_head entry;
312};
313
314union db_prod32 {
315 struct rdma_pwm_val16_data data;
316 u32 raw;
317};
318
319struct qedr_qp_hwq_info {
320
321 struct qed_chain pbl;
322 u64 p_phys_addr_tbl;
323 u32 max_sges;
324
325
326 u16 prod;
327 u16 cons;
328 u16 wqe_cons;
329 u16 gsi_cons;
330 u16 max_wr;
331
332
333 void __iomem *db;
334 union db_prod32 db_data;
335
336 void __iomem *iwarp_db2;
337 union db_prod32 iwarp_db2_data;
338};
339
340#define QEDR_INC_SW_IDX(p_info, index) \
341 do { \
342 p_info->index = (p_info->index + 1) & \
343 qed_chain_get_capacity(p_info->pbl) \
344 } while (0)
345
346struct qedr_srq_hwq_info {
347 u32 max_sges;
348 u32 max_wr;
349 struct qed_chain pbl;
350 u64 p_phys_addr_tbl;
351 u32 wqe_prod;
352 u32 sge_prod;
353 u32 wr_prod_cnt;
354 u32 wr_cons_cnt;
355 u32 num_elems;
356
357 u32 *virt_prod_pair_addr;
358 dma_addr_t phy_prod_pair_addr;
359};
360
361struct qedr_srq {
362 struct ib_srq ibsrq;
363 struct qedr_dev *dev;
364
365 struct qedr_userq usrq;
366 struct qedr_srq_hwq_info hw_srq;
367 struct ib_umem *prod_umem;
368 u16 srq_id;
369 u32 srq_limit;
370
371 spinlock_t lock;
372};
373
374enum qedr_qp_err_bitmap {
375 QEDR_QP_ERR_SQ_FULL = 1,
376 QEDR_QP_ERR_RQ_FULL = 2,
377 QEDR_QP_ERR_BAD_SR = 4,
378 QEDR_QP_ERR_BAD_RR = 8,
379 QEDR_QP_ERR_SQ_PBL_FULL = 16,
380 QEDR_QP_ERR_RQ_PBL_FULL = 32,
381};
382
383struct qedr_qp {
384 struct ib_qp ibqp;
385 struct qedr_dev *dev;
386 struct qedr_iw_ep *ep;
387 struct qedr_qp_hwq_info sq;
388 struct qedr_qp_hwq_info rq;
389
390 u32 max_inline_data;
391
392
393 spinlock_t q_lock;
394 struct qedr_cq *sq_cq;
395 struct qedr_cq *rq_cq;
396 struct qedr_srq *srq;
397 enum qed_roce_qp_state state;
398 u32 id;
399 struct qedr_pd *pd;
400 enum ib_qp_type qp_type;
401 struct qed_rdma_qp *qed_qp;
402 u32 qp_id;
403 u16 icid;
404 u16 mtu;
405 int sgid_idx;
406 u32 rq_psn;
407 u32 sq_psn;
408 u32 qkey;
409 u32 dest_qp_num;
410
411
412 u8 prev_wqe_size;
413 u16 wqe_cons;
414 u32 err_bitmap;
415 bool signaled;
416
417
418 struct {
419 u64 wr_id;
420 enum ib_wc_opcode opcode;
421 u32 bytes_len;
422 u8 wqe_size;
423 bool signaled;
424 dma_addr_t icrc_mapping;
425 u32 *icrc;
426 struct qedr_mr *mr;
427 } *wqe_wr_id;
428
429
430 struct {
431 u64 wr_id;
432 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
433 u8 wqe_size;
434
435 u8 smac[ETH_ALEN];
436 u16 vlan;
437 int rc;
438 } *rqe_wr_id;
439
440
441 struct qedr_userq usq;
442 struct qedr_userq urq;
443 atomic_t refcnt;
444 bool destroyed;
445};
446
447struct qedr_ah {
448 struct ib_ah ibah;
449 struct rdma_ah_attr attr;
450};
451
452enum qedr_mr_type {
453 QEDR_MR_USER,
454 QEDR_MR_KERNEL,
455 QEDR_MR_DMA,
456 QEDR_MR_FRMR,
457};
458
459struct mr_info {
460 struct qedr_pbl *pbl_table;
461 struct qedr_pbl_info pbl_info;
462 struct list_head free_pbl_list;
463 struct list_head inuse_pbl_list;
464 u32 completed;
465 u32 completed_handled;
466};
467
468struct qedr_mr {
469 struct ib_mr ibmr;
470 struct ib_umem *umem;
471
472 struct qed_rdma_register_tid_in_params hw_mr;
473 enum qedr_mr_type type;
474
475 struct qedr_dev *dev;
476 struct mr_info info;
477
478 u64 *pages;
479 u32 npages;
480};
481
482#define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
483
484#define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
485 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
486#define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
487 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
488#define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
489 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
490
491static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
492{
493 info->cons = (info->cons + 1) % info->max_wr;
494 info->wqe_cons++;
495}
496
497static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
498{
499 info->prod = (info->prod + 1) % info->max_wr;
500}
501
502static inline int qedr_get_dmac(struct qedr_dev *dev,
503 struct rdma_ah_attr *ah_attr, u8 *mac_addr)
504{
505 union ib_gid zero_sgid = { { 0 } };
506 struct in6_addr in6;
507 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
508 u8 *dmac;
509
510 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
511 DP_ERR(dev, "Local port GID not supported\n");
512 eth_zero_addr(mac_addr);
513 return -EINVAL;
514 }
515
516 memcpy(&in6, grh->dgid.raw, sizeof(in6));
517 dmac = rdma_ah_retrieve_dmac(ah_attr);
518 if (!dmac)
519 return -EINVAL;
520 ether_addr_copy(mac_addr, dmac);
521
522 return 0;
523}
524
525struct qedr_iw_listener {
526 struct qedr_dev *dev;
527 struct iw_cm_id *cm_id;
528 int backlog;
529 void *qed_handle;
530};
531
532struct qedr_iw_ep {
533 struct qedr_dev *dev;
534 struct iw_cm_id *cm_id;
535 struct qedr_qp *qp;
536 void *qed_context;
537 u8 during_connect;
538};
539
540static inline
541struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
542{
543 return container_of(ibucontext, struct qedr_ucontext, ibucontext);
544}
545
546static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
547{
548 return container_of(ibdev, struct qedr_dev, ibdev);
549}
550
551static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
552{
553 return container_of(ibpd, struct qedr_pd, ibpd);
554}
555
556static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
557{
558 return container_of(ibcq, struct qedr_cq, ibcq);
559}
560
561static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
562{
563 return container_of(ibqp, struct qedr_qp, ibqp);
564}
565
566static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
567{
568 return container_of(ibah, struct qedr_ah, ibah);
569}
570
571static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
572{
573 return container_of(ibmr, struct qedr_mr, ibmr);
574}
575
576static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
577{
578 return container_of(ibsrq, struct qedr_srq, ibsrq);
579}
580#endif
581