1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#ifndef __BNXT_QPLIB_FP_H__
40#define __BNXT_QPLIB_FP_H__
41
42struct bnxt_qplib_srq {
43 struct bnxt_qplib_pd *pd;
44 struct bnxt_qplib_dpi *dpi;
45 struct bnxt_qplib_db_info dbinfo;
46 u64 srq_handle;
47 u32 id;
48 u16 wqe_size;
49 u32 max_wqe;
50 u32 max_sge;
51 u32 threshold;
52 bool arm_req;
53 struct bnxt_qplib_cq *cq;
54 struct bnxt_qplib_hwq hwq;
55 struct bnxt_qplib_swq *swq;
56 int start_idx;
57 int last_idx;
58 struct bnxt_qplib_sg_info sg_info;
59 u16 eventq_hw_ring_id;
60 spinlock_t lock;
61};
62
63struct bnxt_qplib_sge {
64 u64 addr;
65 u32 lkey;
66 u32 size;
67};
68
69#define BNXT_QPLIB_QP_MAX_SGL 6
70struct bnxt_qplib_swq {
71 u64 wr_id;
72 int next_idx;
73 u8 type;
74 u8 flags;
75 u32 start_psn;
76 u32 next_psn;
77 struct sq_psn_search *psn_search;
78 struct sq_psn_search_ext *psn_ext;
79};
80
81struct bnxt_qplib_swqe {
82
83#define BNXT_QPLIB_FENCE_WRID 0x46454E43
84 u64 wr_id;
85 u8 reqs_type;
86 u8 type;
87#define BNXT_QPLIB_SWQE_TYPE_SEND 0
88#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1
89#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2
90#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4
91#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5
92#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6
93#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8
94#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11
95#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12
96#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13
97#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13
98#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14
99#define BNXT_QPLIB_SWQE_TYPE_RECV 128
100#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129
101 u8 flags;
102#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0)
103#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1)
104#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2)
105#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3)
106#define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4)
107 struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL];
108 int num_sge;
109
110 u32 inline_len;
111#define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96
112 u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH];
113
114 union {
115
116 struct {
117 union {
118 __be32 imm_data;
119 u32 inv_key;
120 };
121 u32 q_key;
122 u32 dst_qp;
123 u16 avid;
124 } send;
125
126
127 struct {
128 u16 lflags;
129 u16 cfa_action;
130 u32 cfa_meta;
131 } rawqp1;
132
133
134 struct {
135 union {
136 __be32 imm_data;
137 u32 inv_key;
138 };
139 u64 remote_va;
140 u32 r_key;
141 } rdma;
142
143
144 struct {
145 u64 remote_va;
146 u32 r_key;
147 u64 swap_data;
148 u64 cmp_data;
149 } atomic;
150
151
152 struct {
153 u32 inv_l_key;
154 } local_inv;
155
156
157 struct {
158 u8 access_cntl;
159 u8 pg_sz_log;
160 bool zero_based;
161 u32 l_key;
162 u32 length;
163 u8 pbl_pg_sz_log;
164#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0
165#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1
166#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4
167#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6
168#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8
169#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9
170#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10
171#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18
172 u8 levels;
173#define PAGE_SHIFT_4K 12
174 __le64 *pbl_ptr;
175 dma_addr_t pbl_dma_ptr;
176 u64 *page_list;
177 u16 page_list_len;
178 u64 va;
179 } frmr;
180
181
182 struct {
183 u8 access_cntl;
184#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0)
185#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1)
186#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2)
187#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3)
188#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4)
189 bool zero_based;
190 u8 mw_type;
191 u32 parent_l_key;
192 u32 r_key;
193 u64 va;
194 u32 length;
195 } bind;
196 };
197};
198
199struct bnxt_qplib_q {
200 struct bnxt_qplib_hwq hwq;
201 struct bnxt_qplib_swq *swq;
202 struct bnxt_qplib_db_info dbinfo;
203 struct bnxt_qplib_sg_info sg_info;
204 u32 max_wqe;
205 u16 wqe_size;
206 u16 q_full_delta;
207 u16 max_sge;
208 u32 psn;
209 bool condition;
210 bool single;
211 bool send_phantom;
212 u32 phantom_wqe_cnt;
213 u32 phantom_cqe_cnt;
214 u32 next_cq_cons;
215 bool flushed;
216};
217
218struct bnxt_qplib_qp {
219 struct bnxt_qplib_pd *pd;
220 struct bnxt_qplib_dpi *dpi;
221 struct bnxt_qplib_chip_ctx *cctx;
222 u64 qp_handle;
223#define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF
224 u32 id;
225 u8 type;
226 u8 sig_type;
227 u32 modify_flags;
228 u8 state;
229 u8 cur_qp_state;
230 u32 max_inline_data;
231 u32 mtu;
232 u8 path_mtu;
233 bool en_sqd_async_notify;
234 u16 pkey_index;
235 u32 qkey;
236 u32 dest_qp_id;
237 u8 access;
238 u8 timeout;
239 u8 retry_cnt;
240 u8 rnr_retry;
241 u64 wqe_cnt;
242 u32 min_rnr_timer;
243 u32 max_rd_atomic;
244 u32 max_dest_rd_atomic;
245 u32 dest_qpn;
246 u8 smac[6];
247 u16 vlan_id;
248 u8 nw_type;
249 struct bnxt_qplib_ah ah;
250
251#define BTH_PSN_MASK ((1 << 24) - 1)
252
253 struct bnxt_qplib_q sq;
254
255 struct bnxt_qplib_q rq;
256
257 struct bnxt_qplib_srq *srq;
258
259 struct bnxt_qplib_cq *scq;
260 struct bnxt_qplib_cq *rcq;
261
262 struct bnxt_qplib_hwq irrq;
263 struct bnxt_qplib_hwq orrq;
264
265 int sq_hdr_buf_size;
266 int rq_hdr_buf_size;
267
268
269
270
271
272#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86
273
274
275
276
277
278#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14
279#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512
280#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20
281#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40
282#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20
283 void *sq_hdr_buf;
284 dma_addr_t sq_hdr_buf_map;
285 void *rq_hdr_buf;
286 dma_addr_t rq_hdr_buf_map;
287 struct list_head sq_flush;
288 struct list_head rq_flush;
289};
290
291#define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base)
292
293#define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE)
294#define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1)
295#define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG)
296#define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG)
297
298#define ROCE_CQE_CMP_V 0
299#define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \
300 (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
301 !((raw_cons) & (cp_bit)))
302
303static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q)
304{
305 return HWQ_CMP((qplib_q->hwq.prod + qplib_q->q_full_delta),
306 &qplib_q->hwq) == HWQ_CMP(qplib_q->hwq.cons,
307 &qplib_q->hwq);
308}
309
310struct bnxt_qplib_cqe {
311 u8 status;
312 u8 type;
313 u8 opcode;
314 u32 length;
315 u16 cfa_meta;
316 u64 wr_id;
317 union {
318 __be32 immdata;
319 u32 invrkey;
320 };
321 u64 qp_handle;
322 u64 mr_handle;
323 u16 flags;
324 u8 smac[6];
325 u32 src_qp;
326 u16 raweth_qp1_flags;
327 u16 raweth_qp1_errors;
328 u16 raweth_qp1_cfa_code;
329 u32 raweth_qp1_flags2;
330 u32 raweth_qp1_metadata;
331 u8 raweth_qp1_payload_offset;
332 u16 pkey_index;
333};
334
335#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01
336struct bnxt_qplib_cq {
337 struct bnxt_qplib_dpi *dpi;
338 struct bnxt_qplib_db_info dbinfo;
339 u32 max_wqe;
340 u32 id;
341 u16 count;
342 u16 period;
343 struct bnxt_qplib_hwq hwq;
344 u32 cnq_hw_ring_id;
345 struct bnxt_qplib_nq *nq;
346 bool resize_in_progress;
347 struct bnxt_qplib_sg_info sg_info;
348 u64 cq_handle;
349
350#define CQ_RESIZE_WAIT_TIME_MS 500
351 unsigned long flags;
352#define CQ_FLAGS_RESIZE_IN_PROG 1
353 wait_queue_head_t waitq;
354 struct list_head sqf_head, rqf_head;
355 atomic_t arm_state;
356 spinlock_t compl_lock;
357
358
359
360
361
362
363
364
365
366
367
368 spinlock_t flush_lock;
369 u16 cnq_events;
370};
371
372#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq)
373#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq)
374#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2)
375#define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1)
376#define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1)
377#define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1)
378
379#define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base)
380
381#define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE)
382#define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1)
383#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)
384#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG)
385
386#define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \
387 (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \
388 !((raw_cons) & (cp_bit)))
389
390#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024)
391
392#define NQ_CONS_PCI_BAR_REGION 2
393#define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT)
394#define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID
395#define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK
396#define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \
397 NQ_DB_IDX_VALID)
398#define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \
399 NQ_DB_IDX_VALID | \
400 NQ_DB_IRQ_DIS)
401
402struct bnxt_qplib_nq_db {
403 struct bnxt_qplib_reg_desc reg;
404 struct bnxt_qplib_db_info dbinfo;
405};
406
407typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq,
408 struct bnxt_qplib_cq *cq);
409typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq,
410 struct bnxt_qplib_srq *srq, u8 event);
411
412struct bnxt_qplib_nq {
413 struct pci_dev *pdev;
414 struct bnxt_qplib_res *res;
415 char name[32];
416 struct bnxt_qplib_hwq hwq;
417 struct bnxt_qplib_nq_db nq_db;
418 u16 ring_id;
419 int msix_vec;
420 cpumask_t mask;
421 struct tasklet_struct nq_tasklet;
422 bool requested;
423 int budget;
424
425 cqn_handler_t cqn_handler;
426 srqn_handler_t srqn_handler;
427 struct workqueue_struct *cqn_wq;
428};
429
430struct bnxt_qplib_nq_work {
431 struct work_struct work;
432 struct bnxt_qplib_nq *nq;
433 struct bnxt_qplib_cq *cq;
434};
435
436void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill);
437void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
438int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
439 int msix_vector, bool need_init);
440int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
441 int nq_idx, int msix_vector, int bar_reg_offset,
442 cqn_handler_t cqn_handler,
443 srqn_handler_t srq_handler);
444int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
445 struct bnxt_qplib_srq *srq);
446int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
447 struct bnxt_qplib_srq *srq);
448int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
449 struct bnxt_qplib_srq *srq);
450void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
451 struct bnxt_qplib_srq *srq);
452int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
453 struct bnxt_qplib_swqe *wqe);
454int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
455int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
456int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
457int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
458int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
459void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp);
460void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
461 struct bnxt_qplib_qp *qp);
462void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
463 struct bnxt_qplib_sge *sge);
464void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
465 struct bnxt_qplib_sge *sge);
466u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp);
467dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp,
468 u32 index);
469void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp);
470int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
471 struct bnxt_qplib_swqe *wqe);
472void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);
473int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
474 struct bnxt_qplib_swqe *wqe);
475int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
476int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
477int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
478 int num, struct bnxt_qplib_qp **qp);
479bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq);
480void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
481void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
482int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq);
483void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
484void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
485 unsigned long *flags);
486void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
487 unsigned long *flags);
488int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
489 struct bnxt_qplib_cqe *cqe,
490 int num_cqes);
491void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);
492#endif
493
