1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#ifndef MLX5_QP_H
34#define MLX5_QP_H
35
36#include <linux/mlx5/device.h>
37#include <linux/mlx5/driver.h>
38
39#define MLX5_INVALID_LKEY 0x100
40
41#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 8)
42#define MLX5_DIF_SIZE 8
43#define MLX5_STRIDE_BLOCK_OP 0x400
44#define MLX5_CPY_GRD_MASK 0xc0
45#define MLX5_CPY_APP_MASK 0x30
46#define MLX5_CPY_REF_MASK 0x0f
47#define MLX5_BSF_INC_REFTAG (1 << 6)
48#define MLX5_BSF_INL_VALID (1 << 15)
49#define MLX5_BSF_REFRESH_DIF (1 << 14)
50#define MLX5_BSF_REPEAT_BLOCK (1 << 7)
51#define MLX5_BSF_APPTAG_ESCAPE 0x1
52#define MLX5_BSF_APPREF_ESCAPE 0x2
53
54enum mlx5_qp_optpar {
55 MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
56 MLX5_QP_OPTPAR_RRE = 1 << 1,
57 MLX5_QP_OPTPAR_RAE = 1 << 2,
58 MLX5_QP_OPTPAR_RWE = 1 << 3,
59 MLX5_QP_OPTPAR_PKEY_INDEX = 1 << 4,
60 MLX5_QP_OPTPAR_Q_KEY = 1 << 5,
61 MLX5_QP_OPTPAR_RNR_TIMEOUT = 1 << 6,
62 MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,
63 MLX5_QP_OPTPAR_SRA_MAX = 1 << 8,
64 MLX5_QP_OPTPAR_RRA_MAX = 1 << 9,
65 MLX5_QP_OPTPAR_PM_STATE = 1 << 10,
66 MLX5_QP_OPTPAR_RETRY_COUNT = 1 << 12,
67 MLX5_QP_OPTPAR_RNR_RETRY = 1 << 13,
68 MLX5_QP_OPTPAR_ACK_TIMEOUT = 1 << 14,
69 MLX5_QP_OPTPAR_PRI_PORT = 1 << 16,
70 MLX5_QP_OPTPAR_SRQN = 1 << 18,
71 MLX5_QP_OPTPAR_CQN_RCV = 1 << 19,
72 MLX5_QP_OPTPAR_DC_HS = 1 << 20,
73 MLX5_QP_OPTPAR_DC_KEY = 1 << 21,
74 MLX5_QP_OPTPAR_COUNTER_SET_ID = 1 << 25,
75};
76
77enum mlx5_qp_state {
78 MLX5_QP_STATE_RST = 0,
79 MLX5_QP_STATE_INIT = 1,
80 MLX5_QP_STATE_RTR = 2,
81 MLX5_QP_STATE_RTS = 3,
82 MLX5_QP_STATE_SQER = 4,
83 MLX5_QP_STATE_SQD = 5,
84 MLX5_QP_STATE_ERR = 6,
85 MLX5_QP_STATE_SQ_DRAINING = 7,
86 MLX5_QP_STATE_SUSPENDED = 9,
87 MLX5_QP_NUM_STATE,
88 MLX5_QP_STATE,
89 MLX5_QP_STATE_BAD,
90};
91
92enum {
93 MLX5_SQ_STATE_NA = MLX5_SQC_STATE_ERR + 1,
94 MLX5_SQ_NUM_STATE = MLX5_SQ_STATE_NA + 1,
95 MLX5_RQ_STATE_NA = MLX5_RQC_STATE_ERR + 1,
96 MLX5_RQ_NUM_STATE = MLX5_RQ_STATE_NA + 1,
97};
98
99enum {
100 MLX5_QP_ST_RC = 0x0,
101 MLX5_QP_ST_UC = 0x1,
102 MLX5_QP_ST_UD = 0x2,
103 MLX5_QP_ST_XRC = 0x3,
104 MLX5_QP_ST_MLX = 0x4,
105 MLX5_QP_ST_DCI = 0x5,
106 MLX5_QP_ST_DCT = 0x6,
107 MLX5_QP_ST_QP0 = 0x7,
108 MLX5_QP_ST_QP1 = 0x8,
109 MLX5_QP_ST_RAW_ETHERTYPE = 0x9,
110 MLX5_QP_ST_RAW_IPV6 = 0xa,
111 MLX5_QP_ST_SNIFFER = 0xb,
112 MLX5_QP_ST_SYNC_UMR = 0xe,
113 MLX5_QP_ST_PTP_1588 = 0xd,
114 MLX5_QP_ST_REG_UMR = 0xc,
115 MLX5_QP_ST_MAX
116};
117
118enum {
119 MLX5_QP_PM_MIGRATED = 0x3,
120 MLX5_QP_PM_ARMED = 0x0,
121 MLX5_QP_PM_REARM = 0x1
122};
123
124enum {
125 MLX5_NON_ZERO_RQ = 0x0,
126 MLX5_SRQ_RQ = 0x1,
127 MLX5_CRQ_RQ = 0x2,
128 MLX5_ZERO_LEN_RQ = 0x3
129};
130
131
132enum {
133
134 MLX5_QP_BIT_SRE = 1 << 15,
135 MLX5_QP_BIT_SWE = 1 << 14,
136 MLX5_QP_BIT_SAE = 1 << 13,
137
138 MLX5_QP_BIT_RRE = 1 << 15,
139 MLX5_QP_BIT_RWE = 1 << 14,
140 MLX5_QP_BIT_RAE = 1 << 13,
141 MLX5_QP_BIT_RIC = 1 << 4,
142 MLX5_QP_BIT_CC_SLAVE_RECV = 1 << 2,
143 MLX5_QP_BIT_CC_SLAVE_SEND = 1 << 1,
144 MLX5_QP_BIT_CC_MASTER = 1 << 0
145};
146
147enum {
148 MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2,
149 MLX5_WQE_CTRL_CQ_UPDATE_AND_EQE = 3 << 2,
150 MLX5_WQE_CTRL_SOLICITED = 1 << 1,
151};
152
153enum {
154 MLX5_SEND_WQE_DS = 16,
155 MLX5_SEND_WQE_BB = 64,
156};
157
158#define MLX5_SEND_WQEBB_NUM_DS (MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
159
160enum {
161 MLX5_SEND_WQE_MAX_WQEBBS = 16,
162};
163
164enum {
165 MLX5_WQE_FMR_PERM_LOCAL_READ = 1 << 27,
166 MLX5_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28,
167 MLX5_WQE_FMR_PERM_REMOTE_READ = 1 << 29,
168 MLX5_WQE_FMR_PERM_REMOTE_WRITE = 1 << 30,
169 MLX5_WQE_FMR_PERM_ATOMIC = 1 << 31
170};
171
172enum {
173 MLX5_FENCE_MODE_NONE = 0 << 5,
174 MLX5_FENCE_MODE_INITIATOR_SMALL = 1 << 5,
175 MLX5_FENCE_MODE_FENCE = 2 << 5,
176 MLX5_FENCE_MODE_STRONG_ORDERING = 3 << 5,
177 MLX5_FENCE_MODE_SMALL_AND_FENCE = 4 << 5,
178};
179
180enum {
181 MLX5_RCV_DBR = 0,
182 MLX5_SND_DBR = 1,
183};
184
185enum {
186 MLX5_FLAGS_INLINE = 1<<7,
187 MLX5_FLAGS_CHECK_FREE = 1<<5,
188};
189
190struct mlx5_wqe_fmr_seg {
191 __be32 flags;
192 __be32 mem_key;
193 __be64 buf_list;
194 __be64 start_addr;
195 __be64 reg_len;
196 __be32 offset;
197 __be32 page_size;
198 u32 reserved[2];
199};
200
201struct mlx5_wqe_ctrl_seg {
202 __be32 opmod_idx_opcode;
203 __be32 qpn_ds;
204 u8 signature;
205 u8 rsvd[2];
206 u8 fm_ce_se;
207 union {
208 __be32 general_id;
209 __be32 imm;
210 __be32 umr_mkey;
211 __be32 tisn;
212 };
213};
214
215#define MLX5_WQE_CTRL_DS_MASK 0x3f
216#define MLX5_WQE_CTRL_QPN_MASK 0xffffff00
217#define MLX5_WQE_CTRL_QPN_SHIFT 8
218#define MLX5_WQE_DS_UNITS 16
219#define MLX5_WQE_CTRL_OPCODE_MASK 0xff
220#define MLX5_WQE_CTRL_WQE_INDEX_MASK 0x00ffff00
221#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8
222
223enum {
224 MLX5_ETH_WQE_L3_INNER_CSUM = 1 << 4,
225 MLX5_ETH_WQE_L4_INNER_CSUM = 1 << 5,
226 MLX5_ETH_WQE_L3_CSUM = 1 << 6,
227 MLX5_ETH_WQE_L4_CSUM = 1 << 7,
228};
229
230enum {
231 MLX5_ETH_WQE_SVLAN = 1 << 0,
232 MLX5_ETH_WQE_INSERT_VLAN = 1 << 15,
233};
234
235enum {
236 MLX5_ETH_WQE_SWP_INNER_L3_IPV6 = 1 << 0,
237 MLX5_ETH_WQE_SWP_INNER_L4_UDP = 1 << 1,
238 MLX5_ETH_WQE_SWP_OUTER_L3_IPV6 = 1 << 4,
239 MLX5_ETH_WQE_SWP_OUTER_L4_UDP = 1 << 5,
240};
241
242struct mlx5_wqe_eth_seg {
243 u8 swp_outer_l4_offset;
244 u8 swp_outer_l3_offset;
245 u8 swp_inner_l4_offset;
246 u8 swp_inner_l3_offset;
247 u8 cs_flags;
248 u8 swp_flags;
249 __be16 mss;
250 __be32 rsvd2;
251 union {
252 struct {
253 __be16 sz;
254 u8 start[2];
255 } inline_hdr;
256 struct {
257 __be16 type;
258 __be16 vlan_tci;
259 } insert;
260 };
261};
262
263struct mlx5_wqe_xrc_seg {
264 __be32 xrc_srqn;
265 u8 rsvd[12];
266};
267
268struct mlx5_wqe_masked_atomic_seg {
269 __be64 swap_add;
270 __be64 compare;
271 __be64 swap_add_mask;
272 __be64 compare_mask;
273};
274
275struct mlx5_base_av {
276 union {
277 struct {
278 __be32 qkey;
279 __be32 reserved;
280 } qkey;
281 __be64 dc_key;
282 } key;
283 __be32 dqp_dct;
284 u8 stat_rate_sl;
285 u8 fl_mlid;
286 union {
287 __be16 rlid;
288 __be16 udp_sport;
289 };
290};
291
292struct mlx5_av {
293 union {
294 struct {
295 __be32 qkey;
296 __be32 reserved;
297 } qkey;
298 __be64 dc_key;
299 } key;
300 __be32 dqp_dct;
301 u8 stat_rate_sl;
302 u8 fl_mlid;
303 union {
304 __be16 rlid;
305 __be16 udp_sport;
306 };
307 u8 reserved0[4];
308 u8 rmac[6];
309 u8 tclass;
310 u8 hop_limit;
311 __be32 grh_gid_fl;
312 u8 rgid[16];
313};
314
315struct mlx5_ib_ah {
316 struct ib_ah ibah;
317 struct mlx5_av av;
318};
319
320static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
321{
322 return container_of(ibah, struct mlx5_ib_ah, ibah);
323}
324
325struct mlx5_wqe_datagram_seg {
326 struct mlx5_av av;
327};
328
329struct mlx5_wqe_raddr_seg {
330 __be64 raddr;
331 __be32 rkey;
332 u32 reserved;
333};
334
335struct mlx5_wqe_atomic_seg {
336 __be64 swap_add;
337 __be64 compare;
338};
339
340struct mlx5_wqe_data_seg {
341 __be32 byte_count;
342 __be32 lkey;
343 __be64 addr;
344};
345
346struct mlx5_wqe_umr_ctrl_seg {
347 u8 flags;
348 u8 rsvd0[3];
349 __be16 xlt_octowords;
350 union {
351 __be16 xlt_offset;
352 __be16 bsf_octowords;
353 };
354 __be64 mkey_mask;
355 __be32 xlt_offset_47_16;
356 u8 rsvd1[28];
357};
358
359struct mlx5_seg_set_psv {
360 __be32 psv_num;
361 __be16 syndrome;
362 __be16 status;
363 __be32 transient_sig;
364 __be32 ref_tag;
365};
366
367struct mlx5_seg_get_psv {
368 u8 rsvd[19];
369 u8 num_psv;
370 __be32 l_key;
371 __be64 va;
372 __be32 psv_index[4];
373};
374
375struct mlx5_seg_check_psv {
376 u8 rsvd0[2];
377 __be16 err_coalescing_op;
378 u8 rsvd1[2];
379 __be16 xport_err_op;
380 u8 rsvd2[2];
381 __be16 xport_err_mask;
382 u8 rsvd3[7];
383 u8 num_psv;
384 __be32 l_key;
385 __be64 va;
386 __be32 psv_index[4];
387};
388
389struct mlx5_rwqe_sig {
390 u8 rsvd0[4];
391 u8 signature;
392 u8 rsvd1[11];
393};
394
395struct mlx5_wqe_signature_seg {
396 u8 rsvd0[4];
397 u8 signature;
398 u8 rsvd1[11];
399};
400
401#define MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK 0x3ff
402
403struct mlx5_wqe_inline_seg {
404 __be32 byte_count;
405 __be32 data[0];
406};
407
408enum mlx5_sig_type {
409 MLX5_DIF_CRC = 0x1,
410 MLX5_DIF_IPCS = 0x2,
411};
412
413struct mlx5_bsf_inl {
414 __be16 vld_refresh;
415 __be16 dif_apptag;
416 __be32 dif_reftag;
417 u8 sig_type;
418 u8 rp_inv_seed;
419 u8 rsvd[3];
420 u8 dif_inc_ref_guard_check;
421 __be16 dif_app_bitmask_check;
422};
423
424struct mlx5_bsf {
425 struct mlx5_bsf_basic {
426 u8 bsf_size_sbs;
427 u8 check_byte_mask;
428 union {
429 u8 copy_byte_mask;
430 u8 bs_selector;
431 u8 rsvd_wflags;
432 } wire;
433 union {
434 u8 bs_selector;
435 u8 rsvd_mflags;
436 } mem;
437 __be32 raw_data_size;
438 __be32 w_bfs_psv;
439 __be32 m_bfs_psv;
440 } basic;
441 struct mlx5_bsf_ext {
442 __be32 t_init_gen_pro_size;
443 __be32 rsvd_epi_size;
444 __be32 w_tfs_psv;
445 __be32 m_tfs_psv;
446 } ext;
447 struct mlx5_bsf_inl w_inl;
448 struct mlx5_bsf_inl m_inl;
449};
450
451struct mlx5_mtt {
452 __be64 ptag;
453};
454
455struct mlx5_klm {
456 __be32 bcount;
457 __be32 key;
458 __be64 va;
459};
460
461struct mlx5_stride_block_entry {
462 __be16 stride;
463 __be16 bcount;
464 __be32 key;
465 __be64 va;
466};
467
468struct mlx5_stride_block_ctrl_seg {
469 __be32 bcount_per_cycle;
470 __be32 op;
471 __be32 repeat_count;
472 u16 rsvd;
473 __be16 num_entries;
474};
475
476struct mlx5_core_qp {
477 struct mlx5_core_rsc_common common;
478 void (*event) (struct mlx5_core_qp *, int);
479 int qpn;
480 struct mlx5_rsc_debug *dbg;
481 int pid;
482 u16 uid;
483};
484
485struct mlx5_core_dct {
486 struct mlx5_core_qp mqp;
487 struct completion drained;
488};
489
490struct mlx5_qp_path {
491 u8 fl_free_ar;
492 u8 rsvd3;
493 __be16 pkey_index;
494 u8 rsvd0;
495 u8 grh_mlid;
496 __be16 rlid;
497 u8 ackto_lt;
498 u8 mgid_index;
499 u8 static_rate;
500 u8 hop_limit;
501 __be32 tclass_flowlabel;
502 union {
503 u8 rgid[16];
504 u8 rip[16];
505 };
506 u8 f_dscp_ecn_prio;
507 u8 ecn_dscp;
508 __be16 udp_sport;
509 u8 dci_cfi_prio_sl;
510 u8 port;
511 u8 rmac[6];
512};
513
514
515struct mlx5_qp_context {
516 __be32 flags;
517 __be32 flags_pd;
518 u8 mtu_msgmax;
519 u8 rq_size_stride;
520 __be16 sq_crq_size;
521 __be32 qp_counter_set_usr_page;
522 __be32 wire_qpn;
523 __be32 log_pg_sz_remote_qpn;
524 struct mlx5_qp_path pri_path;
525 struct mlx5_qp_path alt_path;
526 __be32 params1;
527 u8 reserved2[4];
528 __be32 next_send_psn;
529 __be32 cqn_send;
530 __be32 deth_sqpn;
531 u8 reserved3[4];
532 __be32 last_acked_psn;
533 __be32 ssn;
534 __be32 params2;
535 __be32 rnr_nextrecvpsn;
536 __be32 xrcd;
537 __be32 cqn_recv;
538 __be64 db_rec_addr;
539 __be32 qkey;
540 __be32 rq_type_srqn;
541 __be32 rmsn;
542 __be16 hw_sq_wqe_counter;
543 __be16 sw_sq_wqe_counter;
544 __be16 hw_rcyclic_byte_counter;
545 __be16 hw_rq_counter;
546 __be16 sw_rcyclic_byte_counter;
547 __be16 sw_rq_counter;
548 u8 rsvd0[5];
549 u8 cgs;
550 u8 cs_req;
551 u8 cs_res;
552 __be64 dc_access_key;
553 u8 rsvd1[24];
554};
555
556static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u32 qpn)
557{
558 return radix_tree_lookup(&dev->priv.qp_table.tree, qpn);
559}
560
561int mlx5_core_create_dct(struct mlx5_core_dev *dev,
562 struct mlx5_core_dct *qp,
563 u32 *in, int inlen,
564 u32 *out, int outlen);
565int mlx5_core_create_qp(struct mlx5_core_dev *dev,
566 struct mlx5_core_qp *qp,
567 u32 *in,
568 int inlen);
569int mlx5_core_qp_modify(struct mlx5_core_dev *dev, u16 opcode,
570 u32 opt_param_mask, void *qpc,
571 struct mlx5_core_qp *qp);
572int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
573 struct mlx5_core_qp *qp);
574int mlx5_core_destroy_dct(struct mlx5_core_dev *dev,
575 struct mlx5_core_dct *dct);
576int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
577 u32 *out, int outlen);
578int mlx5_core_dct_query(struct mlx5_core_dev *dev, struct mlx5_core_dct *dct,
579 u32 *out, int outlen);
580
581int mlx5_core_set_delay_drop(struct mlx5_core_dev *dev,
582 u32 timeout_usec);
583
584int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn);
585int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn);
586void mlx5_init_qp_table(struct mlx5_core_dev *dev);
587void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev);
588int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
589void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
590int mlx5_core_create_rq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
591 struct mlx5_core_qp *rq);
592void mlx5_core_destroy_rq_tracked(struct mlx5_core_dev *dev,
593 struct mlx5_core_qp *rq);
594int mlx5_core_create_sq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
595 struct mlx5_core_qp *sq);
596void mlx5_core_destroy_sq_tracked(struct mlx5_core_dev *dev,
597 struct mlx5_core_qp *sq);
598int mlx5_core_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id);
599int mlx5_core_dealloc_q_counter(struct mlx5_core_dev *dev, u16 counter_id);
600int mlx5_core_query_q_counter(struct mlx5_core_dev *dev, u16 counter_id,
601 int reset, void *out, int out_size);
602
603struct mlx5_core_rsc_common *mlx5_core_res_hold(struct mlx5_core_dev *dev,
604 int res_num,
605 enum mlx5_res_type res_type);
606void mlx5_core_res_put(struct mlx5_core_rsc_common *res);
607
608static inline const char *mlx5_qp_type_str(int type)
609{
610 switch (type) {
611 case MLX5_QP_ST_RC: return "RC";
612 case MLX5_QP_ST_UC: return "C";
613 case MLX5_QP_ST_UD: return "UD";
614 case MLX5_QP_ST_XRC: return "XRC";
615 case MLX5_QP_ST_MLX: return "MLX";
616 case MLX5_QP_ST_QP0: return "QP0";
617 case MLX5_QP_ST_QP1: return "QP1";
618 case MLX5_QP_ST_RAW_ETHERTYPE: return "RAW_ETHERTYPE";
619 case MLX5_QP_ST_RAW_IPV6: return "RAW_IPV6";
620 case MLX5_QP_ST_SNIFFER: return "SNIFFER";
621 case MLX5_QP_ST_SYNC_UMR: return "SYNC_UMR";
622 case MLX5_QP_ST_PTP_1588: return "PTP_1588";
623 case MLX5_QP_ST_REG_UMR: return "REG_UMR";
624 default: return "Invalid transport type";
625 }
626}
627
628static inline const char *mlx5_qp_state_str(int state)
629{
630 switch (state) {
631 case MLX5_QP_STATE_RST:
632 return "RST";
633 case MLX5_QP_STATE_INIT:
634 return "INIT";
635 case MLX5_QP_STATE_RTR:
636 return "RTR";
637 case MLX5_QP_STATE_RTS:
638 return "RTS";
639 case MLX5_QP_STATE_SQER:
640 return "SQER";
641 case MLX5_QP_STATE_SQD:
642 return "SQD";
643 case MLX5_QP_STATE_ERR:
644 return "ERR";
645 case MLX5_QP_STATE_SQ_DRAINING:
646 return "SQ_DRAINING";
647 case MLX5_QP_STATE_SUSPENDED:
648 return "SUSPENDED";
649 default: return "Invalid QP state";
650 }
651}
652
653#endif
654
