1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#ifndef MLX5_QP_H
34#define MLX5_QP_H
35
36#include <linux/mlx5/device.h>
37#include <linux/mlx5/driver.h>
38
39#define MLX5_INVALID_LKEY 0x100
40#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5)
41#define MLX5_DIF_SIZE 8
42#define MLX5_STRIDE_BLOCK_OP 0x400
43#define MLX5_CPY_GRD_MASK 0xc0
44#define MLX5_CPY_APP_MASK 0x30
45#define MLX5_CPY_REF_MASK 0x0f
46#define MLX5_BSF_INC_REFTAG (1 << 6)
47#define MLX5_BSF_INL_VALID (1 << 15)
48#define MLX5_BSF_REFRESH_DIF (1 << 14)
49#define MLX5_BSF_REPEAT_BLOCK (1 << 7)
50#define MLX5_BSF_APPTAG_ESCAPE 0x1
51#define MLX5_BSF_APPREF_ESCAPE 0x2
52
53enum mlx5_qp_optpar {
54 MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
55 MLX5_QP_OPTPAR_RRE = 1 << 1,
56 MLX5_QP_OPTPAR_RAE = 1 << 2,
57 MLX5_QP_OPTPAR_RWE = 1 << 3,
58 MLX5_QP_OPTPAR_PKEY_INDEX = 1 << 4,
59 MLX5_QP_OPTPAR_Q_KEY = 1 << 5,
60 MLX5_QP_OPTPAR_RNR_TIMEOUT = 1 << 6,
61 MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,
62 MLX5_QP_OPTPAR_SRA_MAX = 1 << 8,
63 MLX5_QP_OPTPAR_RRA_MAX = 1 << 9,
64 MLX5_QP_OPTPAR_PM_STATE = 1 << 10,
65 MLX5_QP_OPTPAR_RETRY_COUNT = 1 << 12,
66 MLX5_QP_OPTPAR_RNR_RETRY = 1 << 13,
67 MLX5_QP_OPTPAR_ACK_TIMEOUT = 1 << 14,
68 MLX5_QP_OPTPAR_PRI_PORT = 1 << 16,
69 MLX5_QP_OPTPAR_SRQN = 1 << 18,
70 MLX5_QP_OPTPAR_CQN_RCV = 1 << 19,
71 MLX5_QP_OPTPAR_DC_HS = 1 << 20,
72 MLX5_QP_OPTPAR_DC_KEY = 1 << 21,
73};
74
75enum mlx5_qp_state {
76 MLX5_QP_STATE_RST = 0,
77 MLX5_QP_STATE_INIT = 1,
78 MLX5_QP_STATE_RTR = 2,
79 MLX5_QP_STATE_RTS = 3,
80 MLX5_QP_STATE_SQER = 4,
81 MLX5_QP_STATE_SQD = 5,
82 MLX5_QP_STATE_ERR = 6,
83 MLX5_QP_STATE_SQ_DRAINING = 7,
84 MLX5_QP_STATE_SUSPENDED = 9,
85 MLX5_QP_NUM_STATE,
86 MLX5_QP_STATE,
87 MLX5_QP_STATE_BAD,
88};
89
90enum {
91 MLX5_SQ_STATE_NA = MLX5_SQC_STATE_ERR + 1,
92 MLX5_SQ_NUM_STATE = MLX5_SQ_STATE_NA + 1,
93 MLX5_RQ_STATE_NA = MLX5_RQC_STATE_ERR + 1,
94 MLX5_RQ_NUM_STATE = MLX5_RQ_STATE_NA + 1,
95};
96
97enum {
98 MLX5_QP_ST_RC = 0x0,
99 MLX5_QP_ST_UC = 0x1,
100 MLX5_QP_ST_UD = 0x2,
101 MLX5_QP_ST_XRC = 0x3,
102 MLX5_QP_ST_MLX = 0x4,
103 MLX5_QP_ST_DCI = 0x5,
104 MLX5_QP_ST_DCT = 0x6,
105 MLX5_QP_ST_QP0 = 0x7,
106 MLX5_QP_ST_QP1 = 0x8,
107 MLX5_QP_ST_RAW_ETHERTYPE = 0x9,
108 MLX5_QP_ST_RAW_IPV6 = 0xa,
109 MLX5_QP_ST_SNIFFER = 0xb,
110 MLX5_QP_ST_SYNC_UMR = 0xe,
111 MLX5_QP_ST_PTP_1588 = 0xd,
112 MLX5_QP_ST_REG_UMR = 0xc,
113 MLX5_QP_ST_MAX
114};
115
116enum {
117 MLX5_QP_PM_MIGRATED = 0x3,
118 MLX5_QP_PM_ARMED = 0x0,
119 MLX5_QP_PM_REARM = 0x1
120};
121
122enum {
123 MLX5_NON_ZERO_RQ = 0x0,
124 MLX5_SRQ_RQ = 0x1,
125 MLX5_CRQ_RQ = 0x2,
126 MLX5_ZERO_LEN_RQ = 0x3
127};
128
129
130enum {
131
132 MLX5_QP_BIT_SRE = 1 << 15,
133 MLX5_QP_BIT_SWE = 1 << 14,
134 MLX5_QP_BIT_SAE = 1 << 13,
135
136 MLX5_QP_BIT_RRE = 1 << 15,
137 MLX5_QP_BIT_RWE = 1 << 14,
138 MLX5_QP_BIT_RAE = 1 << 13,
139 MLX5_QP_BIT_RIC = 1 << 4,
140 MLX5_QP_BIT_CC_SLAVE_RECV = 1 << 2,
141 MLX5_QP_BIT_CC_SLAVE_SEND = 1 << 1,
142 MLX5_QP_BIT_CC_MASTER = 1 << 0
143};
144
145enum {
146 MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2,
147 MLX5_WQE_CTRL_CQ_UPDATE_AND_EQE = 3 << 2,
148 MLX5_WQE_CTRL_SOLICITED = 1 << 1,
149};
150
151enum {
152 MLX5_SEND_WQE_DS = 16,
153 MLX5_SEND_WQE_BB = 64,
154};
155
156#define MLX5_SEND_WQEBB_NUM_DS (MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
157
158enum {
159 MLX5_SEND_WQE_MAX_WQEBBS = 16,
160};
161
162enum {
163 MLX5_WQE_FMR_PERM_LOCAL_READ = 1 << 27,
164 MLX5_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28,
165 MLX5_WQE_FMR_PERM_REMOTE_READ = 1 << 29,
166 MLX5_WQE_FMR_PERM_REMOTE_WRITE = 1 << 30,
167 MLX5_WQE_FMR_PERM_ATOMIC = 1 << 31
168};
169
170enum {
171 MLX5_FENCE_MODE_NONE = 0 << 5,
172 MLX5_FENCE_MODE_INITIATOR_SMALL = 1 << 5,
173 MLX5_FENCE_MODE_FENCE = 2 << 5,
174 MLX5_FENCE_MODE_STRONG_ORDERING = 3 << 5,
175 MLX5_FENCE_MODE_SMALL_AND_FENCE = 4 << 5,
176};
177
178enum {
179 MLX5_RCV_DBR = 0,
180 MLX5_SND_DBR = 1,
181};
182
183enum {
184 MLX5_FLAGS_INLINE = 1<<7,
185 MLX5_FLAGS_CHECK_FREE = 1<<5,
186};
187
188struct mlx5_wqe_fmr_seg {
189 __be32 flags;
190 __be32 mem_key;
191 __be64 buf_list;
192 __be64 start_addr;
193 __be64 reg_len;
194 __be32 offset;
195 __be32 page_size;
196 u32 reserved[2];
197};
198
199struct mlx5_wqe_ctrl_seg {
200 __be32 opmod_idx_opcode;
201 __be32 qpn_ds;
202 u8 signature;
203 u8 rsvd[2];
204 u8 fm_ce_se;
205 __be32 imm;
206};
207
208#define MLX5_WQE_CTRL_DS_MASK 0x3f
209#define MLX5_WQE_CTRL_QPN_MASK 0xffffff00
210#define MLX5_WQE_CTRL_QPN_SHIFT 8
211#define MLX5_WQE_DS_UNITS 16
212#define MLX5_WQE_CTRL_OPCODE_MASK 0xff
213#define MLX5_WQE_CTRL_WQE_INDEX_MASK 0x00ffff00
214#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8
215
216enum {
217 MLX5_ETH_WQE_L3_INNER_CSUM = 1 << 4,
218 MLX5_ETH_WQE_L4_INNER_CSUM = 1 << 5,
219 MLX5_ETH_WQE_L3_CSUM = 1 << 6,
220 MLX5_ETH_WQE_L4_CSUM = 1 << 7,
221};
222
223enum {
224 MLX5_ETH_WQE_INSERT_VLAN = 1 << 15,
225};
226
227enum {
228 MLX5_ETH_WQE_SWP_INNER_L3_IPV6 = 1 << 0,
229 MLX5_ETH_WQE_SWP_INNER_L4_UDP = 1 << 1,
230 MLX5_ETH_WQE_SWP_OUTER_L3_IPV6 = 1 << 4,
231 MLX5_ETH_WQE_SWP_OUTER_L4_UDP = 1 << 5,
232};
233
234struct mlx5_wqe_eth_seg {
235 u8 swp_outer_l4_offset;
236 u8 swp_outer_l3_offset;
237 u8 swp_inner_l4_offset;
238 u8 swp_inner_l3_offset;
239 u8 cs_flags;
240 u8 swp_flags;
241 __be16 mss;
242 __be32 rsvd2;
243 union {
244 struct {
245 __be16 sz;
246 u8 start[2];
247 } inline_hdr;
248 struct {
249 __be16 type;
250 __be16 vlan_tci;
251 } insert;
252 };
253};
254
255struct mlx5_wqe_xrc_seg {
256 __be32 xrc_srqn;
257 u8 rsvd[12];
258};
259
260struct mlx5_wqe_masked_atomic_seg {
261 __be64 swap_add;
262 __be64 compare;
263 __be64 swap_add_mask;
264 __be64 compare_mask;
265};
266
267struct mlx5_base_av {
268 union {
269 struct {
270 __be32 qkey;
271 __be32 reserved;
272 } qkey;
273 __be64 dc_key;
274 } key;
275 __be32 dqp_dct;
276 u8 stat_rate_sl;
277 u8 fl_mlid;
278 union {
279 __be16 rlid;
280 __be16 udp_sport;
281 };
282};
283
284struct mlx5_av {
285 union {
286 struct {
287 __be32 qkey;
288 __be32 reserved;
289 } qkey;
290 __be64 dc_key;
291 } key;
292 __be32 dqp_dct;
293 u8 stat_rate_sl;
294 u8 fl_mlid;
295 union {
296 __be16 rlid;
297 __be16 udp_sport;
298 };
299 u8 reserved0[4];
300 u8 rmac[6];
301 u8 tclass;
302 u8 hop_limit;
303 __be32 grh_gid_fl;
304 u8 rgid[16];
305};
306
307struct mlx5_ib_ah {
308 struct ib_ah ibah;
309 struct mlx5_av av;
310};
311
312static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
313{
314 return container_of(ibah, struct mlx5_ib_ah, ibah);
315}
316
317struct mlx5_wqe_datagram_seg {
318 struct mlx5_av av;
319};
320
321struct mlx5_wqe_raddr_seg {
322 __be64 raddr;
323 __be32 rkey;
324 u32 reserved;
325};
326
327struct mlx5_wqe_atomic_seg {
328 __be64 swap_add;
329 __be64 compare;
330};
331
332struct mlx5_wqe_data_seg {
333 __be32 byte_count;
334 __be32 lkey;
335 __be64 addr;
336};
337
338struct mlx5_wqe_umr_ctrl_seg {
339 u8 flags;
340 u8 rsvd0[3];
341 __be16 xlt_octowords;
342 union {
343 __be16 xlt_offset;
344 __be16 bsf_octowords;
345 };
346 __be64 mkey_mask;
347 __be32 xlt_offset_47_16;
348 u8 rsvd1[28];
349};
350
351struct mlx5_seg_set_psv {
352 __be32 psv_num;
353 __be16 syndrome;
354 __be16 status;
355 __be32 transient_sig;
356 __be32 ref_tag;
357};
358
359struct mlx5_seg_get_psv {
360 u8 rsvd[19];
361 u8 num_psv;
362 __be32 l_key;
363 __be64 va;
364 __be32 psv_index[4];
365};
366
367struct mlx5_seg_check_psv {
368 u8 rsvd0[2];
369 __be16 err_coalescing_op;
370 u8 rsvd1[2];
371 __be16 xport_err_op;
372 u8 rsvd2[2];
373 __be16 xport_err_mask;
374 u8 rsvd3[7];
375 u8 num_psv;
376 __be32 l_key;
377 __be64 va;
378 __be32 psv_index[4];
379};
380
381struct mlx5_rwqe_sig {
382 u8 rsvd0[4];
383 u8 signature;
384 u8 rsvd1[11];
385};
386
387struct mlx5_wqe_signature_seg {
388 u8 rsvd0[4];
389 u8 signature;
390 u8 rsvd1[11];
391};
392
393#define MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK 0x3ff
394
395struct mlx5_wqe_inline_seg {
396 __be32 byte_count;
397};
398
399enum mlx5_sig_type {
400 MLX5_DIF_CRC = 0x1,
401 MLX5_DIF_IPCS = 0x2,
402};
403
404struct mlx5_bsf_inl {
405 __be16 vld_refresh;
406 __be16 dif_apptag;
407 __be32 dif_reftag;
408 u8 sig_type;
409 u8 rp_inv_seed;
410 u8 rsvd[3];
411 u8 dif_inc_ref_guard_check;
412 __be16 dif_app_bitmask_check;
413};
414
415struct mlx5_bsf {
416 struct mlx5_bsf_basic {
417 u8 bsf_size_sbs;
418 u8 check_byte_mask;
419 union {
420 u8 copy_byte_mask;
421 u8 bs_selector;
422 u8 rsvd_wflags;
423 } wire;
424 union {
425 u8 bs_selector;
426 u8 rsvd_mflags;
427 } mem;
428 __be32 raw_data_size;
429 __be32 w_bfs_psv;
430 __be32 m_bfs_psv;
431 } basic;
432 struct mlx5_bsf_ext {
433 __be32 t_init_gen_pro_size;
434 __be32 rsvd_epi_size;
435 __be32 w_tfs_psv;
436 __be32 m_tfs_psv;
437 } ext;
438 struct mlx5_bsf_inl w_inl;
439 struct mlx5_bsf_inl m_inl;
440};
441
442struct mlx5_mtt {
443 __be64 ptag;
444};
445
446struct mlx5_klm {
447 __be32 bcount;
448 __be32 key;
449 __be64 va;
450};
451
452struct mlx5_stride_block_entry {
453 __be16 stride;
454 __be16 bcount;
455 __be32 key;
456 __be64 va;
457};
458
459struct mlx5_stride_block_ctrl_seg {
460 __be32 bcount_per_cycle;
461 __be32 op;
462 __be32 repeat_count;
463 u16 rsvd;
464 __be16 num_entries;
465};
466
467struct mlx5_core_qp {
468 struct mlx5_core_rsc_common common;
469 void (*event) (struct mlx5_core_qp *, int);
470 int qpn;
471 struct mlx5_rsc_debug *dbg;
472 int pid;
473};
474
475struct mlx5_qp_path {
476 u8 fl_free_ar;
477 u8 rsvd3;
478 __be16 pkey_index;
479 u8 rsvd0;
480 u8 grh_mlid;
481 __be16 rlid;
482 u8 ackto_lt;
483 u8 mgid_index;
484 u8 static_rate;
485 u8 hop_limit;
486 __be32 tclass_flowlabel;
487 union {
488 u8 rgid[16];
489 u8 rip[16];
490 };
491 u8 f_dscp_ecn_prio;
492 u8 ecn_dscp;
493 __be16 udp_sport;
494 u8 dci_cfi_prio_sl;
495 u8 port;
496 u8 rmac[6];
497};
498
499
500struct mlx5_qp_context {
501 __be32 flags;
502 __be32 flags_pd;
503 u8 mtu_msgmax;
504 u8 rq_size_stride;
505 __be16 sq_crq_size;
506 __be32 qp_counter_set_usr_page;
507 __be32 wire_qpn;
508 __be32 log_pg_sz_remote_qpn;
509 struct mlx5_qp_path pri_path;
510 struct mlx5_qp_path alt_path;
511 __be32 params1;
512 u8 reserved2[4];
513 __be32 next_send_psn;
514 __be32 cqn_send;
515 __be32 deth_sqpn;
516 u8 reserved3[4];
517 __be32 last_acked_psn;
518 __be32 ssn;
519 __be32 params2;
520 __be32 rnr_nextrecvpsn;
521 __be32 xrcd;
522 __be32 cqn_recv;
523 __be64 db_rec_addr;
524 __be32 qkey;
525 __be32 rq_type_srqn;
526 __be32 rmsn;
527 __be16 hw_sq_wqe_counter;
528 __be16 sw_sq_wqe_counter;
529 __be16 hw_rcyclic_byte_counter;
530 __be16 hw_rq_counter;
531 __be16 sw_rcyclic_byte_counter;
532 __be16 sw_rq_counter;
533 u8 rsvd0[5];
534 u8 cgs;
535 u8 cs_req;
536 u8 cs_res;
537 __be64 dc_access_key;
538 u8 rsvd1[24];
539};
540
541static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u32 qpn)
542{
543 return radix_tree_lookup(&dev->priv.qp_table.tree, qpn);
544}
545
546static inline struct mlx5_core_mkey *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key)
547{
548 return radix_tree_lookup(&dev->priv.mkey_table.tree, key);
549}
550
551int mlx5_core_create_qp(struct mlx5_core_dev *dev,
552 struct mlx5_core_qp *qp,
553 u32 *in,
554 int inlen);
555int mlx5_core_qp_modify(struct mlx5_core_dev *dev, u16 opcode,
556 u32 opt_param_mask, void *qpc,
557 struct mlx5_core_qp *qp);
558int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
559 struct mlx5_core_qp *qp);
560int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
561 u32 *out, int outlen);
562
563int mlx5_core_set_delay_drop(struct mlx5_core_dev *dev,
564 u32 timeout_usec);
565
566int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn);
567int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn);
568void mlx5_init_qp_table(struct mlx5_core_dev *dev);
569void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev);
570int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
571void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
572int mlx5_core_create_rq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
573 struct mlx5_core_qp *rq);
574void mlx5_core_destroy_rq_tracked(struct mlx5_core_dev *dev,
575 struct mlx5_core_qp *rq);
576int mlx5_core_create_sq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
577 struct mlx5_core_qp *sq);
578void mlx5_core_destroy_sq_tracked(struct mlx5_core_dev *dev,
579 struct mlx5_core_qp *sq);
580int mlx5_core_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id);
581int mlx5_core_dealloc_q_counter(struct mlx5_core_dev *dev, u16 counter_id);
582int mlx5_core_query_q_counter(struct mlx5_core_dev *dev, u16 counter_id,
583 int reset, void *out, int out_size);
584
585static inline const char *mlx5_qp_type_str(int type)
586{
587 switch (type) {
588 case MLX5_QP_ST_RC: return "RC";
589 case MLX5_QP_ST_UC: return "C";
590 case MLX5_QP_ST_UD: return "UD";
591 case MLX5_QP_ST_XRC: return "XRC";
592 case MLX5_QP_ST_MLX: return "MLX";
593 case MLX5_QP_ST_QP0: return "QP0";
594 case MLX5_QP_ST_QP1: return "QP1";
595 case MLX5_QP_ST_RAW_ETHERTYPE: return "RAW_ETHERTYPE";
596 case MLX5_QP_ST_RAW_IPV6: return "RAW_IPV6";
597 case MLX5_QP_ST_SNIFFER: return "SNIFFER";
598 case MLX5_QP_ST_SYNC_UMR: return "SYNC_UMR";
599 case MLX5_QP_ST_PTP_1588: return "PTP_1588";
600 case MLX5_QP_ST_REG_UMR: return "REG_UMR";
601 default: return "Invalid transport type";
602 }
603}
604
605static inline const char *mlx5_qp_state_str(int state)
606{
607 switch (state) {
608 case MLX5_QP_STATE_RST:
609 return "RST";
610 case MLX5_QP_STATE_INIT:
611 return "INIT";
612 case MLX5_QP_STATE_RTR:
613 return "RTR";
614 case MLX5_QP_STATE_RTS:
615 return "RTS";
616 case MLX5_QP_STATE_SQER:
617 return "SQER";
618 case MLX5_QP_STATE_SQD:
619 return "SQD";
620 case MLX5_QP_STATE_ERR:
621 return "ERR";
622 case MLX5_QP_STATE_SQ_DRAINING:
623 return "SQ_DRAINING";
624 case MLX5_QP_STATE_SUSPENDED:
625 return "SUSPENDED";
626 default: return "Invalid QP state";
627 }
628}
629
630#endif
631
