1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#ifndef MLX5_QP_H
34#define MLX5_QP_H
35
36#include <linux/mlx5/device.h>
37#include <linux/mlx5/driver.h>
38
39#define MLX5_INVALID_LKEY 0x100
40#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5)
41#define MLX5_DIF_SIZE 8
42#define MLX5_STRIDE_BLOCK_OP 0x400
43#define MLX5_CPY_GRD_MASK 0xc0
44#define MLX5_CPY_APP_MASK 0x30
45#define MLX5_CPY_REF_MASK 0x0f
46#define MLX5_BSF_INC_REFTAG (1 << 6)
47#define MLX5_BSF_INL_VALID (1 << 15)
48#define MLX5_BSF_REFRESH_DIF (1 << 14)
49#define MLX5_BSF_REPEAT_BLOCK (1 << 7)
50#define MLX5_BSF_APPTAG_ESCAPE 0x1
51#define MLX5_BSF_APPREF_ESCAPE 0x2
52
53enum mlx5_qp_optpar {
54 MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
55 MLX5_QP_OPTPAR_RRE = 1 << 1,
56 MLX5_QP_OPTPAR_RAE = 1 << 2,
57 MLX5_QP_OPTPAR_RWE = 1 << 3,
58 MLX5_QP_OPTPAR_PKEY_INDEX = 1 << 4,
59 MLX5_QP_OPTPAR_Q_KEY = 1 << 5,
60 MLX5_QP_OPTPAR_RNR_TIMEOUT = 1 << 6,
61 MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,
62 MLX5_QP_OPTPAR_SRA_MAX = 1 << 8,
63 MLX5_QP_OPTPAR_RRA_MAX = 1 << 9,
64 MLX5_QP_OPTPAR_PM_STATE = 1 << 10,
65 MLX5_QP_OPTPAR_RETRY_COUNT = 1 << 12,
66 MLX5_QP_OPTPAR_RNR_RETRY = 1 << 13,
67 MLX5_QP_OPTPAR_ACK_TIMEOUT = 1 << 14,
68 MLX5_QP_OPTPAR_PRI_PORT = 1 << 16,
69 MLX5_QP_OPTPAR_SRQN = 1 << 18,
70 MLX5_QP_OPTPAR_CQN_RCV = 1 << 19,
71 MLX5_QP_OPTPAR_DC_HS = 1 << 20,
72 MLX5_QP_OPTPAR_DC_KEY = 1 << 21,
73};
74
75enum mlx5_qp_state {
76 MLX5_QP_STATE_RST = 0,
77 MLX5_QP_STATE_INIT = 1,
78 MLX5_QP_STATE_RTR = 2,
79 MLX5_QP_STATE_RTS = 3,
80 MLX5_QP_STATE_SQER = 4,
81 MLX5_QP_STATE_SQD = 5,
82 MLX5_QP_STATE_ERR = 6,
83 MLX5_QP_STATE_SQ_DRAINING = 7,
84 MLX5_QP_STATE_SUSPENDED = 9,
85 MLX5_QP_NUM_STATE,
86 MLX5_QP_STATE,
87 MLX5_QP_STATE_BAD,
88};
89
90enum {
91 MLX5_SQ_STATE_NA = MLX5_SQC_STATE_ERR + 1,
92 MLX5_SQ_NUM_STATE = MLX5_SQ_STATE_NA + 1,
93 MLX5_RQ_STATE_NA = MLX5_RQC_STATE_ERR + 1,
94 MLX5_RQ_NUM_STATE = MLX5_RQ_STATE_NA + 1,
95};
96
97enum {
98 MLX5_QP_ST_RC = 0x0,
99 MLX5_QP_ST_UC = 0x1,
100 MLX5_QP_ST_UD = 0x2,
101 MLX5_QP_ST_XRC = 0x3,
102 MLX5_QP_ST_MLX = 0x4,
103 MLX5_QP_ST_DCI = 0x5,
104 MLX5_QP_ST_DCT = 0x6,
105 MLX5_QP_ST_QP0 = 0x7,
106 MLX5_QP_ST_QP1 = 0x8,
107 MLX5_QP_ST_RAW_ETHERTYPE = 0x9,
108 MLX5_QP_ST_RAW_IPV6 = 0xa,
109 MLX5_QP_ST_SNIFFER = 0xb,
110 MLX5_QP_ST_SYNC_UMR = 0xe,
111 MLX5_QP_ST_PTP_1588 = 0xd,
112 MLX5_QP_ST_REG_UMR = 0xc,
113 MLX5_QP_ST_MAX
114};
115
116enum {
117 MLX5_QP_PM_MIGRATED = 0x3,
118 MLX5_QP_PM_ARMED = 0x0,
119 MLX5_QP_PM_REARM = 0x1
120};
121
122enum {
123 MLX5_NON_ZERO_RQ = 0x0,
124 MLX5_SRQ_RQ = 0x1,
125 MLX5_CRQ_RQ = 0x2,
126 MLX5_ZERO_LEN_RQ = 0x3
127};
128
129
130enum {
131
132 MLX5_QP_BIT_SRE = 1 << 15,
133 MLX5_QP_BIT_SWE = 1 << 14,
134 MLX5_QP_BIT_SAE = 1 << 13,
135
136 MLX5_QP_BIT_RRE = 1 << 15,
137 MLX5_QP_BIT_RWE = 1 << 14,
138 MLX5_QP_BIT_RAE = 1 << 13,
139 MLX5_QP_BIT_RIC = 1 << 4,
140 MLX5_QP_BIT_CC_SLAVE_RECV = 1 << 2,
141 MLX5_QP_BIT_CC_SLAVE_SEND = 1 << 1,
142 MLX5_QP_BIT_CC_MASTER = 1 << 0
143};
144
145enum {
146 MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2,
147 MLX5_WQE_CTRL_CQ_UPDATE_AND_EQE = 3 << 2,
148 MLX5_WQE_CTRL_SOLICITED = 1 << 1,
149};
150
151enum {
152 MLX5_SEND_WQE_DS = 16,
153 MLX5_SEND_WQE_BB = 64,
154};
155
156#define MLX5_SEND_WQEBB_NUM_DS (MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
157
158enum {
159 MLX5_SEND_WQE_MAX_WQEBBS = 16,
160};
161
162enum {
163 MLX5_WQE_FMR_PERM_LOCAL_READ = 1 << 27,
164 MLX5_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28,
165 MLX5_WQE_FMR_PERM_REMOTE_READ = 1 << 29,
166 MLX5_WQE_FMR_PERM_REMOTE_WRITE = 1 << 30,
167 MLX5_WQE_FMR_PERM_ATOMIC = 1 << 31
168};
169
170enum {
171 MLX5_FENCE_MODE_NONE = 0 << 5,
172 MLX5_FENCE_MODE_INITIATOR_SMALL = 1 << 5,
173 MLX5_FENCE_MODE_FENCE = 2 << 5,
174 MLX5_FENCE_MODE_STRONG_ORDERING = 3 << 5,
175 MLX5_FENCE_MODE_SMALL_AND_FENCE = 4 << 5,
176};
177
178enum {
179 MLX5_RCV_DBR = 0,
180 MLX5_SND_DBR = 1,
181};
182
183enum {
184 MLX5_FLAGS_INLINE = 1<<7,
185 MLX5_FLAGS_CHECK_FREE = 1<<5,
186};
187
188struct mlx5_wqe_fmr_seg {
189 __be32 flags;
190 __be32 mem_key;
191 __be64 buf_list;
192 __be64 start_addr;
193 __be64 reg_len;
194 __be32 offset;
195 __be32 page_size;
196 u32 reserved[2];
197};
198
199struct mlx5_wqe_ctrl_seg {
200 __be32 opmod_idx_opcode;
201 __be32 qpn_ds;
202 u8 signature;
203 u8 rsvd[2];
204 u8 fm_ce_se;
205 __be32 imm;
206};
207
208#define MLX5_WQE_CTRL_DS_MASK 0x3f
209#define MLX5_WQE_CTRL_QPN_MASK 0xffffff00
210#define MLX5_WQE_CTRL_QPN_SHIFT 8
211#define MLX5_WQE_DS_UNITS 16
212#define MLX5_WQE_CTRL_OPCODE_MASK 0xff
213#define MLX5_WQE_CTRL_WQE_INDEX_MASK 0x00ffff00
214#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8
215
216enum {
217 MLX5_ETH_WQE_L3_INNER_CSUM = 1 << 4,
218 MLX5_ETH_WQE_L4_INNER_CSUM = 1 << 5,
219 MLX5_ETH_WQE_L3_CSUM = 1 << 6,
220 MLX5_ETH_WQE_L4_CSUM = 1 << 7,
221};
222
223enum {
224 MLX5_ETH_WQE_SVLAN = 1 << 0,
225 MLX5_ETH_WQE_INSERT_VLAN = 1 << 15,
226};
227
228enum {
229 MLX5_ETH_WQE_SWP_INNER_L3_IPV6 = 1 << 0,
230 MLX5_ETH_WQE_SWP_INNER_L4_UDP = 1 << 1,
231 MLX5_ETH_WQE_SWP_OUTER_L3_IPV6 = 1 << 4,
232 MLX5_ETH_WQE_SWP_OUTER_L4_UDP = 1 << 5,
233};
234
235struct mlx5_wqe_eth_seg {
236 u8 swp_outer_l4_offset;
237 u8 swp_outer_l3_offset;
238 u8 swp_inner_l4_offset;
239 u8 swp_inner_l3_offset;
240 u8 cs_flags;
241 u8 swp_flags;
242 __be16 mss;
243 __be32 rsvd2;
244 union {
245 struct {
246 __be16 sz;
247 u8 start[2];
248 } inline_hdr;
249 struct {
250 __be16 type;
251 __be16 vlan_tci;
252 } insert;
253 };
254};
255
256struct mlx5_wqe_xrc_seg {
257 __be32 xrc_srqn;
258 u8 rsvd[12];
259};
260
261struct mlx5_wqe_masked_atomic_seg {
262 __be64 swap_add;
263 __be64 compare;
264 __be64 swap_add_mask;
265 __be64 compare_mask;
266};
267
268struct mlx5_base_av {
269 union {
270 struct {
271 __be32 qkey;
272 __be32 reserved;
273 } qkey;
274 __be64 dc_key;
275 } key;
276 __be32 dqp_dct;
277 u8 stat_rate_sl;
278 u8 fl_mlid;
279 union {
280 __be16 rlid;
281 __be16 udp_sport;
282 };
283};
284
285struct mlx5_av {
286 union {
287 struct {
288 __be32 qkey;
289 __be32 reserved;
290 } qkey;
291 __be64 dc_key;
292 } key;
293 __be32 dqp_dct;
294 u8 stat_rate_sl;
295 u8 fl_mlid;
296 union {
297 __be16 rlid;
298 __be16 udp_sport;
299 };
300 u8 reserved0[4];
301 u8 rmac[6];
302 u8 tclass;
303 u8 hop_limit;
304 __be32 grh_gid_fl;
305 u8 rgid[16];
306};
307
308struct mlx5_ib_ah {
309 struct ib_ah ibah;
310 struct mlx5_av av;
311};
312
313static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
314{
315 return container_of(ibah, struct mlx5_ib_ah, ibah);
316}
317
318struct mlx5_wqe_datagram_seg {
319 struct mlx5_av av;
320};
321
322struct mlx5_wqe_raddr_seg {
323 __be64 raddr;
324 __be32 rkey;
325 u32 reserved;
326};
327
328struct mlx5_wqe_atomic_seg {
329 __be64 swap_add;
330 __be64 compare;
331};
332
333struct mlx5_wqe_data_seg {
334 __be32 byte_count;
335 __be32 lkey;
336 __be64 addr;
337};
338
339struct mlx5_wqe_umr_ctrl_seg {
340 u8 flags;
341 u8 rsvd0[3];
342 __be16 xlt_octowords;
343 union {
344 __be16 xlt_offset;
345 __be16 bsf_octowords;
346 };
347 __be64 mkey_mask;
348 __be32 xlt_offset_47_16;
349 u8 rsvd1[28];
350};
351
352struct mlx5_seg_set_psv {
353 __be32 psv_num;
354 __be16 syndrome;
355 __be16 status;
356 __be32 transient_sig;
357 __be32 ref_tag;
358};
359
360struct mlx5_seg_get_psv {
361 u8 rsvd[19];
362 u8 num_psv;
363 __be32 l_key;
364 __be64 va;
365 __be32 psv_index[4];
366};
367
368struct mlx5_seg_check_psv {
369 u8 rsvd0[2];
370 __be16 err_coalescing_op;
371 u8 rsvd1[2];
372 __be16 xport_err_op;
373 u8 rsvd2[2];
374 __be16 xport_err_mask;
375 u8 rsvd3[7];
376 u8 num_psv;
377 __be32 l_key;
378 __be64 va;
379 __be32 psv_index[4];
380};
381
382struct mlx5_rwqe_sig {
383 u8 rsvd0[4];
384 u8 signature;
385 u8 rsvd1[11];
386};
387
388struct mlx5_wqe_signature_seg {
389 u8 rsvd0[4];
390 u8 signature;
391 u8 rsvd1[11];
392};
393
394#define MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK 0x3ff
395
396struct mlx5_wqe_inline_seg {
397 __be32 byte_count;
398};
399
400enum mlx5_sig_type {
401 MLX5_DIF_CRC = 0x1,
402 MLX5_DIF_IPCS = 0x2,
403};
404
405struct mlx5_bsf_inl {
406 __be16 vld_refresh;
407 __be16 dif_apptag;
408 __be32 dif_reftag;
409 u8 sig_type;
410 u8 rp_inv_seed;
411 u8 rsvd[3];
412 u8 dif_inc_ref_guard_check;
413 __be16 dif_app_bitmask_check;
414};
415
416struct mlx5_bsf {
417 struct mlx5_bsf_basic {
418 u8 bsf_size_sbs;
419 u8 check_byte_mask;
420 union {
421 u8 copy_byte_mask;
422 u8 bs_selector;
423 u8 rsvd_wflags;
424 } wire;
425 union {
426 u8 bs_selector;
427 u8 rsvd_mflags;
428 } mem;
429 __be32 raw_data_size;
430 __be32 w_bfs_psv;
431 __be32 m_bfs_psv;
432 } basic;
433 struct mlx5_bsf_ext {
434 __be32 t_init_gen_pro_size;
435 __be32 rsvd_epi_size;
436 __be32 w_tfs_psv;
437 __be32 m_tfs_psv;
438 } ext;
439 struct mlx5_bsf_inl w_inl;
440 struct mlx5_bsf_inl m_inl;
441};
442
443struct mlx5_mtt {
444 __be64 ptag;
445};
446
447struct mlx5_klm {
448 __be32 bcount;
449 __be32 key;
450 __be64 va;
451};
452
453struct mlx5_stride_block_entry {
454 __be16 stride;
455 __be16 bcount;
456 __be32 key;
457 __be64 va;
458};
459
460struct mlx5_stride_block_ctrl_seg {
461 __be32 bcount_per_cycle;
462 __be32 op;
463 __be32 repeat_count;
464 u16 rsvd;
465 __be16 num_entries;
466};
467
468struct mlx5_core_qp {
469 struct mlx5_core_rsc_common common;
470 void (*event) (struct mlx5_core_qp *, int);
471 int qpn;
472 struct mlx5_rsc_debug *dbg;
473 int pid;
474 u16 uid;
475};
476
477struct mlx5_core_dct {
478 struct mlx5_core_qp mqp;
479 struct completion drained;
480};
481
482struct mlx5_qp_path {
483 u8 fl_free_ar;
484 u8 rsvd3;
485 __be16 pkey_index;
486 u8 rsvd0;
487 u8 grh_mlid;
488 __be16 rlid;
489 u8 ackto_lt;
490 u8 mgid_index;
491 u8 static_rate;
492 u8 hop_limit;
493 __be32 tclass_flowlabel;
494 union {
495 u8 rgid[16];
496 u8 rip[16];
497 };
498 u8 f_dscp_ecn_prio;
499 u8 ecn_dscp;
500 __be16 udp_sport;
501 u8 dci_cfi_prio_sl;
502 u8 port;
503 u8 rmac[6];
504};
505
506
507struct mlx5_qp_context {
508 __be32 flags;
509 __be32 flags_pd;
510 u8 mtu_msgmax;
511 u8 rq_size_stride;
512 __be16 sq_crq_size;
513 __be32 qp_counter_set_usr_page;
514 __be32 wire_qpn;
515 __be32 log_pg_sz_remote_qpn;
516 struct mlx5_qp_path pri_path;
517 struct mlx5_qp_path alt_path;
518 __be32 params1;
519 u8 reserved2[4];
520 __be32 next_send_psn;
521 __be32 cqn_send;
522 __be32 deth_sqpn;
523 u8 reserved3[4];
524 __be32 last_acked_psn;
525 __be32 ssn;
526 __be32 params2;
527 __be32 rnr_nextrecvpsn;
528 __be32 xrcd;
529 __be32 cqn_recv;
530 __be64 db_rec_addr;
531 __be32 qkey;
532 __be32 rq_type_srqn;
533 __be32 rmsn;
534 __be16 hw_sq_wqe_counter;
535 __be16 sw_sq_wqe_counter;
536 __be16 hw_rcyclic_byte_counter;
537 __be16 hw_rq_counter;
538 __be16 sw_rcyclic_byte_counter;
539 __be16 sw_rq_counter;
540 u8 rsvd0[5];
541 u8 cgs;
542 u8 cs_req;
543 u8 cs_res;
544 __be64 dc_access_key;
545 u8 rsvd1[24];
546};
547
548static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u32 qpn)
549{
550 return radix_tree_lookup(&dev->priv.qp_table.tree, qpn);
551}
552
553static inline struct mlx5_core_mkey *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key)
554{
555 return radix_tree_lookup(&dev->priv.mkey_table.tree, key);
556}
557
558int mlx5_core_create_dct(struct mlx5_core_dev *dev,
559 struct mlx5_core_dct *qp,
560 u32 *in, int inlen);
561int mlx5_core_create_qp(struct mlx5_core_dev *dev,
562 struct mlx5_core_qp *qp,
563 u32 *in,
564 int inlen);
565int mlx5_core_qp_modify(struct mlx5_core_dev *dev, u16 opcode,
566 u32 opt_param_mask, void *qpc,
567 struct mlx5_core_qp *qp);
568int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
569 struct mlx5_core_qp *qp);
570int mlx5_core_destroy_dct(struct mlx5_core_dev *dev,
571 struct mlx5_core_dct *dct);
572int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
573 u32 *out, int outlen);
574int mlx5_core_dct_query(struct mlx5_core_dev *dev, struct mlx5_core_dct *dct,
575 u32 *out, int outlen);
576
577int mlx5_core_set_delay_drop(struct mlx5_core_dev *dev,
578 u32 timeout_usec);
579
580int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn);
581int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn);
582void mlx5_init_qp_table(struct mlx5_core_dev *dev);
583void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev);
584int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
585void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
586int mlx5_core_create_rq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
587 struct mlx5_core_qp *rq);
588void mlx5_core_destroy_rq_tracked(struct mlx5_core_dev *dev,
589 struct mlx5_core_qp *rq);
590int mlx5_core_create_sq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
591 struct mlx5_core_qp *sq);
592void mlx5_core_destroy_sq_tracked(struct mlx5_core_dev *dev,
593 struct mlx5_core_qp *sq);
594int mlx5_core_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id);
595int mlx5_core_dealloc_q_counter(struct mlx5_core_dev *dev, u16 counter_id);
596int mlx5_core_query_q_counter(struct mlx5_core_dev *dev, u16 counter_id,
597 int reset, void *out, int out_size);
598
599struct mlx5_core_rsc_common *mlx5_core_res_hold(struct mlx5_core_dev *dev,
600 int res_num,
601 enum mlx5_res_type res_type);
602void mlx5_core_res_put(struct mlx5_core_rsc_common *res);
603
604static inline const char *mlx5_qp_type_str(int type)
605{
606 switch (type) {
607 case MLX5_QP_ST_RC: return "RC";
608 case MLX5_QP_ST_UC: return "C";
609 case MLX5_QP_ST_UD: return "UD";
610 case MLX5_QP_ST_XRC: return "XRC";
611 case MLX5_QP_ST_MLX: return "MLX";
612 case MLX5_QP_ST_QP0: return "QP0";
613 case MLX5_QP_ST_QP1: return "QP1";
614 case MLX5_QP_ST_RAW_ETHERTYPE: return "RAW_ETHERTYPE";
615 case MLX5_QP_ST_RAW_IPV6: return "RAW_IPV6";
616 case MLX5_QP_ST_SNIFFER: return "SNIFFER";
617 case MLX5_QP_ST_SYNC_UMR: return "SYNC_UMR";
618 case MLX5_QP_ST_PTP_1588: return "PTP_1588";
619 case MLX5_QP_ST_REG_UMR: return "REG_UMR";
620 default: return "Invalid transport type";
621 }
622}
623
624static inline const char *mlx5_qp_state_str(int state)
625{
626 switch (state) {
627 case MLX5_QP_STATE_RST:
628 return "RST";
629 case MLX5_QP_STATE_INIT:
630 return "INIT";
631 case MLX5_QP_STATE_RTR:
632 return "RTR";
633 case MLX5_QP_STATE_RTS:
634 return "RTS";
635 case MLX5_QP_STATE_SQER:
636 return "SQER";
637 case MLX5_QP_STATE_SQD:
638 return "SQD";
639 case MLX5_QP_STATE_ERR:
640 return "ERR";
641 case MLX5_QP_STATE_SQ_DRAINING:
642 return "SQ_DRAINING";
643 case MLX5_QP_STATE_SUSPENDED:
644 return "SUSPENDED";
645 default: return "Invalid QP state";
646 }
647}
648
649#endif
650
