1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#ifndef __MLX5_WQ_H__
34#define __MLX5_WQ_H__
35
36#include <linux/mlx5/mlx5_ifc.h>
37#include <linux/mlx5/cq.h>
38#include <linux/mlx5/qp.h>
39
40struct mlx5_wq_param {
41 int linear;
42 int buf_numa_node;
43 int db_numa_node;
44};
45
46struct mlx5_wq_ctrl {
47 struct mlx5_core_dev *mdev;
48 struct mlx5_frag_buf buf;
49 struct mlx5_db db;
50};
51
52struct mlx5_frag_wq_ctrl {
53 struct mlx5_core_dev *mdev;
54 struct mlx5_frag_buf frag_buf;
55 struct mlx5_db db;
56};
57
58struct mlx5_wq_cyc {
59 void *buf;
60 __be32 *db;
61 u16 sz_m1;
62 u8 log_stride;
63};
64
65struct mlx5_wq_qp {
66 struct mlx5_wq_cyc rq;
67 struct mlx5_wq_cyc sq;
68};
69
70struct mlx5_cqwq {
71 struct mlx5_frag_buf_ctrl fbc;
72 __be32 *db;
73 u32 cc;
74};
75
76struct mlx5_wq_ll {
77 void *buf;
78 __be32 *db;
79 __be16 *tail_next;
80 u16 sz_m1;
81 u16 head;
82 u16 wqe_ctr;
83 u16 cur_sz;
84 u8 log_stride;
85};
86
87int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
88 void *wqc, struct mlx5_wq_cyc *wq,
89 struct mlx5_wq_ctrl *wq_ctrl);
90u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq);
91
92int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
93 void *qpc, struct mlx5_wq_qp *wq,
94 struct mlx5_wq_ctrl *wq_ctrl);
95
96int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
97 void *cqc, struct mlx5_cqwq *wq,
98 struct mlx5_frag_wq_ctrl *wq_ctrl);
99u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq);
100
101int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
102 void *wqc, struct mlx5_wq_ll *wq,
103 struct mlx5_wq_ctrl *wq_ctrl);
104u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq);
105
106void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl);
107void mlx5_cqwq_destroy(struct mlx5_frag_wq_ctrl *wq_ctrl);
108
109static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)
110{
111 return ctr & wq->sz_m1;
112}
113
114static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix)
115{
116 return wq->buf + (ix << wq->log_stride);
117}
118
119static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2)
120{
121 int equal = (cc1 == cc2);
122 int smaller = 0x8000 & (cc1 - cc2);
123
124 return !equal && !smaller;
125}
126
127static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq)
128{
129 return wq->cc & wq->fbc.sz_m1;
130}
131
132static inline void *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix)
133{
134 return mlx5_frag_buf_get_wqe(&wq->fbc, ix);
135}
136
137static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq)
138{
139 return wq->cc >> wq->fbc.log_sz;
140}
141
142static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq)
143{
144 wq->cc++;
145}
146
147static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq)
148{
149 *wq->db = cpu_to_be32(wq->cc & 0xffffff);
150}
151
152static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq)
153{
154 u32 ci = mlx5_cqwq_get_ci(wq);
155 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
156 u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK;
157 u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1;
158
159 if (cqe_ownership_bit != sw_ownership_val)
160 return NULL;
161
162
163 dma_rmb();
164
165 return cqe;
166}
167
168static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq)
169{
170 return wq->cur_sz == wq->sz_m1;
171}
172
173static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq)
174{
175 return !wq->cur_sz;
176}
177
178static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix)
179{
180 return wq->buf + (ix << wq->log_stride);
181}
182
183static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next)
184{
185 wq->head = head_next;
186 wq->wqe_ctr++;
187 wq->cur_sz++;
188}
189
190static inline void mlx5_wq_ll_pop(struct mlx5_wq_ll *wq, __be16 ix,
191 __be16 *next_tail_next)
192{
193 *wq->tail_next = ix;
194 wq->tail_next = next_tail_next;
195 wq->cur_sz--;
196}
197
198static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq)
199{
200 *wq->db = cpu_to_be32(wq->wqe_ctr);
201}
202
203#endif
204