1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/mlx5/driver.h>
34#include "wq.h"
35#include "mlx5_core.h"
36
37u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq)
38{
39 return (u32)wq->sz_m1 + 1;
40}
41
42u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq)
43{
44 return wq->fbc.sz_m1 + 1;
45}
46
47u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq)
48{
49 return (u32)wq->sz_m1 + 1;
50}
51
52static u32 mlx5_wq_cyc_get_byte_size(struct mlx5_wq_cyc *wq)
53{
54 return mlx5_wq_cyc_get_size(wq) << wq->log_stride;
55}
56
57static u32 mlx5_wq_qp_get_byte_size(struct mlx5_wq_qp *wq)
58{
59 return mlx5_wq_cyc_get_byte_size(&wq->rq) +
60 mlx5_wq_cyc_get_byte_size(&wq->sq);
61}
62
63static u32 mlx5_cqwq_get_byte_size(struct mlx5_cqwq *wq)
64{
65 return mlx5_cqwq_get_size(wq) << wq->fbc.log_stride;
66}
67
68static u32 mlx5_wq_ll_get_byte_size(struct mlx5_wq_ll *wq)
69{
70 return mlx5_wq_ll_get_size(wq) << wq->log_stride;
71}
72
73int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
74 void *wqc, struct mlx5_wq_cyc *wq,
75 struct mlx5_wq_ctrl *wq_ctrl)
76{
77 int err;
78
79 wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
80 wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
81
82 err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
83 if (err) {
84 mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
85 return err;
86 }
87
88 err = mlx5_buf_alloc_node(mdev, mlx5_wq_cyc_get_byte_size(wq),
89 &wq_ctrl->buf, param->buf_numa_node);
90 if (err) {
91 mlx5_core_warn(mdev, "mlx5_buf_alloc_node() failed, %d\n", err);
92 goto err_db_free;
93 }
94
95 wq->buf = wq_ctrl->buf.frags->buf;
96 wq->db = wq_ctrl->db.db;
97
98 wq_ctrl->mdev = mdev;
99
100 return 0;
101
102err_db_free:
103 mlx5_db_free(mdev, &wq_ctrl->db);
104
105 return err;
106}
107
108int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
109 void *qpc, struct mlx5_wq_qp *wq,
110 struct mlx5_wq_ctrl *wq_ctrl)
111{
112 int err;
113
114 wq->rq.log_stride = MLX5_GET(qpc, qpc, log_rq_stride) + 4;
115 wq->rq.sz_m1 = (1 << MLX5_GET(qpc, qpc, log_rq_size)) - 1;
116
117 wq->sq.log_stride = ilog2(MLX5_SEND_WQE_BB);
118 wq->sq.sz_m1 = (1 << MLX5_GET(qpc, qpc, log_sq_size)) - 1;
119
120 err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
121 if (err) {
122 mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
123 return err;
124 }
125
126 err = mlx5_buf_alloc_node(mdev, mlx5_wq_qp_get_byte_size(wq),
127 &wq_ctrl->buf, param->buf_numa_node);
128 if (err) {
129 mlx5_core_warn(mdev, "mlx5_buf_alloc_node() failed, %d\n", err);
130 goto err_db_free;
131 }
132
133 wq->rq.buf = wq_ctrl->buf.frags->buf;
134 wq->sq.buf = wq->rq.buf + mlx5_wq_cyc_get_byte_size(&wq->rq);
135 wq->rq.db = &wq_ctrl->db.db[MLX5_RCV_DBR];
136 wq->sq.db = &wq_ctrl->db.db[MLX5_SND_DBR];
137
138 wq_ctrl->mdev = mdev;
139
140 return 0;
141
142err_db_free:
143 mlx5_db_free(mdev, &wq_ctrl->db);
144
145 return err;
146}
147
148int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
149 void *cqc, struct mlx5_cqwq *wq,
150 struct mlx5_frag_wq_ctrl *wq_ctrl)
151{
152 int err;
153
154 mlx5_core_init_cq_frag_buf(&wq->fbc, cqc);
155
156 err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
157 if (err) {
158 mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
159 return err;
160 }
161
162 err = mlx5_frag_buf_alloc_node(mdev, mlx5_cqwq_get_byte_size(wq),
163 &wq_ctrl->frag_buf,
164 param->buf_numa_node);
165 if (err) {
166 mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n",
167 err);
168 goto err_db_free;
169 }
170
171 wq->fbc.frag_buf = wq_ctrl->frag_buf;
172 wq->db = wq_ctrl->db.db;
173
174 wq_ctrl->mdev = mdev;
175
176 return 0;
177
178err_db_free:
179 mlx5_db_free(mdev, &wq_ctrl->db);
180
181 return err;
182}
183
184int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
185 void *wqc, struct mlx5_wq_ll *wq,
186 struct mlx5_wq_ctrl *wq_ctrl)
187{
188 struct mlx5_wqe_srq_next_seg *next_seg;
189 int err;
190 int i;
191
192 wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
193 wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
194
195 err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
196 if (err) {
197 mlx5_core_warn(mdev, "mlx5_db_alloc_node() failed, %d\n", err);
198 return err;
199 }
200
201 err = mlx5_buf_alloc_node(mdev, mlx5_wq_ll_get_byte_size(wq),
202 &wq_ctrl->buf, param->buf_numa_node);
203 if (err) {
204 mlx5_core_warn(mdev, "mlx5_buf_alloc_node() failed, %d\n", err);
205 goto err_db_free;
206 }
207
208 wq->buf = wq_ctrl->buf.frags->buf;
209 wq->db = wq_ctrl->db.db;
210
211 for (i = 0; i < wq->sz_m1; i++) {
212 next_seg = mlx5_wq_ll_get_wqe(wq, i);
213 next_seg->next_wqe_index = cpu_to_be16(i + 1);
214 }
215 next_seg = mlx5_wq_ll_get_wqe(wq, i);
216 wq->tail_next = &next_seg->next_wqe_index;
217
218 wq_ctrl->mdev = mdev;
219
220 return 0;
221
222err_db_free:
223 mlx5_db_free(mdev, &wq_ctrl->db);
224
225 return err;
226}
227
228void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl)
229{
230 mlx5_buf_free(wq_ctrl->mdev, &wq_ctrl->buf);
231 mlx5_db_free(wq_ctrl->mdev, &wq_ctrl->db);
232}
233
234void mlx5_cqwq_destroy(struct mlx5_frag_wq_ctrl *wq_ctrl)
235{
236 mlx5_frag_buf_free(wq_ctrl->mdev, &wq_ctrl->frag_buf);
237 mlx5_db_free(wq_ctrl->mdev, &wq_ctrl->db);
238}
239