1
2
3
4#include <linux/mlx5/driver.h>
5#include "mlx5_vdpa.h"
6
7static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid)
8{
9 struct mlx5_core_dev *mdev = dev->mdev;
10
11 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
12 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
13 int err;
14
15 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
16 MLX5_SET(alloc_pd_in, in, uid, uid);
17
18 err = mlx5_cmd_exec_inout(mdev, alloc_pd, in, out);
19 if (!err)
20 *pdn = MLX5_GET(alloc_pd_out, out, pd);
21
22 return err;
23}
24
25static int dealloc_pd(struct mlx5_vdpa_dev *dev, u32 pdn, u16 uid)
26{
27 u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {};
28 struct mlx5_core_dev *mdev = dev->mdev;
29
30 MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
31 MLX5_SET(dealloc_pd_in, in, pd, pdn);
32 MLX5_SET(dealloc_pd_in, in, uid, uid);
33 return mlx5_cmd_exec_in(mdev, dealloc_pd, in);
34}
35
36static int get_null_mkey(struct mlx5_vdpa_dev *dev, u32 *null_mkey)
37{
38 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
39 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
40 struct mlx5_core_dev *mdev = dev->mdev;
41 int err;
42
43 MLX5_SET(query_special_contexts_in, in, opcode, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
44 err = mlx5_cmd_exec_inout(mdev, query_special_contexts, in, out);
45 if (!err)
46 *null_mkey = MLX5_GET(query_special_contexts_out, out, null_mkey);
47 return err;
48}
49
50static int create_uctx(struct mlx5_vdpa_dev *mvdev, u16 *uid)
51{
52 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
53 int inlen;
54 void *in;
55 int err;
56
57 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0))
58 return 0;
59
60
61 if (!MLX5_CAP_GEN(mvdev->mdev, log_max_uctx))
62 return -EOPNOTSUPP;
63
64 inlen = MLX5_ST_SZ_BYTES(create_uctx_in);
65 in = kzalloc(inlen, GFP_KERNEL);
66 if (!in)
67 return -ENOMEM;
68
69 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
70 MLX5_SET(create_uctx_in, in, uctx.cap, MLX5_UCTX_CAP_RAW_TX);
71
72 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
73 kfree(in);
74 if (!err)
75 *uid = MLX5_GET(create_uctx_out, out, uid);
76
77 return err;
78}
79
80static void destroy_uctx(struct mlx5_vdpa_dev *mvdev, u32 uid)
81{
82 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
83 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
84
85 if (!uid)
86 return;
87
88 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
89 MLX5_SET(destroy_uctx_in, in, uid, uid);
90
91 mlx5_cmd_exec(mvdev->mdev, in, sizeof(in), out, sizeof(out));
92}
93
94int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tisn)
95{
96 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
97 int err;
98
99 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
100 MLX5_SET(create_tis_in, in, uid, mvdev->res.uid);
101 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tis, in, out);
102 if (!err)
103 *tisn = MLX5_GET(create_tis_out, out, tisn);
104
105 return err;
106}
107
108void mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev *mvdev, u32 tisn)
109{
110 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};
111
112 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
113 MLX5_SET(destroy_tis_in, in, uid, mvdev->res.uid);
114 MLX5_SET(destroy_tis_in, in, tisn, tisn);
115 mlx5_cmd_exec_in(mvdev->mdev, destroy_tis, in);
116}
117
118int mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 *rqtn)
119{
120 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
121 int err;
122
123 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
124 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out));
125 if (!err)
126 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
127
128 return err;
129}
130
131void mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev *mvdev, u32 rqtn)
132{
133 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};
134
135 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
136 MLX5_SET(destroy_rqt_in, in, uid, mvdev->res.uid);
137 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
138 mlx5_cmd_exec_in(mvdev->mdev, destroy_rqt, in);
139}
140
141int mlx5_vdpa_create_tir(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tirn)
142{
143 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
144 int err;
145
146 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
147 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tir, in, out);
148 if (!err)
149 *tirn = MLX5_GET(create_tir_out, out, tirn);
150
151 return err;
152}
153
154void mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev *mvdev, u32 tirn)
155{
156 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
157
158 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
159 MLX5_SET(destroy_tir_in, in, uid, mvdev->res.uid);
160 MLX5_SET(destroy_tir_in, in, tirn, tirn);
161 mlx5_cmd_exec_in(mvdev->mdev, destroy_tir, in);
162}
163
164int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 *tdn)
165{
166 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
167 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
168 int err;
169
170 MLX5_SET(alloc_transport_domain_in, in, opcode, MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
171 MLX5_SET(alloc_transport_domain_in, in, uid, mvdev->res.uid);
172
173 err = mlx5_cmd_exec_inout(mvdev->mdev, alloc_transport_domain, in, out);
174 if (!err)
175 *tdn = MLX5_GET(alloc_transport_domain_out, out, transport_domain);
176
177 return err;
178}
179
180void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn)
181{
182 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};
183
184 MLX5_SET(dealloc_transport_domain_in, in, opcode, MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
185 MLX5_SET(dealloc_transport_domain_in, in, uid, mvdev->res.uid);
186 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
187 mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in);
188}
189
190int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey, u32 *in,
191 int inlen)
192{
193 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {};
194 u32 mkey_index;
195 void *mkc;
196 int err;
197
198 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
199 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
200
201 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, lout, sizeof(lout));
202 if (err)
203 return err;
204
205 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
206 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
207 mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
208 mkey->size = MLX5_GET64(mkc, mkc, len);
209 mkey->key |= mlx5_idx_to_mkey(mkey_index);
210 mkey->pd = MLX5_GET(mkc, mkc, pd);
211 return 0;
212}
213
214int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey)
215{
216 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {};
217
218 MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
219 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
220 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
221 return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in);
222}
223
224int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev)
225{
226 u64 offset = MLX5_CAP64_DEV_VDPA_EMULATION(mvdev->mdev, doorbell_bar_offset);
227 struct mlx5_vdpa_resources *res = &mvdev->res;
228 struct mlx5_core_dev *mdev = mvdev->mdev;
229 u64 kick_addr;
230 int err;
231
232 if (res->valid) {
233 mlx5_vdpa_warn(mvdev, "resources already allocated\n");
234 return -EINVAL;
235 }
236 mutex_init(&mvdev->mr.mkey_mtx);
237 res->uar = mlx5_get_uars_page(mdev);
238 if (IS_ERR(res->uar)) {
239 err = PTR_ERR(res->uar);
240 goto err_uars;
241 }
242
243 err = create_uctx(mvdev, &res->uid);
244 if (err)
245 goto err_uctx;
246
247 err = alloc_pd(mvdev, &res->pdn, res->uid);
248 if (err)
249 goto err_pd;
250
251 err = get_null_mkey(mvdev, &res->null_mkey);
252 if (err)
253 goto err_key;
254
255 kick_addr = mdev->bar_addr + offset;
256 res->phys_kick_addr = kick_addr;
257
258 res->kick_addr = ioremap(kick_addr, PAGE_SIZE);
259 if (!res->kick_addr) {
260 err = -ENOMEM;
261 goto err_key;
262 }
263 res->valid = true;
264
265 return 0;
266
267err_key:
268 dealloc_pd(mvdev, res->pdn, res->uid);
269err_pd:
270 destroy_uctx(mvdev, res->uid);
271err_uctx:
272 mlx5_put_uars_page(mdev, res->uar);
273err_uars:
274 mutex_destroy(&mvdev->mr.mkey_mtx);
275 return err;
276}
277
278void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev)
279{
280 struct mlx5_vdpa_resources *res = &mvdev->res;
281
282 if (!res->valid)
283 return;
284
285 iounmap(res->kick_addr);
286 res->kick_addr = NULL;
287 dealloc_pd(mvdev, res->pdn, res->uid);
288 destroy_uctx(mvdev, res->uid);
289 mlx5_put_uars_page(mvdev->mdev, res->uar);
290 mutex_destroy(&mvdev->mr.mkey_mtx);
291 res->valid = false;
292}
293
