1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/export.h>
34#include <linux/mlx5/driver.h>
35#include <linux/mlx5/flow_table.h>
36#include "mlx5_core.h"
37
38struct mlx5_ftg {
39 struct mlx5_flow_table_group g;
40 u32 id;
41 u32 start_ix;
42};
43
44struct mlx5_flow_table {
45 struct mlx5_core_dev *dev;
46 u8 level;
47 u8 type;
48 u32 id;
49 struct mutex mutex;
50 u16 num_groups;
51 struct mlx5_ftg *group;
52 unsigned long *bitmap;
53 u32 size;
54};
55
56static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table *ft, u32 group_ix,
57 u32 flow_index, void *flow_context)
58{
59 u32 out[MLX5_ST_SZ_DW(set_fte_out)];
60 u32 *in;
61 void *in_flow_context;
62 int fcdls =
63 MLX5_GET(flow_context, flow_context, destination_list_size) *
64 MLX5_ST_SZ_BYTES(dest_format_struct);
65 int inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fcdls;
66 int err;
67
68 in = mlx5_vzalloc(inlen);
69 if (!in) {
70 mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
71 return -ENOMEM;
72 }
73
74 MLX5_SET(set_fte_in, in, table_type, ft->type);
75 MLX5_SET(set_fte_in, in, table_id, ft->id);
76 MLX5_SET(set_fte_in, in, flow_index, flow_index);
77 MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
78
79 in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
80 memcpy(in_flow_context, flow_context,
81 MLX5_ST_SZ_BYTES(flow_context) + fcdls);
82
83 MLX5_SET(flow_context, in_flow_context, group_id,
84 ft->group[group_ix].id);
85
86 memset(out, 0, sizeof(out));
87 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
88 sizeof(out));
89 kvfree(in);
90
91 return err;
92}
93
94static void mlx5_del_flow_entry_cmd(struct mlx5_flow_table *ft, u32 flow_index)
95{
96 u32 in[MLX5_ST_SZ_DW(delete_fte_in)];
97 u32 out[MLX5_ST_SZ_DW(delete_fte_out)];
98
99 memset(in, 0, sizeof(in));
100 memset(out, 0, sizeof(out));
101
102#define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v)
103 MLX5_SET_DFTEI(in, table_type, ft->type);
104 MLX5_SET_DFTEI(in, table_id, ft->id);
105 MLX5_SET_DFTEI(in, flow_index, flow_index);
106 MLX5_SET_DFTEI(in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
107
108 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
109}
110
111static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table *ft, int i)
112{
113 u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)];
114 u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)];
115
116 memset(in, 0, sizeof(in));
117 memset(out, 0, sizeof(out));
118
119#define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v)
120 MLX5_SET_DFGI(in, table_type, ft->type);
121 MLX5_SET_DFGI(in, table_id, ft->id);
122 MLX5_SET_DFGI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP);
123 MLX5_SET_DFGI(in, group_id, ft->group[i].id);
124 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
125}
126
127static int mlx5_create_flow_group_cmd(struct mlx5_flow_table *ft, int i)
128{
129 u32 out[MLX5_ST_SZ_DW(create_flow_group_out)];
130 u32 *in;
131 void *in_match_criteria;
132 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
133 struct mlx5_flow_table_group *g = &ft->group[i].g;
134 u32 start_ix = ft->group[i].start_ix;
135 u32 end_ix = start_ix + (1 << g->log_sz) - 1;
136 int err;
137
138 in = mlx5_vzalloc(inlen);
139 if (!in) {
140 mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
141 return -ENOMEM;
142 }
143 in_match_criteria = MLX5_ADDR_OF(create_flow_group_in, in,
144 match_criteria);
145
146 memset(out, 0, sizeof(out));
147
148#define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v)
149 MLX5_SET_CFGI(in, table_type, ft->type);
150 MLX5_SET_CFGI(in, table_id, ft->id);
151 MLX5_SET_CFGI(in, opcode, MLX5_CMD_OP_CREATE_FLOW_GROUP);
152 MLX5_SET_CFGI(in, start_flow_index, start_ix);
153 MLX5_SET_CFGI(in, end_flow_index, end_ix);
154 MLX5_SET_CFGI(in, match_criteria_enable, g->match_criteria_enable);
155
156 memcpy(in_match_criteria, g->match_criteria,
157 MLX5_ST_SZ_BYTES(fte_match_param));
158
159 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
160 sizeof(out));
161 if (!err)
162 ft->group[i].id = MLX5_GET(create_flow_group_out, out,
163 group_id);
164
165 kvfree(in);
166
167 return err;
168}
169
170static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table *ft)
171{
172 int i;
173
174 for (i = 0; i < ft->num_groups; i++)
175 mlx5_destroy_flow_group_cmd(ft, i);
176}
177
178static int mlx5_create_flow_table_groups(struct mlx5_flow_table *ft)
179{
180 int err;
181 int i;
182
183 for (i = 0; i < ft->num_groups; i++) {
184 err = mlx5_create_flow_group_cmd(ft, i);
185 if (err)
186 goto err_destroy_flow_table_groups;
187 }
188
189 return 0;
190
191err_destroy_flow_table_groups:
192 for (i--; i >= 0; i--)
193 mlx5_destroy_flow_group_cmd(ft, i);
194
195 return err;
196}
197
198static int mlx5_create_flow_table_cmd(struct mlx5_flow_table *ft)
199{
200 u32 in[MLX5_ST_SZ_DW(create_flow_table_in)];
201 u32 out[MLX5_ST_SZ_DW(create_flow_table_out)];
202 int err;
203
204 memset(in, 0, sizeof(in));
205
206 MLX5_SET(create_flow_table_in, in, table_type, ft->type);
207 MLX5_SET(create_flow_table_in, in, level, ft->level);
208 MLX5_SET(create_flow_table_in, in, log_size, order_base_2(ft->size));
209
210 MLX5_SET(create_flow_table_in, in, opcode,
211 MLX5_CMD_OP_CREATE_FLOW_TABLE);
212
213 memset(out, 0, sizeof(out));
214 err = mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out,
215 sizeof(out));
216 if (err)
217 return err;
218
219 ft->id = MLX5_GET(create_flow_table_out, out, table_id);
220
221 return 0;
222}
223
224static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table *ft)
225{
226 u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)];
227 u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)];
228
229 memset(in, 0, sizeof(in));
230 memset(out, 0, sizeof(out));
231
232#define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v)
233 MLX5_SET_DFTI(in, table_type, ft->type);
234 MLX5_SET_DFTI(in, table_id, ft->id);
235 MLX5_SET_DFTI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE);
236
237 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
238}
239
240static int mlx5_find_group(struct mlx5_flow_table *ft, u8 match_criteria_enable,
241 u32 *match_criteria, int *group_ix)
242{
243 void *mc_outer = MLX5_ADDR_OF(fte_match_param, match_criteria,
244 outer_headers);
245 void *mc_misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
246 misc_parameters);
247 void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
248 inner_headers);
249 int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
250 int mc_misc_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc);
251 int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
252 int i;
253
254 for (i = 0; i < ft->num_groups; i++) {
255 struct mlx5_flow_table_group *g = &ft->group[i].g;
256 void *gmc_outer = MLX5_ADDR_OF(fte_match_param,
257 g->match_criteria,
258 outer_headers);
259 void *gmc_misc = MLX5_ADDR_OF(fte_match_param,
260 g->match_criteria,
261 misc_parameters);
262 void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
263 g->match_criteria,
264 inner_headers);
265
266 if (g->match_criteria_enable != match_criteria_enable)
267 continue;
268
269 if (match_criteria_enable & MLX5_MATCH_OUTER_HEADERS)
270 if (memcmp(mc_outer, gmc_outer, mc_outer_sz))
271 continue;
272
273 if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS)
274 if (memcmp(mc_misc, gmc_misc, mc_misc_sz))
275 continue;
276
277 if (match_criteria_enable & MLX5_MATCH_INNER_HEADERS)
278 if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
279 continue;
280
281 *group_ix = i;
282 return 0;
283 }
284
285 return -EINVAL;
286}
287
288static int alloc_flow_index(struct mlx5_flow_table *ft, int group_ix, u32 *ix)
289{
290 struct mlx5_ftg *g = &ft->group[group_ix];
291 int err = 0;
292
293 mutex_lock(&ft->mutex);
294
295 *ix = find_next_zero_bit(ft->bitmap, ft->size, g->start_ix);
296 if (*ix >= (g->start_ix + (1 << g->g.log_sz)))
297 err = -ENOSPC;
298 else
299 __set_bit(*ix, ft->bitmap);
300
301 mutex_unlock(&ft->mutex);
302
303 return err;
304}
305
306static void mlx5_free_flow_index(struct mlx5_flow_table *ft, u32 ix)
307{
308 __clear_bit(ix, ft->bitmap);
309}
310
311int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
312 void *match_criteria, void *flow_context,
313 u32 *flow_index)
314{
315 struct mlx5_flow_table *ft = flow_table;
316 int group_ix;
317 int err;
318
319 err = mlx5_find_group(ft, match_criteria_enable, match_criteria,
320 &group_ix);
321 if (err) {
322 mlx5_core_warn(ft->dev, "mlx5_find_group failed\n");
323 return err;
324 }
325
326 err = alloc_flow_index(ft, group_ix, flow_index);
327 if (err) {
328 mlx5_core_warn(ft->dev, "alloc_flow_index failed\n");
329 return err;
330 }
331
332 return mlx5_set_flow_entry_cmd(ft, group_ix, *flow_index, flow_context);
333}
334EXPORT_SYMBOL(mlx5_add_flow_table_entry);
335
336void mlx5_del_flow_table_entry(void *flow_table, u32 flow_index)
337{
338 struct mlx5_flow_table *ft = flow_table;
339
340 mlx5_del_flow_entry_cmd(ft, flow_index);
341 mlx5_free_flow_index(ft, flow_index);
342}
343EXPORT_SYMBOL(mlx5_del_flow_table_entry);
344
345void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
346 u16 num_groups,
347 struct mlx5_flow_table_group *group)
348{
349 struct mlx5_flow_table *ft;
350 u32 start_ix = 0;
351 u32 ft_size = 0;
352 void *gr;
353 void *bm;
354 int err;
355 int i;
356
357 for (i = 0; i < num_groups; i++)
358 ft_size += (1 << group[i].log_sz);
359
360 ft = kzalloc(sizeof(*ft), GFP_KERNEL);
361 gr = kcalloc(num_groups, sizeof(struct mlx5_ftg), GFP_KERNEL);
362 bm = kcalloc(BITS_TO_LONGS(ft_size), sizeof(uintptr_t), GFP_KERNEL);
363 if (!ft || !gr || !bm)
364 goto err_free_ft;
365
366 ft->group = gr;
367 ft->bitmap = bm;
368 ft->num_groups = num_groups;
369 ft->level = level;
370 ft->type = table_type;
371 ft->size = ft_size;
372 ft->dev = dev;
373 mutex_init(&ft->mutex);
374
375 for (i = 0; i < ft->num_groups; i++) {
376 memcpy(&ft->group[i].g, &group[i], sizeof(*group));
377 ft->group[i].start_ix = start_ix;
378 start_ix += 1 << group[i].log_sz;
379 }
380
381 err = mlx5_create_flow_table_cmd(ft);
382 if (err)
383 goto err_free_ft;
384
385 err = mlx5_create_flow_table_groups(ft);
386 if (err)
387 goto err_destroy_flow_table_cmd;
388
389 return ft;
390
391err_destroy_flow_table_cmd:
392 mlx5_destroy_flow_table_cmd(ft);
393
394err_free_ft:
395 mlx5_core_warn(dev, "failed to alloc flow table\n");
396 kfree(bm);
397 kfree(gr);
398 kfree(ft);
399
400 return NULL;
401}
402EXPORT_SYMBOL(mlx5_create_flow_table);
403
404void mlx5_destroy_flow_table(void *flow_table)
405{
406 struct mlx5_flow_table *ft = flow_table;
407
408 mlx5_destroy_flow_table_groups(ft);
409 mlx5_destroy_flow_table_cmd(ft);
410 kfree(ft->bitmap);
411 kfree(ft->group);
412 kfree(ft);
413}
414EXPORT_SYMBOL(mlx5_destroy_flow_table);
415
416u32 mlx5_get_flow_table_id(void *flow_table)
417{
418 struct mlx5_flow_table *ft = flow_table;
419
420 return ft->id;
421}
422EXPORT_SYMBOL(mlx5_get_flow_table_id);
423
