1
2
3
4#include <linux/netdevice.h>
5#include "en/fs_tt_redirect.h"
6#include "fs_core.h"
7
8enum fs_udp_type {
9 FS_IPV4_UDP,
10 FS_IPV6_UDP,
11 FS_UDP_NUM_TYPES,
12};
13
14struct mlx5e_fs_udp {
15 struct mlx5e_flow_table tables[FS_UDP_NUM_TYPES];
16 struct mlx5_flow_handle *default_rules[FS_UDP_NUM_TYPES];
17 int ref_cnt;
18};
19
20struct mlx5e_fs_any {
21 struct mlx5e_flow_table table;
22 struct mlx5_flow_handle *default_rule;
23 int ref_cnt;
24};
25
26static char *fs_udp_type2str(enum fs_udp_type i)
27{
28 switch (i) {
29 case FS_IPV4_UDP:
30 return "UDP v4";
31 default:
32 return "UDP v6";
33 }
34}
35
36static enum mlx5_traffic_types fs_udp2tt(enum fs_udp_type i)
37{
38 switch (i) {
39 case FS_IPV4_UDP:
40 return MLX5_TT_IPV4_UDP;
41 default:
42 return MLX5_TT_IPV6_UDP;
43 }
44}
45
46static enum fs_udp_type tt2fs_udp(enum mlx5_traffic_types i)
47{
48 switch (i) {
49 case MLX5_TT_IPV4_UDP:
50 return FS_IPV4_UDP;
51 case MLX5_TT_IPV6_UDP:
52 return FS_IPV6_UDP;
53 default:
54 return FS_UDP_NUM_TYPES;
55 }
56}
57
58void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule)
59{
60 mlx5_del_flow_rules(rule);
61}
62
63static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type type,
64 u16 udp_dport)
65{
66 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
67 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
68 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
69 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
70 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
71 type == FS_IPV4_UDP ? 4 : 6);
72 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
73 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, udp_dport);
74}
75
76struct mlx5_flow_handle *
77mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
78 enum mlx5_traffic_types ttc_type,
79 u32 tir_num, u16 d_port)
80{
81 enum fs_udp_type type = tt2fs_udp(ttc_type);
82 struct mlx5_flow_destination dest = {};
83 struct mlx5_flow_table *ft = NULL;
84 MLX5_DECLARE_FLOW_ACT(flow_act);
85 struct mlx5_flow_handle *rule;
86 struct mlx5_flow_spec *spec;
87 struct mlx5e_fs_udp *fs_udp;
88 int err;
89
90 if (type == FS_UDP_NUM_TYPES)
91 return ERR_PTR(-EINVAL);
92
93 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
94 if (!spec)
95 return ERR_PTR(-ENOMEM);
96
97 fs_udp = priv->fs.udp;
98 ft = fs_udp->tables[type].t;
99
100 fs_udp_set_dport_flow(spec, type, d_port);
101 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
102 dest.tir_num = tir_num;
103
104 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
105 kvfree(spec);
106
107 if (IS_ERR(rule)) {
108 err = PTR_ERR(rule);
109 netdev_err(priv->netdev, "%s: add %s rule failed, err %d\n",
110 __func__, fs_udp_type2str(type), err);
111 }
112 return rule;
113}
114
115static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type type)
116{
117 struct mlx5e_flow_table *fs_udp_t;
118 struct mlx5_flow_destination dest;
119 MLX5_DECLARE_FLOW_ACT(flow_act);
120 struct mlx5_flow_handle *rule;
121 struct mlx5e_fs_udp *fs_udp;
122 int err;
123
124 fs_udp = priv->fs.udp;
125 fs_udp_t = &fs_udp->tables[type];
126
127 dest = mlx5_ttc_get_default_dest(priv->fs.ttc, fs_udp2tt(type));
128 rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1);
129 if (IS_ERR(rule)) {
130 err = PTR_ERR(rule);
131 netdev_err(priv->netdev,
132 "%s: add default rule failed, fs type=%d, err %d\n",
133 __func__, type, err);
134 return err;
135 }
136
137 fs_udp->default_rules[type] = rule;
138 return 0;
139}
140
141#define MLX5E_FS_UDP_NUM_GROUPS (2)
142#define MLX5E_FS_UDP_GROUP1_SIZE (BIT(16))
143#define MLX5E_FS_UDP_GROUP2_SIZE (BIT(0))
144#define MLX5E_FS_UDP_TABLE_SIZE (MLX5E_FS_UDP_GROUP1_SIZE +\
145 MLX5E_FS_UDP_GROUP2_SIZE)
146static int fs_udp_create_groups(struct mlx5e_flow_table *ft, enum fs_udp_type type)
147{
148 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
149 void *outer_headers_c;
150 int ix = 0;
151 u32 *in;
152 int err;
153 u8 *mc;
154
155 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
156 in = kvzalloc(inlen, GFP_KERNEL);
157 if (!in || !ft->g) {
158 kfree(ft->g);
159 kvfree(in);
160 return -ENOMEM;
161 }
162
163 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
164 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
165 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
166 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
167
168 switch (type) {
169 case FS_IPV4_UDP:
170 case FS_IPV6_UDP:
171 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
172 break;
173 default:
174 err = -EINVAL;
175 goto out;
176 }
177
178 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
179 MLX5_SET_CFG(in, start_flow_index, ix);
180 ix += MLX5E_FS_UDP_GROUP1_SIZE;
181 MLX5_SET_CFG(in, end_flow_index, ix - 1);
182 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
183 if (IS_ERR(ft->g[ft->num_groups]))
184 goto err;
185 ft->num_groups++;
186
187
188 memset(in, 0, inlen);
189 MLX5_SET_CFG(in, start_flow_index, ix);
190 ix += MLX5E_FS_UDP_GROUP2_SIZE;
191 MLX5_SET_CFG(in, end_flow_index, ix - 1);
192 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
193 if (IS_ERR(ft->g[ft->num_groups]))
194 goto err;
195 ft->num_groups++;
196
197 kvfree(in);
198 return 0;
199
200err:
201 err = PTR_ERR(ft->g[ft->num_groups]);
202 ft->g[ft->num_groups] = NULL;
203out:
204 kvfree(in);
205
206 return err;
207}
208
209static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type)
210{
211 struct mlx5e_flow_table *ft = &priv->fs.udp->tables[type];
212 struct mlx5_flow_table_attr ft_attr = {};
213 int err;
214
215 ft->num_groups = 0;
216
217 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
218 ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL;
219 ft_attr.prio = MLX5E_NIC_PRIO;
220
221 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
222 if (IS_ERR(ft->t)) {
223 err = PTR_ERR(ft->t);
224 ft->t = NULL;
225 return err;
226 }
227
228 netdev_dbg(priv->netdev, "Created fs %s table id %u level %u\n",
229 fs_udp_type2str(type), ft->t->id, ft->t->level);
230
231 err = fs_udp_create_groups(ft, type);
232 if (err)
233 goto err;
234
235 err = fs_udp_add_default_rule(priv, type);
236 if (err)
237 goto err;
238
239 return 0;
240
241err:
242 mlx5e_destroy_flow_table(ft);
243 return err;
244}
245
246static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i)
247{
248 if (IS_ERR_OR_NULL(fs_udp->tables[i].t))
249 return;
250
251 mlx5_del_flow_rules(fs_udp->default_rules[i]);
252 mlx5e_destroy_flow_table(&fs_udp->tables[i]);
253 fs_udp->tables[i].t = NULL;
254}
255
256static int fs_udp_disable(struct mlx5e_priv *priv)
257{
258 int err, i;
259
260 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
261
262 err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_udp2tt(i));
263 if (err) {
264 netdev_err(priv->netdev,
265 "%s: modify ttc[%d] default destination failed, err(%d)\n",
266 __func__, fs_udp2tt(i), err);
267 return err;
268 }
269 }
270
271 return 0;
272}
273
274static int fs_udp_enable(struct mlx5e_priv *priv)
275{
276 struct mlx5_flow_destination dest = {};
277 int err, i;
278
279 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
280 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
281 dest.ft = priv->fs.udp->tables[i].t;
282
283
284 err = mlx5_ttc_fwd_dest(priv->fs.ttc, fs_udp2tt(i), &dest);
285 if (err) {
286 netdev_err(priv->netdev,
287 "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
288 __func__, fs_udp2tt(i), err);
289 return err;
290 }
291 }
292 return 0;
293}
294
295void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv)
296{
297 struct mlx5e_fs_udp *fs_udp = priv->fs.udp;
298 int i;
299
300 if (!fs_udp)
301 return;
302
303 if (--fs_udp->ref_cnt)
304 return;
305
306 fs_udp_disable(priv);
307
308 for (i = 0; i < FS_UDP_NUM_TYPES; i++)
309 fs_udp_destroy_table(fs_udp, i);
310
311 kfree(fs_udp);
312 priv->fs.udp = NULL;
313}
314
315int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv)
316{
317 int i, err;
318
319 if (priv->fs.udp) {
320 priv->fs.udp->ref_cnt++;
321 return 0;
322 }
323
324 priv->fs.udp = kzalloc(sizeof(*priv->fs.udp), GFP_KERNEL);
325 if (!priv->fs.udp)
326 return -ENOMEM;
327
328 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
329 err = fs_udp_create_table(priv, i);
330 if (err)
331 goto err_destroy_tables;
332 }
333
334 err = fs_udp_enable(priv);
335 if (err)
336 goto err_destroy_tables;
337
338 priv->fs.udp->ref_cnt = 1;
339
340 return 0;
341
342err_destroy_tables:
343 while (--i >= 0)
344 fs_udp_destroy_table(priv->fs.udp, i);
345
346 kfree(priv->fs.udp);
347 priv->fs.udp = NULL;
348 return err;
349}
350
351static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_type)
352{
353 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
354 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
355 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ether_type);
356}
357
358struct mlx5_flow_handle *
359mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv,
360 u32 tir_num, u16 ether_type)
361{
362 struct mlx5_flow_destination dest = {};
363 struct mlx5_flow_table *ft = NULL;
364 MLX5_DECLARE_FLOW_ACT(flow_act);
365 struct mlx5_flow_handle *rule;
366 struct mlx5_flow_spec *spec;
367 struct mlx5e_fs_any *fs_any;
368 int err;
369
370 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
371 if (!spec)
372 return ERR_PTR(-ENOMEM);
373
374 fs_any = priv->fs.any;
375 ft = fs_any->table.t;
376
377 fs_any_set_ethertype_flow(spec, ether_type);
378 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
379 dest.tir_num = tir_num;
380
381 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
382 kvfree(spec);
383
384 if (IS_ERR(rule)) {
385 err = PTR_ERR(rule);
386 netdev_err(priv->netdev, "%s: add ANY rule failed, err %d\n",
387 __func__, err);
388 }
389 return rule;
390}
391
392static int fs_any_add_default_rule(struct mlx5e_priv *priv)
393{
394 struct mlx5e_flow_table *fs_any_t;
395 struct mlx5_flow_destination dest;
396 MLX5_DECLARE_FLOW_ACT(flow_act);
397 struct mlx5_flow_handle *rule;
398 struct mlx5e_fs_any *fs_any;
399 int err;
400
401 fs_any = priv->fs.any;
402 fs_any_t = &fs_any->table;
403
404 dest = mlx5_ttc_get_default_dest(priv->fs.ttc, MLX5_TT_ANY);
405 rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1);
406 if (IS_ERR(rule)) {
407 err = PTR_ERR(rule);
408 netdev_err(priv->netdev,
409 "%s: add default rule failed, fs type=ANY, err %d\n",
410 __func__, err);
411 return err;
412 }
413
414 fs_any->default_rule = rule;
415 return 0;
416}
417
418#define MLX5E_FS_ANY_NUM_GROUPS (2)
419#define MLX5E_FS_ANY_GROUP1_SIZE (BIT(16))
420#define MLX5E_FS_ANY_GROUP2_SIZE (BIT(0))
421#define MLX5E_FS_ANY_TABLE_SIZE (MLX5E_FS_ANY_GROUP1_SIZE +\
422 MLX5E_FS_ANY_GROUP2_SIZE)
423
424static int fs_any_create_groups(struct mlx5e_flow_table *ft)
425{
426 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
427 void *outer_headers_c;
428 int ix = 0;
429 u32 *in;
430 int err;
431 u8 *mc;
432
433 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
434 in = kvzalloc(inlen, GFP_KERNEL);
435 if (!in || !ft->g) {
436 kfree(ft->g);
437 kvfree(in);
438 return -ENOMEM;
439 }
440
441
442 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
443 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
444 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
445 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
446 MLX5_SET_CFG(in, start_flow_index, ix);
447 ix += MLX5E_FS_ANY_GROUP1_SIZE;
448 MLX5_SET_CFG(in, end_flow_index, ix - 1);
449 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
450 if (IS_ERR(ft->g[ft->num_groups]))
451 goto err;
452 ft->num_groups++;
453
454
455 memset(in, 0, inlen);
456 MLX5_SET_CFG(in, start_flow_index, ix);
457 ix += MLX5E_FS_ANY_GROUP2_SIZE;
458 MLX5_SET_CFG(in, end_flow_index, ix - 1);
459 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
460 if (IS_ERR(ft->g[ft->num_groups]))
461 goto err;
462 ft->num_groups++;
463
464 kvfree(in);
465 return 0;
466
467err:
468 err = PTR_ERR(ft->g[ft->num_groups]);
469 ft->g[ft->num_groups] = NULL;
470 kvfree(in);
471
472 return err;
473}
474
475static int fs_any_create_table(struct mlx5e_priv *priv)
476{
477 struct mlx5e_flow_table *ft = &priv->fs.any->table;
478 struct mlx5_flow_table_attr ft_attr = {};
479 int err;
480
481 ft->num_groups = 0;
482
483 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
484 ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL;
485 ft_attr.prio = MLX5E_NIC_PRIO;
486
487 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
488 if (IS_ERR(ft->t)) {
489 err = PTR_ERR(ft->t);
490 ft->t = NULL;
491 return err;
492 }
493
494 netdev_dbg(priv->netdev, "Created fs ANY table id %u level %u\n",
495 ft->t->id, ft->t->level);
496
497 err = fs_any_create_groups(ft);
498 if (err)
499 goto err;
500
501 err = fs_any_add_default_rule(priv);
502 if (err)
503 goto err;
504
505 return 0;
506
507err:
508 mlx5e_destroy_flow_table(ft);
509 return err;
510}
511
512static int fs_any_disable(struct mlx5e_priv *priv)
513{
514 int err;
515
516
517 err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, MLX5_TT_ANY);
518 if (err) {
519 netdev_err(priv->netdev,
520 "%s: modify ttc[%d] default destination failed, err(%d)\n",
521 __func__, MLX5_TT_ANY, err);
522 return err;
523 }
524 return 0;
525}
526
527static int fs_any_enable(struct mlx5e_priv *priv)
528{
529 struct mlx5_flow_destination dest = {};
530 int err;
531
532 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
533 dest.ft = priv->fs.any->table.t;
534
535
536 err = mlx5_ttc_fwd_dest(priv->fs.ttc, MLX5_TT_ANY, &dest);
537 if (err) {
538 netdev_err(priv->netdev,
539 "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
540 __func__, MLX5_TT_ANY, err);
541 return err;
542 }
543 return 0;
544}
545
546static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any)
547{
548 if (IS_ERR_OR_NULL(fs_any->table.t))
549 return;
550
551 mlx5_del_flow_rules(fs_any->default_rule);
552 mlx5e_destroy_flow_table(&fs_any->table);
553 fs_any->table.t = NULL;
554}
555
556void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv)
557{
558 struct mlx5e_fs_any *fs_any = priv->fs.any;
559
560 if (!fs_any)
561 return;
562
563 if (--fs_any->ref_cnt)
564 return;
565
566 fs_any_disable(priv);
567
568 fs_any_destroy_table(fs_any);
569
570 kfree(fs_any);
571 priv->fs.any = NULL;
572}
573
574int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv)
575{
576 int err;
577
578 if (priv->fs.any) {
579 priv->fs.any->ref_cnt++;
580 return 0;
581 }
582
583 priv->fs.any = kzalloc(sizeof(*priv->fs.any), GFP_KERNEL);
584 if (!priv->fs.any)
585 return -ENOMEM;
586
587 err = fs_any_create_table(priv);
588 if (err)
589 return err;
590
591 err = fs_any_enable(priv);
592 if (err)
593 goto err_destroy_table;
594
595 priv->fs.any->ref_cnt = 1;
596
597 return 0;
598
599err_destroy_table:
600 fs_any_destroy_table(priv->fs.any);
601
602 kfree(priv->fs.any);
603 priv->fs.any = NULL;
604 return err;
605}
606
