1
2
3
4#include <linux/mlx5/driver.h>
5#include <linux/mlx5/mlx5_ifc.h>
6#include <linux/mlx5/fs.h>
7
8#include "lib/fs_chains.h"
9#include "en/mapping.h"
10#include "fs_core.h"
11#include "en_tc.h"
12
13#define chains_lock(chains) ((chains)->lock)
14#define chains_ht(chains) ((chains)->chains_ht)
15#define prios_ht(chains) ((chains)->prios_ht)
16#define ft_pool_left(chains) ((chains)->ft_left)
17#define tc_default_ft(chains) ((chains)->tc_default_ft)
18#define tc_end_ft(chains) ((chains)->tc_end_ft)
19#define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
20 FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
21
22
23
24
25
26
27
28
29
30#define FT_SIZE (16 * 1024 * 1024)
31static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
32 1 * 1024 * 1024,
33 64 * 1024,
34 128 };
35#define FT_TBL_SZ (64 * 1024)
36
37struct mlx5_fs_chains {
38 struct mlx5_core_dev *dev;
39
40 struct rhashtable chains_ht;
41 struct rhashtable prios_ht;
42
43 struct mutex lock;
44
45 struct mlx5_flow_table *tc_default_ft;
46 struct mlx5_flow_table *tc_end_ft;
47 struct mapping_ctx *chains_mapping;
48
49 enum mlx5_flow_namespace_type ns;
50 u32 group_num;
51 u32 flags;
52
53 int ft_left[ARRAY_SIZE(FT_POOLS)];
54};
55
56struct fs_chain {
57 struct rhash_head node;
58
59 u32 chain;
60
61 int ref;
62 int id;
63
64 struct mlx5_fs_chains *chains;
65 struct list_head prios_list;
66 struct mlx5_flow_handle *restore_rule;
67 struct mlx5_modify_hdr *miss_modify_hdr;
68};
69
70struct prio_key {
71 u32 chain;
72 u32 prio;
73 u32 level;
74};
75
76struct prio {
77 struct rhash_head node;
78 struct list_head list;
79
80 struct prio_key key;
81
82 int ref;
83
84 struct fs_chain *chain;
85 struct mlx5_flow_table *ft;
86 struct mlx5_flow_table *next_ft;
87 struct mlx5_flow_group *miss_group;
88 struct mlx5_flow_handle *miss_rule;
89};
90
91static const struct rhashtable_params chain_params = {
92 .head_offset = offsetof(struct fs_chain, node),
93 .key_offset = offsetof(struct fs_chain, chain),
94 .key_len = sizeof_field(struct fs_chain, chain),
95 .automatic_shrinking = true,
96};
97
98static const struct rhashtable_params prio_params = {
99 .head_offset = offsetof(struct prio, node),
100 .key_offset = offsetof(struct prio, key),
101 .key_len = sizeof_field(struct prio, key),
102 .automatic_shrinking = true,
103};
104
105bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
106{
107 return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
108}
109
110bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
111{
112 return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
113}
114
115bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
116{
117 return mlx5_chains_prios_supported(chains) &&
118 mlx5_chains_ignore_flow_level_supported(chains);
119}
120
121u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
122{
123 if (!mlx5_chains_prios_supported(chains))
124 return 1;
125
126 if (mlx5_chains_ignore_flow_level_supported(chains))
127 return UINT_MAX - 1;
128
129
130 return FDB_TC_MAX_CHAIN;
131}
132
133u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
134{
135 return mlx5_chains_get_chain_range(chains) + 1;
136}
137
138u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
139{
140 if (mlx5_chains_ignore_flow_level_supported(chains))
141 return UINT_MAX;
142
143
144 return FDB_TC_MAX_PRIO;
145}
146
147static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
148{
149 if (mlx5_chains_ignore_flow_level_supported(chains))
150 return UINT_MAX;
151
152
153 return FDB_TC_LEVELS_PER_PRIO;
154}
155
156void
157mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
158 struct mlx5_flow_table *ft)
159{
160 tc_end_ft(chains) = ft;
161}
162
163#define POOL_NEXT_SIZE 0
164static int
165mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains *chains,
166 int desired_size)
167{
168 int i, found_i = -1;
169
170 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
171 if (ft_pool_left(chains)[i] && FT_POOLS[i] > desired_size) {
172 found_i = i;
173 if (desired_size != POOL_NEXT_SIZE)
174 break;
175 }
176 }
177
178 if (found_i != -1) {
179 --ft_pool_left(chains)[found_i];
180 return FT_POOLS[found_i];
181 }
182
183 return 0;
184}
185
186static void
187mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains *chains, int sz)
188{
189 int i;
190
191 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
192 if (sz == FT_POOLS[i]) {
193 ++ft_pool_left(chains)[i];
194 return;
195 }
196 }
197
198 WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
199}
200
201static void
202mlx5_chains_init_sz_pool(struct mlx5_fs_chains *chains, u32 ft_max)
203{
204 int i;
205
206 for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
207 ft_pool_left(chains)[i] =
208 FT_POOLS[i] <= ft_max ? FT_SIZE / FT_POOLS[i] : 0;
209}
210
211static struct mlx5_flow_table *
212mlx5_chains_create_table(struct mlx5_fs_chains *chains,
213 u32 chain, u32 prio, u32 level)
214{
215 struct mlx5_flow_table_attr ft_attr = {};
216 struct mlx5_flow_namespace *ns;
217 struct mlx5_flow_table *ft;
218 int sz;
219
220 if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
221 ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
222 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
223
224 sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
225 mlx5_chains_get_avail_sz_from_pool(chains, FT_TBL_SZ) :
226 mlx5_chains_get_avail_sz_from_pool(chains, POOL_NEXT_SIZE);
227 if (!sz)
228 return ERR_PTR(-ENOSPC);
229 ft_attr.max_fte = sz;
230
231
232
233
234
235 ft_attr.next_ft = tc_default_ft(chains);
236
237
238
239
240
241
242 if (!mlx5_chains_ignore_flow_level_supported(chains) ||
243 (chain == 0 && prio == 1 && level == 0)) {
244 ft_attr.level = level;
245 ft_attr.prio = prio - 1;
246 ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
247 mlx5_get_fdb_sub_ns(chains->dev, chain) :
248 mlx5_get_flow_namespace(chains->dev, chains->ns);
249 } else {
250 ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
251 ft_attr.prio = ns_to_chains_fs_prio(chains->ns);
252
253
254
255
256
257
258
259 ft_attr.level = 1;
260 ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
261 }
262
263 ft_attr.autogroup.num_reserved_entries = 2;
264 ft_attr.autogroup.max_num_groups = chains->group_num;
265 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
266 if (IS_ERR(ft)) {
267 mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
268 (int)PTR_ERR(ft), chain, prio, level, sz);
269 mlx5_chains_put_sz_to_pool(chains, sz);
270 return ft;
271 }
272
273 return ft;
274}
275
276static void
277mlx5_chains_destroy_table(struct mlx5_fs_chains *chains,
278 struct mlx5_flow_table *ft)
279{
280 mlx5_chains_put_sz_to_pool(chains, ft->max_fte);
281 mlx5_destroy_flow_table(ft);
282}
283
284static int
285create_chain_restore(struct fs_chain *chain)
286{
287 struct mlx5_eswitch *esw = chain->chains->dev->priv.eswitch;
288 char modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)];
289 struct mlx5_fs_chains *chains = chain->chains;
290 enum mlx5e_tc_attr_to_reg chain_to_reg;
291 struct mlx5_modify_hdr *mod_hdr;
292 u32 index;
293 int err;
294
295 if (chain->chain == mlx5_chains_get_nf_ft_chain(chains) ||
296 !mlx5_chains_prios_supported(chains))
297 return 0;
298
299 err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
300 if (err)
301 return err;
302 if (index == MLX5_FS_DEFAULT_FLOW_TAG) {
303
304
305
306
307
308
309 err = mlx5_chains_get_chain_mapping(chains, chain->chain, &index);
310 mapping_remove(chains->chains_mapping, MLX5_FS_DEFAULT_FLOW_TAG);
311 if (err)
312 return err;
313 }
314
315 chain->id = index;
316
317 if (chains->ns == MLX5_FLOW_NAMESPACE_FDB) {
318 chain_to_reg = CHAIN_TO_REG;
319 chain->restore_rule = esw_add_restore_rule(esw, chain->id);
320 if (IS_ERR(chain->restore_rule)) {
321 err = PTR_ERR(chain->restore_rule);
322 goto err_rule;
323 }
324 } else if (chains->ns == MLX5_FLOW_NAMESPACE_KERNEL) {
325
326
327
328
329 chain_to_reg = NIC_CHAIN_TO_REG;
330 } else {
331 err = -EINVAL;
332 goto err_rule;
333 }
334
335 MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
336 MLX5_SET(set_action_in, modact, field,
337 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield);
338 MLX5_SET(set_action_in, modact, offset,
339 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset * 8);
340 MLX5_SET(set_action_in, modact, length,
341 mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen * 8);
342 MLX5_SET(set_action_in, modact, data, chain->id);
343 mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns,
344 1, modact);
345 if (IS_ERR(mod_hdr)) {
346 err = PTR_ERR(mod_hdr);
347 goto err_mod_hdr;
348 }
349 chain->miss_modify_hdr = mod_hdr;
350
351 return 0;
352
353err_mod_hdr:
354 if (!IS_ERR_OR_NULL(chain->restore_rule))
355 mlx5_del_flow_rules(chain->restore_rule);
356err_rule:
357
358 mapping_remove(chains->chains_mapping, chain->id);
359 return err;
360}
361
362static void destroy_chain_restore(struct fs_chain *chain)
363{
364 struct mlx5_fs_chains *chains = chain->chains;
365
366 if (!chain->miss_modify_hdr)
367 return;
368
369 if (chain->restore_rule)
370 mlx5_del_flow_rules(chain->restore_rule);
371
372 mlx5_modify_header_dealloc(chains->dev, chain->miss_modify_hdr);
373 mapping_remove(chains->chains_mapping, chain->id);
374}
375
376static struct fs_chain *
377mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
378{
379 struct fs_chain *chain_s = NULL;
380 int err;
381
382 chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
383 if (!chain_s)
384 return ERR_PTR(-ENOMEM);
385
386 chain_s->chains = chains;
387 chain_s->chain = chain;
388 INIT_LIST_HEAD(&chain_s->prios_list);
389
390 err = create_chain_restore(chain_s);
391 if (err)
392 goto err_restore;
393
394 err = rhashtable_insert_fast(&chains_ht(chains), &chain_s->node,
395 chain_params);
396 if (err)
397 goto err_insert;
398
399 return chain_s;
400
401err_insert:
402 destroy_chain_restore(chain_s);
403err_restore:
404 kvfree(chain_s);
405 return ERR_PTR(err);
406}
407
408static void
409mlx5_chains_destroy_chain(struct fs_chain *chain)
410{
411 struct mlx5_fs_chains *chains = chain->chains;
412
413 rhashtable_remove_fast(&chains_ht(chains), &chain->node,
414 chain_params);
415
416 destroy_chain_restore(chain);
417 kvfree(chain);
418}
419
420static struct fs_chain *
421mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
422{
423 struct fs_chain *chain_s;
424
425 chain_s = rhashtable_lookup_fast(&chains_ht(chains), &chain,
426 chain_params);
427 if (!chain_s) {
428 chain_s = mlx5_chains_create_chain(chains, chain);
429 if (IS_ERR(chain_s))
430 return chain_s;
431 }
432
433 chain_s->ref++;
434
435 return chain_s;
436}
437
438static struct mlx5_flow_handle *
439mlx5_chains_add_miss_rule(struct fs_chain *chain,
440 struct mlx5_flow_table *ft,
441 struct mlx5_flow_table *next_ft)
442{
443 struct mlx5_fs_chains *chains = chain->chains;
444 struct mlx5_flow_destination dest = {};
445 struct mlx5_flow_act act = {};
446
447 act.flags = FLOW_ACT_NO_APPEND;
448 if (mlx5_chains_ignore_flow_level_supported(chain->chains))
449 act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
450
451 act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
452 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
453 dest.ft = next_ft;
454
455 if (next_ft == tc_end_ft(chains) &&
456 chain->chain != mlx5_chains_get_nf_ft_chain(chains) &&
457 mlx5_chains_prios_supported(chains)) {
458 act.modify_hdr = chain->miss_modify_hdr;
459 act.action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
460 }
461
462 return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
463}
464
465static int
466mlx5_chains_update_prio_prevs(struct prio *prio,
467 struct mlx5_flow_table *next_ft)
468{
469 struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
470 struct fs_chain *chain = prio->chain;
471 struct prio *pos;
472 int n = 0, err;
473
474 if (prio->key.level)
475 return 0;
476
477
478
479
480
481 pos = prio;
482 list_for_each_entry_continue_reverse(pos,
483 &chain->prios_list,
484 list) {
485 miss_rules[n] = mlx5_chains_add_miss_rule(chain,
486 pos->ft,
487 next_ft);
488 if (IS_ERR(miss_rules[n])) {
489 err = PTR_ERR(miss_rules[n]);
490 goto err_prev_rule;
491 }
492
493 n++;
494 if (!pos->key.level)
495 break;
496 }
497
498
499 n = 0;
500 pos = prio;
501 list_for_each_entry_continue_reverse(pos,
502 &chain->prios_list,
503 list) {
504 mlx5_del_flow_rules(pos->miss_rule);
505
506 pos->miss_rule = miss_rules[n];
507 pos->next_ft = next_ft;
508
509 n++;
510 if (!pos->key.level)
511 break;
512 }
513
514 return 0;
515
516err_prev_rule:
517 while (--n >= 0)
518 mlx5_del_flow_rules(miss_rules[n]);
519
520 return err;
521}
522
523static void
524mlx5_chains_put_chain(struct fs_chain *chain)
525{
526 if (--chain->ref == 0)
527 mlx5_chains_destroy_chain(chain);
528}
529
530static struct prio *
531mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
532 u32 chain, u32 prio, u32 level)
533{
534 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
535 struct mlx5_flow_handle *miss_rule;
536 struct mlx5_flow_group *miss_group;
537 struct mlx5_flow_table *next_ft;
538 struct mlx5_flow_table *ft;
539 struct fs_chain *chain_s;
540 struct list_head *pos;
541 struct prio *prio_s;
542 u32 *flow_group_in;
543 int err;
544
545 chain_s = mlx5_chains_get_chain(chains, chain);
546 if (IS_ERR(chain_s))
547 return ERR_CAST(chain_s);
548
549 prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
550 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
551 if (!prio_s || !flow_group_in) {
552 err = -ENOMEM;
553 goto err_alloc;
554 }
555
556
557
558
559
560
561
562
563
564
565
566
567
568 next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
569 tc_default_ft(chains) :
570 tc_end_ft(chains);
571 list_for_each(pos, &chain_s->prios_list) {
572 struct prio *p = list_entry(pos, struct prio, list);
573
574
575 if (prio < p->key.prio || (prio == p->key.prio &&
576 level < p->key.level)) {
577
578 next_ft = p->key.level == 0 ? p->ft : p->next_ft;
579 break;
580 }
581 }
582
583 ft = mlx5_chains_create_table(chains, chain, prio, level);
584 if (IS_ERR(ft)) {
585 err = PTR_ERR(ft);
586 goto err_create;
587 }
588
589 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
590 ft->max_fte - 2);
591 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
592 ft->max_fte - 1);
593 miss_group = mlx5_create_flow_group(ft, flow_group_in);
594 if (IS_ERR(miss_group)) {
595 err = PTR_ERR(miss_group);
596 goto err_group;
597 }
598
599
600 miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
601 if (IS_ERR(miss_rule)) {
602 err = PTR_ERR(miss_rule);
603 goto err_miss_rule;
604 }
605
606 prio_s->miss_group = miss_group;
607 prio_s->miss_rule = miss_rule;
608 prio_s->next_ft = next_ft;
609 prio_s->chain = chain_s;
610 prio_s->key.chain = chain;
611 prio_s->key.prio = prio;
612 prio_s->key.level = level;
613 prio_s->ft = ft;
614
615 err = rhashtable_insert_fast(&prios_ht(chains), &prio_s->node,
616 prio_params);
617 if (err)
618 goto err_insert;
619
620 list_add(&prio_s->list, pos->prev);
621
622
623 err = mlx5_chains_update_prio_prevs(prio_s, ft);
624 if (err)
625 goto err_update;
626
627 kvfree(flow_group_in);
628 return prio_s;
629
630err_update:
631 list_del(&prio_s->list);
632 rhashtable_remove_fast(&prios_ht(chains), &prio_s->node,
633 prio_params);
634err_insert:
635 mlx5_del_flow_rules(miss_rule);
636err_miss_rule:
637 mlx5_destroy_flow_group(miss_group);
638err_group:
639 mlx5_chains_destroy_table(chains, ft);
640err_create:
641err_alloc:
642 kvfree(prio_s);
643 kvfree(flow_group_in);
644 mlx5_chains_put_chain(chain_s);
645 return ERR_PTR(err);
646}
647
648static void
649mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
650 struct prio *prio)
651{
652 struct fs_chain *chain = prio->chain;
653
654 WARN_ON(mlx5_chains_update_prio_prevs(prio,
655 prio->next_ft));
656
657 list_del(&prio->list);
658 rhashtable_remove_fast(&prios_ht(chains), &prio->node,
659 prio_params);
660 mlx5_del_flow_rules(prio->miss_rule);
661 mlx5_destroy_flow_group(prio->miss_group);
662 mlx5_chains_destroy_table(chains, prio->ft);
663 mlx5_chains_put_chain(chain);
664 kvfree(prio);
665}
666
667struct mlx5_flow_table *
668mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
669 u32 level)
670{
671 struct mlx5_flow_table *prev_fts;
672 struct prio *prio_s;
673 struct prio_key key;
674 int l = 0;
675
676 if ((chain > mlx5_chains_get_chain_range(chains) &&
677 chain != mlx5_chains_get_nf_ft_chain(chains)) ||
678 prio > mlx5_chains_get_prio_range(chains) ||
679 level > mlx5_chains_get_level_range(chains))
680 return ERR_PTR(-EOPNOTSUPP);
681
682
683
684
685 for (l = 0; l < level; l++) {
686 prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
687 if (IS_ERR(prev_fts)) {
688 prio_s = ERR_CAST(prev_fts);
689 goto err_get_prevs;
690 }
691 }
692
693 key.chain = chain;
694 key.prio = prio;
695 key.level = level;
696
697 mutex_lock(&chains_lock(chains));
698 prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
699 prio_params);
700 if (!prio_s) {
701 prio_s = mlx5_chains_create_prio(chains, chain,
702 prio, level);
703 if (IS_ERR(prio_s))
704 goto err_create_prio;
705 }
706
707 ++prio_s->ref;
708 mutex_unlock(&chains_lock(chains));
709
710 return prio_s->ft;
711
712err_create_prio:
713 mutex_unlock(&chains_lock(chains));
714err_get_prevs:
715 while (--l >= 0)
716 mlx5_chains_put_table(chains, chain, prio, l);
717 return ERR_CAST(prio_s);
718}
719
720void
721mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
722 u32 level)
723{
724 struct prio *prio_s;
725 struct prio_key key;
726
727 key.chain = chain;
728 key.prio = prio;
729 key.level = level;
730
731 mutex_lock(&chains_lock(chains));
732 prio_s = rhashtable_lookup_fast(&prios_ht(chains), &key,
733 prio_params);
734 if (!prio_s)
735 goto err_get_prio;
736
737 if (--prio_s->ref == 0)
738 mlx5_chains_destroy_prio(chains, prio_s);
739 mutex_unlock(&chains_lock(chains));
740
741 while (level-- > 0)
742 mlx5_chains_put_table(chains, chain, prio, level);
743
744 return;
745
746err_get_prio:
747 mutex_unlock(&chains_lock(chains));
748 WARN_ONCE(1,
749 "Couldn't find table: (chain: %d prio: %d level: %d)",
750 chain, prio, level);
751}
752
753struct mlx5_flow_table *
754mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
755{
756 return tc_end_ft(chains);
757}
758
759struct mlx5_flow_table *
760mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
761{
762 u32 chain, prio, level;
763 int err;
764
765 if (!mlx5_chains_ignore_flow_level_supported(chains)) {
766 err = -EOPNOTSUPP;
767
768 mlx5_core_warn(chains->dev,
769 "Couldn't create global flow table, ignore_flow_level not supported.");
770 goto err_ignore;
771 }
772
773 chain = mlx5_chains_get_chain_range(chains),
774 prio = mlx5_chains_get_prio_range(chains);
775 level = mlx5_chains_get_level_range(chains);
776
777 return mlx5_chains_create_table(chains, chain, prio, level);
778
779err_ignore:
780 return ERR_PTR(err);
781}
782
783void
784mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
785 struct mlx5_flow_table *ft)
786{
787 mlx5_chains_destroy_table(chains, ft);
788}
789
790static struct mlx5_fs_chains *
791mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
792{
793 struct mlx5_fs_chains *chains_priv;
794 u32 max_flow_counter;
795 int err;
796
797 chains_priv = kzalloc(sizeof(*chains_priv), GFP_KERNEL);
798 if (!chains_priv)
799 return ERR_PTR(-ENOMEM);
800
801 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
802 MLX5_CAP_GEN(dev, max_flow_counter_15_0);
803
804 mlx5_core_dbg(dev,
805 "Init flow table chains, max counters(%d), groups(%d), max flow table size(%d)\n",
806 max_flow_counter, attr->max_grp_num, attr->max_ft_sz);
807
808 chains_priv->dev = dev;
809 chains_priv->flags = attr->flags;
810 chains_priv->ns = attr->ns;
811 chains_priv->group_num = attr->max_grp_num;
812 chains_priv->chains_mapping = attr->mapping;
813 tc_default_ft(chains_priv) = tc_end_ft(chains_priv) = attr->default_ft;
814
815 mlx5_core_info(dev, "Supported tc offload range - chains: %u, prios: %u\n",
816 mlx5_chains_get_chain_range(chains_priv),
817 mlx5_chains_get_prio_range(chains_priv));
818
819 mlx5_chains_init_sz_pool(chains_priv, attr->max_ft_sz);
820
821 err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
822 if (err)
823 goto init_chains_ht_err;
824
825 err = rhashtable_init(&prios_ht(chains_priv), &prio_params);
826 if (err)
827 goto init_prios_ht_err;
828
829 mutex_init(&chains_lock(chains_priv));
830
831 return chains_priv;
832
833init_prios_ht_err:
834 rhashtable_destroy(&chains_ht(chains_priv));
835init_chains_ht_err:
836 kfree(chains_priv);
837 return ERR_PTR(err);
838}
839
840static void
841mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
842{
843 mutex_destroy(&chains_lock(chains));
844 rhashtable_destroy(&prios_ht(chains));
845 rhashtable_destroy(&chains_ht(chains));
846
847 kfree(chains);
848}
849
850struct mlx5_fs_chains *
851mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
852{
853 struct mlx5_fs_chains *chains;
854
855 chains = mlx5_chains_init(dev, attr);
856
857 return chains;
858}
859
860void
861mlx5_chains_destroy(struct mlx5_fs_chains *chains)
862{
863 mlx5_chains_cleanup(chains);
864}
865
866int
867mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
868 u32 *chain_mapping)
869{
870 struct mapping_ctx *ctx = chains->chains_mapping;
871 struct mlx5_mapped_obj mapped_obj = {};
872
873 mapped_obj.type = MLX5_MAPPED_OBJ_CHAIN;
874 mapped_obj.chain = chain;
875 return mapping_add(ctx, &mapped_obj, chain_mapping);
876}
877
878int
879mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains, u32 chain_mapping)
880{
881 struct mapping_ctx *ctx = chains->chains_mapping;
882
883 return mapping_remove(ctx, chain_mapping);
884}
885