1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/hash.h>
34#include <linux/mlx5/fs.h>
35#include <linux/ip.h>
36#include <linux/ipv6.h>
37#include "en.h"
38
39struct arfs_tuple {
40 __be16 etype;
41 u8 ip_proto;
42 union {
43 __be32 src_ipv4;
44 struct in6_addr src_ipv6;
45 };
46 union {
47 __be32 dst_ipv4;
48 struct in6_addr dst_ipv6;
49 };
50 __be16 src_port;
51 __be16 dst_port;
52};
53
54struct arfs_rule {
55 struct mlx5e_priv *priv;
56 struct work_struct arfs_work;
57 struct mlx5_flow_handle *rule;
58 struct hlist_node hlist;
59 int rxq;
60
61 int flow_id;
62
63 int filter_id;
64 struct arfs_tuple tuple;
65};
66
67#define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \
68 for (i = 0; i < ARFS_NUM_TYPES; i++) \
69 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j)
70
71#define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \
72 for (j = 0; j < ARFS_HASH_SIZE; j++) \
73 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist)
74
75static enum mlx5e_traffic_types arfs_get_tt(enum arfs_type type)
76{
77 switch (type) {
78 case ARFS_IPV4_TCP:
79 return MLX5E_TT_IPV4_TCP;
80 case ARFS_IPV4_UDP:
81 return MLX5E_TT_IPV4_UDP;
82 case ARFS_IPV6_TCP:
83 return MLX5E_TT_IPV6_TCP;
84 case ARFS_IPV6_UDP:
85 return MLX5E_TT_IPV6_UDP;
86 default:
87 return -EINVAL;
88 }
89}
90
91static int arfs_disable(struct mlx5e_priv *priv)
92{
93 int err, i;
94
95 for (i = 0; i < ARFS_NUM_TYPES; i++) {
96
97 err = mlx5e_ttc_fwd_default_dest(priv, arfs_get_tt(i));
98 if (err) {
99 netdev_err(priv->netdev,
100 "%s: modify ttc[%d] default destination failed, err(%d)\n",
101 __func__, arfs_get_tt(i), err);
102 return err;
103 }
104 }
105 return 0;
106}
107
108static void arfs_del_rules(struct mlx5e_priv *priv);
109
110int mlx5e_arfs_disable(struct mlx5e_priv *priv)
111{
112 arfs_del_rules(priv);
113
114 return arfs_disable(priv);
115}
116
117int mlx5e_arfs_enable(struct mlx5e_priv *priv)
118{
119 struct mlx5_flow_destination dest = {};
120 int err, i;
121
122 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
123 for (i = 0; i < ARFS_NUM_TYPES; i++) {
124 dest.ft = priv->fs.arfs.arfs_tables[i].ft.t;
125
126 err = mlx5e_ttc_fwd_dest(priv, arfs_get_tt(i), &dest);
127 if (err) {
128 netdev_err(priv->netdev,
129 "%s: modify ttc[%d] dest to arfs, failed err(%d)\n",
130 __func__, arfs_get_tt(i), err);
131 arfs_disable(priv);
132 return err;
133 }
134 }
135 return 0;
136}
137
138static void arfs_destroy_table(struct arfs_table *arfs_t)
139{
140 mlx5_del_flow_rules(arfs_t->default_rule);
141 mlx5e_destroy_flow_table(&arfs_t->ft);
142}
143
144void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv)
145{
146 int i;
147
148 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE))
149 return;
150
151 arfs_del_rules(priv);
152 destroy_workqueue(priv->fs.arfs.wq);
153 for (i = 0; i < ARFS_NUM_TYPES; i++) {
154 if (!IS_ERR_OR_NULL(priv->fs.arfs.arfs_tables[i].ft.t))
155 arfs_destroy_table(&priv->fs.arfs.arfs_tables[i]);
156 }
157}
158
159static int arfs_add_default_rule(struct mlx5e_priv *priv,
160 enum arfs_type type)
161{
162 struct arfs_table *arfs_t = &priv->fs.arfs.arfs_tables[type];
163 struct mlx5e_tir *tir = priv->indir_tir;
164 struct mlx5_flow_destination dest = {};
165 MLX5_DECLARE_FLOW_ACT(flow_act);
166 enum mlx5e_traffic_types tt;
167 int err = 0;
168
169 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
170 tt = arfs_get_tt(type);
171 if (tt == -EINVAL) {
172 netdev_err(priv->netdev, "%s: bad arfs_type: %d\n",
173 __func__, type);
174 return -EINVAL;
175 }
176
177
178
179
180 dest.tir_num = tir[tt].tirn;
181 arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL,
182 &flow_act,
183 &dest, 1);
184 if (IS_ERR(arfs_t->default_rule)) {
185 err = PTR_ERR(arfs_t->default_rule);
186 arfs_t->default_rule = NULL;
187 netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n",
188 __func__, type);
189 }
190
191 return err;
192}
193
194#define MLX5E_ARFS_NUM_GROUPS 2
195#define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1)
196#define MLX5E_ARFS_GROUP2_SIZE BIT(0)
197#define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\
198 MLX5E_ARFS_GROUP2_SIZE)
199static int arfs_create_groups(struct mlx5e_flow_table *ft,
200 enum arfs_type type)
201{
202 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
203 void *outer_headers_c;
204 int ix = 0;
205 u32 *in;
206 int err;
207 u8 *mc;
208
209 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS,
210 sizeof(*ft->g), GFP_KERNEL);
211 in = kvzalloc(inlen, GFP_KERNEL);
212 if (!in || !ft->g) {
213 kfree(ft->g);
214 kvfree(in);
215 return -ENOMEM;
216 }
217
218 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
219 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc,
220 outer_headers);
221 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
222 switch (type) {
223 case ARFS_IPV4_TCP:
224 case ARFS_IPV6_TCP:
225 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
226 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
227 break;
228 case ARFS_IPV4_UDP:
229 case ARFS_IPV6_UDP:
230 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
231 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport);
232 break;
233 default:
234 err = -EINVAL;
235 goto out;
236 }
237
238 switch (type) {
239 case ARFS_IPV4_TCP:
240 case ARFS_IPV4_UDP:
241 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
242 src_ipv4_src_ipv6.ipv4_layout.ipv4);
243 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
244 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
245 break;
246 case ARFS_IPV6_TCP:
247 case ARFS_IPV6_UDP:
248 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
249 src_ipv4_src_ipv6.ipv6_layout.ipv6),
250 0xff, 16);
251 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
252 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
253 0xff, 16);
254 break;
255 default:
256 err = -EINVAL;
257 goto out;
258 }
259
260 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
261 MLX5_SET_CFG(in, start_flow_index, ix);
262 ix += MLX5E_ARFS_GROUP1_SIZE;
263 MLX5_SET_CFG(in, end_flow_index, ix - 1);
264 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
265 if (IS_ERR(ft->g[ft->num_groups]))
266 goto err;
267 ft->num_groups++;
268
269 memset(in, 0, inlen);
270 MLX5_SET_CFG(in, start_flow_index, ix);
271 ix += MLX5E_ARFS_GROUP2_SIZE;
272 MLX5_SET_CFG(in, end_flow_index, ix - 1);
273 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
274 if (IS_ERR(ft->g[ft->num_groups]))
275 goto err;
276 ft->num_groups++;
277
278 kvfree(in);
279 return 0;
280
281err:
282 err = PTR_ERR(ft->g[ft->num_groups]);
283 ft->g[ft->num_groups] = NULL;
284out:
285 kvfree(in);
286
287 return err;
288}
289
290static int arfs_create_table(struct mlx5e_priv *priv,
291 enum arfs_type type)
292{
293 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs;
294 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft;
295 struct mlx5_flow_table_attr ft_attr = {};
296 int err;
297
298 ft->num_groups = 0;
299
300 ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE;
301 ft_attr.level = MLX5E_ARFS_FT_LEVEL;
302 ft_attr.prio = MLX5E_NIC_PRIO;
303
304 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
305 if (IS_ERR(ft->t)) {
306 err = PTR_ERR(ft->t);
307 ft->t = NULL;
308 return err;
309 }
310
311 err = arfs_create_groups(ft, type);
312 if (err)
313 goto err;
314
315 err = arfs_add_default_rule(priv, type);
316 if (err)
317 goto err;
318
319 return 0;
320err:
321 mlx5e_destroy_flow_table(ft);
322 return err;
323}
324
325int mlx5e_arfs_create_tables(struct mlx5e_priv *priv)
326{
327 int err = 0;
328 int i;
329
330 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE))
331 return 0;
332
333 spin_lock_init(&priv->fs.arfs.arfs_lock);
334 INIT_LIST_HEAD(&priv->fs.arfs.rules);
335 priv->fs.arfs.wq = create_singlethread_workqueue("mlx5e_arfs");
336 if (!priv->fs.arfs.wq)
337 return -ENOMEM;
338
339 for (i = 0; i < ARFS_NUM_TYPES; i++) {
340 err = arfs_create_table(priv, i);
341 if (err)
342 goto err;
343 }
344 return 0;
345err:
346 mlx5e_arfs_destroy_tables(priv);
347 return err;
348}
349
350#define MLX5E_ARFS_EXPIRY_QUOTA 60
351
352static void arfs_may_expire_flow(struct mlx5e_priv *priv)
353{
354 struct arfs_rule *arfs_rule;
355 struct hlist_node *htmp;
356 int quota = 0;
357 int i;
358 int j;
359
360 HLIST_HEAD(del_list);
361 spin_lock_bh(&priv->fs.arfs.arfs_lock);
362 mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs.arfs_tables, i, j) {
363 if (!work_pending(&arfs_rule->arfs_work) &&
364 rps_may_expire_flow(priv->netdev,
365 arfs_rule->rxq, arfs_rule->flow_id,
366 arfs_rule->filter_id)) {
367 hlist_del_init(&arfs_rule->hlist);
368 hlist_add_head(&arfs_rule->hlist, &del_list);
369 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA)
370 break;
371 }
372 }
373 spin_unlock_bh(&priv->fs.arfs.arfs_lock);
374 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) {
375 if (arfs_rule->rule)
376 mlx5_del_flow_rules(arfs_rule->rule);
377 hlist_del(&arfs_rule->hlist);
378 kfree(arfs_rule);
379 }
380}
381
382static void arfs_del_rules(struct mlx5e_priv *priv)
383{
384 struct hlist_node *htmp;
385 struct arfs_rule *rule;
386 int i;
387 int j;
388
389 HLIST_HEAD(del_list);
390 spin_lock_bh(&priv->fs.arfs.arfs_lock);
391 mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs.arfs_tables, i, j) {
392 hlist_del_init(&rule->hlist);
393 hlist_add_head(&rule->hlist, &del_list);
394 }
395 spin_unlock_bh(&priv->fs.arfs.arfs_lock);
396
397 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) {
398 cancel_work_sync(&rule->arfs_work);
399 if (rule->rule)
400 mlx5_del_flow_rules(rule->rule);
401 hlist_del(&rule->hlist);
402 kfree(rule);
403 }
404}
405
406static struct hlist_head *
407arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port,
408 __be16 dst_port)
409{
410 unsigned long l;
411 int bucket_idx;
412
413 l = (__force unsigned long)src_port |
414 ((__force unsigned long)dst_port << 2);
415
416 bucket_idx = hash_long(l, ARFS_HASH_SHIFT);
417
418 return &arfs_t->rules_hash[bucket_idx];
419}
420
421static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs,
422 u8 ip_proto, __be16 etype)
423{
424 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP)
425 return &arfs->arfs_tables[ARFS_IPV4_TCP];
426 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP)
427 return &arfs->arfs_tables[ARFS_IPV4_UDP];
428 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP)
429 return &arfs->arfs_tables[ARFS_IPV6_TCP];
430 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP)
431 return &arfs->arfs_tables[ARFS_IPV6_UDP];
432
433 return NULL;
434}
435
436static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv,
437 struct arfs_rule *arfs_rule)
438{
439 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs;
440 struct arfs_tuple *tuple = &arfs_rule->tuple;
441 struct mlx5_flow_handle *rule = NULL;
442 struct mlx5_flow_destination dest = {};
443 MLX5_DECLARE_FLOW_ACT(flow_act);
444 struct arfs_table *arfs_table;
445 struct mlx5_flow_spec *spec;
446 struct mlx5_flow_table *ft;
447 int err = 0;
448
449 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
450 if (!spec) {
451 err = -ENOMEM;
452 goto out;
453 }
454 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
455 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
456 outer_headers.ethertype);
457 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype,
458 ntohs(tuple->etype));
459 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype);
460 if (!arfs_table) {
461 err = -EINVAL;
462 goto out;
463 }
464
465 ft = arfs_table->ft.t;
466 if (tuple->ip_proto == IPPROTO_TCP) {
467 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
468 outer_headers.tcp_dport);
469 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
470 outer_headers.tcp_sport);
471 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
472 ntohs(tuple->dst_port));
473 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
474 ntohs(tuple->src_port));
475 } else {
476 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
477 outer_headers.udp_dport);
478 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
479 outer_headers.udp_sport);
480 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport,
481 ntohs(tuple->dst_port));
482 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport,
483 ntohs(tuple->src_port));
484 }
485 if (tuple->etype == htons(ETH_P_IP)) {
486 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
487 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
488 &tuple->src_ipv4,
489 4);
490 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
491 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
492 &tuple->dst_ipv4,
493 4);
494 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
495 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
496 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
497 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
498 } else {
499 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
500 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
501 &tuple->src_ipv6,
502 16);
503 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
504 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
505 &tuple->dst_ipv6,
506 16);
507 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
508 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
509 0xff,
510 16);
511 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
512 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
513 0xff,
514 16);
515 }
516 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
517 dest.tir_num = priv->direct_tir[arfs_rule->rxq].tirn;
518 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
519 if (IS_ERR(rule)) {
520 err = PTR_ERR(rule);
521 priv->channel_stats[arfs_rule->rxq].rq.arfs_err++;
522 mlx5e_dbg(HW, priv,
523 "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n",
524 __func__, arfs_rule->filter_id, arfs_rule->rxq,
525 tuple->ip_proto, err);
526 }
527
528out:
529 kvfree(spec);
530 return err ? ERR_PTR(err) : rule;
531}
532
533static void arfs_modify_rule_rq(struct mlx5e_priv *priv,
534 struct mlx5_flow_handle *rule, u16 rxq)
535{
536 struct mlx5_flow_destination dst = {};
537 int err = 0;
538
539 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
540 dst.tir_num = priv->direct_tir[rxq].tirn;
541 err = mlx5_modify_rule_destination(rule, &dst, NULL);
542 if (err)
543 netdev_warn(priv->netdev,
544 "Failed to modify aRFS rule destination to rq=%d\n", rxq);
545}
546
547static void arfs_handle_work(struct work_struct *work)
548{
549 struct arfs_rule *arfs_rule = container_of(work,
550 struct arfs_rule,
551 arfs_work);
552 struct mlx5e_priv *priv = arfs_rule->priv;
553 struct mlx5_flow_handle *rule;
554
555 mutex_lock(&priv->state_lock);
556 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
557 spin_lock_bh(&priv->fs.arfs.arfs_lock);
558 hlist_del(&arfs_rule->hlist);
559 spin_unlock_bh(&priv->fs.arfs.arfs_lock);
560
561 mutex_unlock(&priv->state_lock);
562 kfree(arfs_rule);
563 goto out;
564 }
565 mutex_unlock(&priv->state_lock);
566
567 if (!arfs_rule->rule) {
568 rule = arfs_add_rule(priv, arfs_rule);
569 if (IS_ERR(rule))
570 goto out;
571 arfs_rule->rule = rule;
572 } else {
573 arfs_modify_rule_rq(priv, arfs_rule->rule,
574 arfs_rule->rxq);
575 }
576out:
577 arfs_may_expire_flow(priv);
578}
579
580static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv,
581 struct arfs_table *arfs_t,
582 const struct flow_keys *fk,
583 u16 rxq, u32 flow_id)
584{
585 struct arfs_rule *rule;
586 struct arfs_tuple *tuple;
587
588 rule = kzalloc(sizeof(*rule), GFP_ATOMIC);
589 if (!rule)
590 return NULL;
591
592 rule->priv = priv;
593 rule->rxq = rxq;
594 INIT_WORK(&rule->arfs_work, arfs_handle_work);
595
596 tuple = &rule->tuple;
597 tuple->etype = fk->basic.n_proto;
598 tuple->ip_proto = fk->basic.ip_proto;
599 if (tuple->etype == htons(ETH_P_IP)) {
600 tuple->src_ipv4 = fk->addrs.v4addrs.src;
601 tuple->dst_ipv4 = fk->addrs.v4addrs.dst;
602 } else {
603 memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src,
604 sizeof(struct in6_addr));
605 memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst,
606 sizeof(struct in6_addr));
607 }
608 tuple->src_port = fk->ports.src;
609 tuple->dst_port = fk->ports.dst;
610
611 rule->flow_id = flow_id;
612 rule->filter_id = priv->fs.arfs.last_filter_id++ % RPS_NO_FILTER;
613
614 hlist_add_head(&rule->hlist,
615 arfs_hash_bucket(arfs_t, tuple->src_port,
616 tuple->dst_port));
617 return rule;
618}
619
620static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk)
621{
622 if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst)
623 return false;
624 if (tuple->etype != fk->basic.n_proto)
625 return false;
626 if (tuple->etype == htons(ETH_P_IP))
627 return tuple->src_ipv4 == fk->addrs.v4addrs.src &&
628 tuple->dst_ipv4 == fk->addrs.v4addrs.dst;
629 if (tuple->etype == htons(ETH_P_IPV6))
630 return !memcmp(&tuple->src_ipv6, &fk->addrs.v6addrs.src,
631 sizeof(struct in6_addr)) &&
632 !memcmp(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst,
633 sizeof(struct in6_addr));
634 return false;
635}
636
637static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t,
638 const struct flow_keys *fk)
639{
640 struct arfs_rule *arfs_rule;
641 struct hlist_head *head;
642
643 head = arfs_hash_bucket(arfs_t, fk->ports.src, fk->ports.dst);
644 hlist_for_each_entry(arfs_rule, head, hlist) {
645 if (arfs_cmp(&arfs_rule->tuple, fk))
646 return arfs_rule;
647 }
648
649 return NULL;
650}
651
652int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
653 u16 rxq_index, u32 flow_id)
654{
655 struct mlx5e_priv *priv = netdev_priv(dev);
656 struct mlx5e_arfs_tables *arfs = &priv->fs.arfs;
657 struct arfs_table *arfs_t;
658 struct arfs_rule *arfs_rule;
659 struct flow_keys fk;
660
661 if (!skb_flow_dissect_flow_keys(skb, &fk, 0))
662 return -EPROTONOSUPPORT;
663
664 if (fk.basic.n_proto != htons(ETH_P_IP) &&
665 fk.basic.n_proto != htons(ETH_P_IPV6))
666 return -EPROTONOSUPPORT;
667
668 if (skb->encapsulation)
669 return -EPROTONOSUPPORT;
670
671 arfs_t = arfs_get_table(arfs, fk.basic.ip_proto, fk.basic.n_proto);
672 if (!arfs_t)
673 return -EPROTONOSUPPORT;
674
675 spin_lock_bh(&arfs->arfs_lock);
676 arfs_rule = arfs_find_rule(arfs_t, &fk);
677 if (arfs_rule) {
678 if (arfs_rule->rxq == rxq_index) {
679 spin_unlock_bh(&arfs->arfs_lock);
680 return arfs_rule->filter_id;
681 }
682 arfs_rule->rxq = rxq_index;
683 } else {
684 arfs_rule = arfs_alloc_rule(priv, arfs_t, &fk, rxq_index, flow_id);
685 if (!arfs_rule) {
686 spin_unlock_bh(&arfs->arfs_lock);
687 return -ENOMEM;
688 }
689 }
690 queue_work(priv->fs.arfs.wq, &arfs_rule->arfs_work);
691 spin_unlock_bh(&arfs->arfs_lock);
692 return arfs_rule->filter_id;
693}
694
695
