1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/hash.h>
34#include <linux/mlx5/fs.h>
35#include <linux/ip.h>
36#include <linux/ipv6.h>
37#include "en.h"
38
39#define ARFS_HASH_SHIFT BITS_PER_BYTE
40#define ARFS_HASH_SIZE BIT(BITS_PER_BYTE)
41
42struct arfs_table {
43 struct mlx5e_flow_table ft;
44 struct mlx5_flow_handle *default_rule;
45 struct hlist_head rules_hash[ARFS_HASH_SIZE];
46};
47
48enum arfs_type {
49 ARFS_IPV4_TCP,
50 ARFS_IPV6_TCP,
51 ARFS_IPV4_UDP,
52 ARFS_IPV6_UDP,
53 ARFS_NUM_TYPES,
54};
55
56struct mlx5e_arfs_tables {
57 struct arfs_table arfs_tables[ARFS_NUM_TYPES];
58
59 spinlock_t arfs_lock;
60 struct list_head rules;
61 int last_filter_id;
62 struct workqueue_struct *wq;
63};
64
65struct arfs_tuple {
66 __be16 etype;
67 u8 ip_proto;
68 union {
69 __be32 src_ipv4;
70 struct in6_addr src_ipv6;
71 };
72 union {
73 __be32 dst_ipv4;
74 struct in6_addr dst_ipv6;
75 };
76 __be16 src_port;
77 __be16 dst_port;
78};
79
80struct arfs_rule {
81 struct mlx5e_priv *priv;
82 struct work_struct arfs_work;
83 struct mlx5_flow_handle *rule;
84 struct hlist_node hlist;
85 int rxq;
86
87 int flow_id;
88
89 int filter_id;
90 struct arfs_tuple tuple;
91};
92
93#define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \
94 for (i = 0; i < ARFS_NUM_TYPES; i++) \
95 mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j)
96
97#define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \
98 for (j = 0; j < ARFS_HASH_SIZE; j++) \
99 hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist)
100
101static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type)
102{
103 switch (type) {
104 case ARFS_IPV4_TCP:
105 return MLX5_TT_IPV4_TCP;
106 case ARFS_IPV4_UDP:
107 return MLX5_TT_IPV4_UDP;
108 case ARFS_IPV6_TCP:
109 return MLX5_TT_IPV6_TCP;
110 case ARFS_IPV6_UDP:
111 return MLX5_TT_IPV6_UDP;
112 default:
113 return -EINVAL;
114 }
115}
116
117static int arfs_disable(struct mlx5e_priv *priv)
118{
119 int err, i;
120
121 for (i = 0; i < ARFS_NUM_TYPES; i++) {
122
123 err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, arfs_get_tt(i));
124 if (err) {
125 netdev_err(priv->netdev,
126 "%s: modify ttc[%d] default destination failed, err(%d)\n",
127 __func__, arfs_get_tt(i), err);
128 return err;
129 }
130 }
131 return 0;
132}
133
134static void arfs_del_rules(struct mlx5e_priv *priv);
135
136int mlx5e_arfs_disable(struct mlx5e_priv *priv)
137{
138 arfs_del_rules(priv);
139
140 return arfs_disable(priv);
141}
142
143int mlx5e_arfs_enable(struct mlx5e_priv *priv)
144{
145 struct mlx5_flow_destination dest = {};
146 int err, i;
147
148 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
149 for (i = 0; i < ARFS_NUM_TYPES; i++) {
150 dest.ft = priv->fs.arfs->arfs_tables[i].ft.t;
151
152 err = mlx5_ttc_fwd_dest(priv->fs.ttc, arfs_get_tt(i), &dest);
153 if (err) {
154 netdev_err(priv->netdev,
155 "%s: modify ttc[%d] dest to arfs, failed err(%d)\n",
156 __func__, arfs_get_tt(i), err);
157 arfs_disable(priv);
158 return err;
159 }
160 }
161 return 0;
162}
163
164static void arfs_destroy_table(struct arfs_table *arfs_t)
165{
166 mlx5_del_flow_rules(arfs_t->default_rule);
167 mlx5e_destroy_flow_table(&arfs_t->ft);
168}
169
170static void _mlx5e_cleanup_tables(struct mlx5e_priv *priv)
171{
172 int i;
173
174 arfs_del_rules(priv);
175 destroy_workqueue(priv->fs.arfs->wq);
176 for (i = 0; i < ARFS_NUM_TYPES; i++) {
177 if (!IS_ERR_OR_NULL(priv->fs.arfs->arfs_tables[i].ft.t))
178 arfs_destroy_table(&priv->fs.arfs->arfs_tables[i]);
179 }
180}
181
182void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv)
183{
184 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE))
185 return;
186
187 _mlx5e_cleanup_tables(priv);
188 kvfree(priv->fs.arfs);
189}
190
191static int arfs_add_default_rule(struct mlx5e_priv *priv,
192 enum arfs_type type)
193{
194 struct arfs_table *arfs_t = &priv->fs.arfs->arfs_tables[type];
195 struct mlx5_flow_destination dest = {};
196 MLX5_DECLARE_FLOW_ACT(flow_act);
197 enum mlx5_traffic_types tt;
198 int err = 0;
199
200 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
201 tt = arfs_get_tt(type);
202 if (tt == -EINVAL) {
203 netdev_err(priv->netdev, "%s: bad arfs_type: %d\n",
204 __func__, type);
205 return -EINVAL;
206 }
207
208
209
210
211 dest.tir_num = mlx5e_rx_res_get_tirn_rss(priv->rx_res, tt);
212 arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL,
213 &flow_act,
214 &dest, 1);
215 if (IS_ERR(arfs_t->default_rule)) {
216 err = PTR_ERR(arfs_t->default_rule);
217 arfs_t->default_rule = NULL;
218 netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n",
219 __func__, type);
220 }
221
222 return err;
223}
224
225#define MLX5E_ARFS_NUM_GROUPS 2
226#define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1)
227#define MLX5E_ARFS_GROUP2_SIZE BIT(0)
228#define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\
229 MLX5E_ARFS_GROUP2_SIZE)
230static int arfs_create_groups(struct mlx5e_flow_table *ft,
231 enum arfs_type type)
232{
233 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
234 void *outer_headers_c;
235 int ix = 0;
236 u32 *in;
237 int err;
238 u8 *mc;
239
240 ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS,
241 sizeof(*ft->g), GFP_KERNEL);
242 in = kvzalloc(inlen, GFP_KERNEL);
243 if (!in || !ft->g) {
244 kfree(ft->g);
245 kvfree(in);
246 return -ENOMEM;
247 }
248
249 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
250 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc,
251 outer_headers);
252 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
253 switch (type) {
254 case ARFS_IPV4_TCP:
255 case ARFS_IPV6_TCP:
256 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
257 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
258 break;
259 case ARFS_IPV4_UDP:
260 case ARFS_IPV6_UDP:
261 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
262 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport);
263 break;
264 default:
265 err = -EINVAL;
266 goto out;
267 }
268
269 switch (type) {
270 case ARFS_IPV4_TCP:
271 case ARFS_IPV4_UDP:
272 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
273 src_ipv4_src_ipv6.ipv4_layout.ipv4);
274 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
275 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
276 break;
277 case ARFS_IPV6_TCP:
278 case ARFS_IPV6_UDP:
279 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
280 src_ipv4_src_ipv6.ipv6_layout.ipv6),
281 0xff, 16);
282 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
283 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
284 0xff, 16);
285 break;
286 default:
287 err = -EINVAL;
288 goto out;
289 }
290
291 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
292 MLX5_SET_CFG(in, start_flow_index, ix);
293 ix += MLX5E_ARFS_GROUP1_SIZE;
294 MLX5_SET_CFG(in, end_flow_index, ix - 1);
295 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
296 if (IS_ERR(ft->g[ft->num_groups]))
297 goto err;
298 ft->num_groups++;
299
300 memset(in, 0, inlen);
301 MLX5_SET_CFG(in, start_flow_index, ix);
302 ix += MLX5E_ARFS_GROUP2_SIZE;
303 MLX5_SET_CFG(in, end_flow_index, ix - 1);
304 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
305 if (IS_ERR(ft->g[ft->num_groups]))
306 goto err;
307 ft->num_groups++;
308
309 kvfree(in);
310 return 0;
311
312err:
313 err = PTR_ERR(ft->g[ft->num_groups]);
314 ft->g[ft->num_groups] = NULL;
315out:
316 kvfree(in);
317
318 return err;
319}
320
321static int arfs_create_table(struct mlx5e_priv *priv,
322 enum arfs_type type)
323{
324 struct mlx5e_arfs_tables *arfs = priv->fs.arfs;
325 struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft;
326 struct mlx5_flow_table_attr ft_attr = {};
327 int err;
328
329 ft->num_groups = 0;
330
331 ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE;
332 ft_attr.level = MLX5E_ARFS_FT_LEVEL;
333 ft_attr.prio = MLX5E_NIC_PRIO;
334
335 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
336 if (IS_ERR(ft->t)) {
337 err = PTR_ERR(ft->t);
338 ft->t = NULL;
339 return err;
340 }
341
342 err = arfs_create_groups(ft, type);
343 if (err)
344 goto err;
345
346 err = arfs_add_default_rule(priv, type);
347 if (err)
348 goto err;
349
350 return 0;
351err:
352 mlx5e_destroy_flow_table(ft);
353 return err;
354}
355
356int mlx5e_arfs_create_tables(struct mlx5e_priv *priv)
357{
358 int err = -ENOMEM;
359 int i;
360
361 if (!(priv->netdev->hw_features & NETIF_F_NTUPLE))
362 return 0;
363
364 priv->fs.arfs = kvzalloc(sizeof(*priv->fs.arfs), GFP_KERNEL);
365 if (!priv->fs.arfs)
366 return -ENOMEM;
367
368 spin_lock_init(&priv->fs.arfs->arfs_lock);
369 INIT_LIST_HEAD(&priv->fs.arfs->rules);
370 priv->fs.arfs->wq = create_singlethread_workqueue("mlx5e_arfs");
371 if (!priv->fs.arfs->wq)
372 goto err;
373
374 for (i = 0; i < ARFS_NUM_TYPES; i++) {
375 err = arfs_create_table(priv, i);
376 if (err)
377 goto err_des;
378 }
379 return 0;
380
381err_des:
382 _mlx5e_cleanup_tables(priv);
383err:
384 kvfree(priv->fs.arfs);
385 return err;
386}
387
388#define MLX5E_ARFS_EXPIRY_QUOTA 60
389
390static void arfs_may_expire_flow(struct mlx5e_priv *priv)
391{
392 struct arfs_rule *arfs_rule;
393 struct hlist_node *htmp;
394 HLIST_HEAD(del_list);
395 int quota = 0;
396 int i;
397 int j;
398
399 spin_lock_bh(&priv->fs.arfs->arfs_lock);
400 mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs->arfs_tables, i, j) {
401 if (!work_pending(&arfs_rule->arfs_work) &&
402 rps_may_expire_flow(priv->netdev,
403 arfs_rule->rxq, arfs_rule->flow_id,
404 arfs_rule->filter_id)) {
405 hlist_del_init(&arfs_rule->hlist);
406 hlist_add_head(&arfs_rule->hlist, &del_list);
407 if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA)
408 break;
409 }
410 }
411 spin_unlock_bh(&priv->fs.arfs->arfs_lock);
412 hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) {
413 if (arfs_rule->rule)
414 mlx5_del_flow_rules(arfs_rule->rule);
415 hlist_del(&arfs_rule->hlist);
416 kfree(arfs_rule);
417 }
418}
419
420static void arfs_del_rules(struct mlx5e_priv *priv)
421{
422 struct hlist_node *htmp;
423 struct arfs_rule *rule;
424 HLIST_HEAD(del_list);
425 int i;
426 int j;
427
428 spin_lock_bh(&priv->fs.arfs->arfs_lock);
429 mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs->arfs_tables, i, j) {
430 hlist_del_init(&rule->hlist);
431 hlist_add_head(&rule->hlist, &del_list);
432 }
433 spin_unlock_bh(&priv->fs.arfs->arfs_lock);
434
435 hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) {
436 cancel_work_sync(&rule->arfs_work);
437 if (rule->rule)
438 mlx5_del_flow_rules(rule->rule);
439 hlist_del(&rule->hlist);
440 kfree(rule);
441 }
442}
443
444static struct hlist_head *
445arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port,
446 __be16 dst_port)
447{
448 unsigned long l;
449 int bucket_idx;
450
451 l = (__force unsigned long)src_port |
452 ((__force unsigned long)dst_port << 2);
453
454 bucket_idx = hash_long(l, ARFS_HASH_SHIFT);
455
456 return &arfs_t->rules_hash[bucket_idx];
457}
458
459static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs,
460 u8 ip_proto, __be16 etype)
461{
462 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP)
463 return &arfs->arfs_tables[ARFS_IPV4_TCP];
464 if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP)
465 return &arfs->arfs_tables[ARFS_IPV4_UDP];
466 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP)
467 return &arfs->arfs_tables[ARFS_IPV6_TCP];
468 if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP)
469 return &arfs->arfs_tables[ARFS_IPV6_UDP];
470
471 return NULL;
472}
473
474static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv,
475 struct arfs_rule *arfs_rule)
476{
477 struct mlx5e_arfs_tables *arfs = priv->fs.arfs;
478 struct arfs_tuple *tuple = &arfs_rule->tuple;
479 struct mlx5_flow_handle *rule = NULL;
480 struct mlx5_flow_destination dest = {};
481 MLX5_DECLARE_FLOW_ACT(flow_act);
482 struct arfs_table *arfs_table;
483 struct mlx5_flow_spec *spec;
484 struct mlx5_flow_table *ft;
485 int err = 0;
486
487 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
488 if (!spec) {
489 err = -ENOMEM;
490 goto out;
491 }
492 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
493 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
494 outer_headers.ethertype);
495 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype,
496 ntohs(tuple->etype));
497 arfs_table = arfs_get_table(arfs, tuple->ip_proto, tuple->etype);
498 if (!arfs_table) {
499 err = -EINVAL;
500 goto out;
501 }
502
503 ft = arfs_table->ft.t;
504 if (tuple->ip_proto == IPPROTO_TCP) {
505 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
506 outer_headers.tcp_dport);
507 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
508 outer_headers.tcp_sport);
509 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
510 ntohs(tuple->dst_port));
511 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
512 ntohs(tuple->src_port));
513 } else {
514 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
515 outer_headers.udp_dport);
516 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
517 outer_headers.udp_sport);
518 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport,
519 ntohs(tuple->dst_port));
520 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport,
521 ntohs(tuple->src_port));
522 }
523 if (tuple->etype == htons(ETH_P_IP)) {
524 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
525 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
526 &tuple->src_ipv4,
527 4);
528 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
529 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
530 &tuple->dst_ipv4,
531 4);
532 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
533 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
534 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
535 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
536 } else {
537 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
538 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
539 &tuple->src_ipv6,
540 16);
541 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
542 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
543 &tuple->dst_ipv6,
544 16);
545 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
546 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
547 0xff,
548 16);
549 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
550 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
551 0xff,
552 16);
553 }
554 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
555 dest.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, arfs_rule->rxq);
556 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
557 if (IS_ERR(rule)) {
558 err = PTR_ERR(rule);
559 priv->channel_stats[arfs_rule->rxq].rq.arfs_err++;
560 mlx5e_dbg(HW, priv,
561 "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n",
562 __func__, arfs_rule->filter_id, arfs_rule->rxq,
563 tuple->ip_proto, err);
564 }
565
566out:
567 kvfree(spec);
568 return err ? ERR_PTR(err) : rule;
569}
570
571static void arfs_modify_rule_rq(struct mlx5e_priv *priv,
572 struct mlx5_flow_handle *rule, u16 rxq)
573{
574 struct mlx5_flow_destination dst = {};
575 int err = 0;
576
577 dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
578 dst.tir_num = mlx5e_rx_res_get_tirn_direct(priv->rx_res, rxq);
579 err = mlx5_modify_rule_destination(rule, &dst, NULL);
580 if (err)
581 netdev_warn(priv->netdev,
582 "Failed to modify aRFS rule destination to rq=%d\n", rxq);
583}
584
585static void arfs_handle_work(struct work_struct *work)
586{
587 struct arfs_rule *arfs_rule = container_of(work,
588 struct arfs_rule,
589 arfs_work);
590 struct mlx5e_priv *priv = arfs_rule->priv;
591 struct mlx5_flow_handle *rule;
592
593 mutex_lock(&priv->state_lock);
594 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
595 spin_lock_bh(&priv->fs.arfs->arfs_lock);
596 hlist_del(&arfs_rule->hlist);
597 spin_unlock_bh(&priv->fs.arfs->arfs_lock);
598
599 mutex_unlock(&priv->state_lock);
600 kfree(arfs_rule);
601 goto out;
602 }
603 mutex_unlock(&priv->state_lock);
604
605 if (!arfs_rule->rule) {
606 rule = arfs_add_rule(priv, arfs_rule);
607 if (IS_ERR(rule))
608 goto out;
609 arfs_rule->rule = rule;
610 } else {
611 arfs_modify_rule_rq(priv, arfs_rule->rule,
612 arfs_rule->rxq);
613 }
614out:
615 arfs_may_expire_flow(priv);
616}
617
618static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv,
619 struct arfs_table *arfs_t,
620 const struct flow_keys *fk,
621 u16 rxq, u32 flow_id)
622{
623 struct arfs_rule *rule;
624 struct arfs_tuple *tuple;
625
626 rule = kzalloc(sizeof(*rule), GFP_ATOMIC);
627 if (!rule)
628 return NULL;
629
630 rule->priv = priv;
631 rule->rxq = rxq;
632 INIT_WORK(&rule->arfs_work, arfs_handle_work);
633
634 tuple = &rule->tuple;
635 tuple->etype = fk->basic.n_proto;
636 tuple->ip_proto = fk->basic.ip_proto;
637 if (tuple->etype == htons(ETH_P_IP)) {
638 tuple->src_ipv4 = fk->addrs.v4addrs.src;
639 tuple->dst_ipv4 = fk->addrs.v4addrs.dst;
640 } else {
641 memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src,
642 sizeof(struct in6_addr));
643 memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst,
644 sizeof(struct in6_addr));
645 }
646 tuple->src_port = fk->ports.src;
647 tuple->dst_port = fk->ports.dst;
648
649 rule->flow_id = flow_id;
650 rule->filter_id = priv->fs.arfs->last_filter_id++ % RPS_NO_FILTER;
651
652 hlist_add_head(&rule->hlist,
653 arfs_hash_bucket(arfs_t, tuple->src_port,
654 tuple->dst_port));
655 return rule;
656}
657
658static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk)
659{
660 if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst)
661 return false;
662 if (tuple->etype != fk->basic.n_proto)
663 return false;
664 if (tuple->etype == htons(ETH_P_IP))
665 return tuple->src_ipv4 == fk->addrs.v4addrs.src &&
666 tuple->dst_ipv4 == fk->addrs.v4addrs.dst;
667 if (tuple->etype == htons(ETH_P_IPV6))
668 return !memcmp(&tuple->src_ipv6, &fk->addrs.v6addrs.src,
669 sizeof(struct in6_addr)) &&
670 !memcmp(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst,
671 sizeof(struct in6_addr));
672 return false;
673}
674
675static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t,
676 const struct flow_keys *fk)
677{
678 struct arfs_rule *arfs_rule;
679 struct hlist_head *head;
680
681 head = arfs_hash_bucket(arfs_t, fk->ports.src, fk->ports.dst);
682 hlist_for_each_entry(arfs_rule, head, hlist) {
683 if (arfs_cmp(&arfs_rule->tuple, fk))
684 return arfs_rule;
685 }
686
687 return NULL;
688}
689
690int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
691 u16 rxq_index, u32 flow_id)
692{
693 struct mlx5e_priv *priv = netdev_priv(dev);
694 struct mlx5e_arfs_tables *arfs = priv->fs.arfs;
695 struct arfs_table *arfs_t;
696 struct arfs_rule *arfs_rule;
697 struct flow_keys fk;
698
699 if (!skb_flow_dissect_flow_keys(skb, &fk, 0))
700 return -EPROTONOSUPPORT;
701
702 if (fk.basic.n_proto != htons(ETH_P_IP) &&
703 fk.basic.n_proto != htons(ETH_P_IPV6))
704 return -EPROTONOSUPPORT;
705
706 if (skb->encapsulation)
707 return -EPROTONOSUPPORT;
708
709 arfs_t = arfs_get_table(arfs, fk.basic.ip_proto, fk.basic.n_proto);
710 if (!arfs_t)
711 return -EPROTONOSUPPORT;
712
713 spin_lock_bh(&arfs->arfs_lock);
714 arfs_rule = arfs_find_rule(arfs_t, &fk);
715 if (arfs_rule) {
716 if (arfs_rule->rxq == rxq_index) {
717 spin_unlock_bh(&arfs->arfs_lock);
718 return arfs_rule->filter_id;
719 }
720 arfs_rule->rxq = rxq_index;
721 } else {
722 arfs_rule = arfs_alloc_rule(priv, arfs_t, &fk, rxq_index, flow_id);
723 if (!arfs_rule) {
724 spin_unlock_bh(&arfs->arfs_lock);
725 return -ENOMEM;
726 }
727 }
728 queue_work(priv->fs.arfs->wq, &arfs_rule->arfs_work);
729 spin_unlock_bh(&arfs->arfs_lock);
730 return arfs_rule->filter_id;
731}
732
733