1
2#ifndef __NET_PKT_CLS_H
3#define __NET_PKT_CLS_H
4
5#include <linux/pkt_cls.h>
6#include <linux/workqueue.h>
7#include <net/sch_generic.h>
8#include <net/act_api.h>
9#include <net/net_namespace.h>
10
11
12#define TC_ACT_CONSUMED (TC_ACT_VALUE_MAX + 1)
13
14
15
16struct tcf_walker {
17 int stop;
18 int skip;
19 int count;
20 bool nonempty;
21 unsigned long cookie;
22 int (*fn)(struct tcf_proto *, void *node, struct tcf_walker *);
23};
24
25int register_tcf_proto_ops(struct tcf_proto_ops *ops);
26int unregister_tcf_proto_ops(struct tcf_proto_ops *ops);
27
28struct tcf_block_ext_info {
29 enum flow_block_binder_type binder_type;
30 tcf_chain_head_change_t *chain_head_change;
31 void *chain_head_change_priv;
32 u32 block_index;
33};
34
35struct tcf_block_cb;
36bool tcf_queue_work(struct rcu_work *rwork, work_func_t func);
37
38#ifdef CONFIG_NET_CLS
39struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block,
40 u32 chain_index);
41void tcf_chain_put_by_act(struct tcf_chain *chain);
42struct tcf_chain *tcf_get_next_chain(struct tcf_block *block,
43 struct tcf_chain *chain);
44struct tcf_proto *tcf_get_next_proto(struct tcf_chain *chain,
45 struct tcf_proto *tp, bool rtnl_held);
46void tcf_block_netif_keep_dst(struct tcf_block *block);
47int tcf_block_get(struct tcf_block **p_block,
48 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
49 struct netlink_ext_ack *extack);
50int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
51 struct tcf_block_ext_info *ei,
52 struct netlink_ext_ack *extack);
53void tcf_block_put(struct tcf_block *block);
54void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
55 struct tcf_block_ext_info *ei);
56
57static inline bool tcf_block_shared(struct tcf_block *block)
58{
59 return block->index;
60}
61
62static inline bool tcf_block_non_null_shared(struct tcf_block *block)
63{
64 return block && block->index;
65}
66
67static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
68{
69 WARN_ON(tcf_block_shared(block));
70 return block->q;
71}
72
73int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
74 tc_indr_block_bind_cb_t *cb, void *cb_ident);
75int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
76 tc_indr_block_bind_cb_t *cb, void *cb_ident);
77void __tc_indr_block_cb_unregister(struct net_device *dev,
78 tc_indr_block_bind_cb_t *cb, void *cb_ident);
79void tc_indr_block_cb_unregister(struct net_device *dev,
80 tc_indr_block_bind_cb_t *cb, void *cb_ident);
81
82int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
83 struct tcf_result *res, bool compat_mode);
84
85#else
86static inline bool tcf_block_shared(struct tcf_block *block)
87{
88 return false;
89}
90
91static inline bool tcf_block_non_null_shared(struct tcf_block *block)
92{
93 return false;
94}
95
96static inline
97int tcf_block_get(struct tcf_block **p_block,
98 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
99 struct netlink_ext_ack *extack)
100{
101 return 0;
102}
103
104static inline
105int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
106 struct tcf_block_ext_info *ei,
107 struct netlink_ext_ack *extack)
108{
109 return 0;
110}
111
112static inline void tcf_block_put(struct tcf_block *block)
113{
114}
115
116static inline
117void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
118 struct tcf_block_ext_info *ei)
119{
120}
121
122static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
123{
124 return NULL;
125}
126
127static inline
128int tc_setup_cb_block_register(struct tcf_block *block, flow_setup_cb_t *cb,
129 void *cb_priv)
130{
131 return 0;
132}
133
134static inline
135void tc_setup_cb_block_unregister(struct tcf_block *block, flow_setup_cb_t *cb,
136 void *cb_priv)
137{
138}
139
140static inline
141int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
142 tc_indr_block_bind_cb_t *cb, void *cb_ident)
143{
144 return 0;
145}
146
147static inline
148int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
149 tc_indr_block_bind_cb_t *cb, void *cb_ident)
150{
151 return 0;
152}
153
154static inline
155void __tc_indr_block_cb_unregister(struct net_device *dev,
156 tc_indr_block_bind_cb_t *cb, void *cb_ident)
157{
158}
159
160static inline
161void tc_indr_block_cb_unregister(struct net_device *dev,
162 tc_indr_block_bind_cb_t *cb, void *cb_ident)
163{
164}
165
166static inline int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
167 struct tcf_result *res, bool compat_mode)
168{
169 return TC_ACT_UNSPEC;
170}
171#endif
172
173static inline unsigned long
174__cls_set_class(unsigned long *clp, unsigned long cl)
175{
176 return xchg(clp, cl);
177}
178
179static inline unsigned long
180cls_set_class(struct Qdisc *q, unsigned long *clp, unsigned long cl)
181{
182 unsigned long old_cl;
183
184 sch_tree_lock(q);
185 old_cl = __cls_set_class(clp, cl);
186 sch_tree_unlock(q);
187 return old_cl;
188}
189
190static inline void
191tcf_bind_filter(struct tcf_proto *tp, struct tcf_result *r, unsigned long base)
192{
193 struct Qdisc *q = tp->chain->block->q;
194 unsigned long cl;
195
196
197
198
199 if (!q)
200 return;
201 cl = q->ops->cl_ops->bind_tcf(q, base, r->classid);
202 cl = cls_set_class(q, &r->class, cl);
203 if (cl)
204 q->ops->cl_ops->unbind_tcf(q, cl);
205}
206
207static inline void
208tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r)
209{
210 struct Qdisc *q = tp->chain->block->q;
211 unsigned long cl;
212
213 if (!q)
214 return;
215 if ((cl = __cls_set_class(&r->class, 0)) != 0)
216 q->ops->cl_ops->unbind_tcf(q, cl);
217}
218
219struct tcf_exts {
220#ifdef CONFIG_NET_CLS_ACT
221 __u32 type;
222 int nr_actions;
223 struct tc_action **actions;
224 struct net *net;
225#endif
226
227
228
229 int action;
230 int police;
231};
232
233static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net,
234 int action, int police)
235{
236#ifdef CONFIG_NET_CLS_ACT
237 exts->type = 0;
238 exts->nr_actions = 0;
239 exts->net = net;
240 exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *),
241 GFP_KERNEL);
242 if (!exts->actions)
243 return -ENOMEM;
244#endif
245 exts->action = action;
246 exts->police = police;
247 return 0;
248}
249
250
251
252
253
254static inline bool tcf_exts_get_net(struct tcf_exts *exts)
255{
256#ifdef CONFIG_NET_CLS_ACT
257 exts->net = maybe_get_net(exts->net);
258 return exts->net != NULL;
259#else
260 return true;
261#endif
262}
263
264static inline void tcf_exts_put_net(struct tcf_exts *exts)
265{
266#ifdef CONFIG_NET_CLS_ACT
267 if (exts->net)
268 put_net(exts->net);
269#endif
270}
271
272#ifdef CONFIG_NET_CLS_ACT
273#define tcf_exts_for_each_action(i, a, exts) \
274 for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = (exts)->actions[i]); i++)
275#else
276#define tcf_exts_for_each_action(i, a, exts) \
277 for (; 0; (void)(i), (void)(a), (void)(exts))
278#endif
279
280static inline void
281tcf_exts_stats_update(const struct tcf_exts *exts,
282 u64 bytes, u64 packets, u64 lastuse)
283{
284#ifdef CONFIG_NET_CLS_ACT
285 int i;
286
287 preempt_disable();
288
289 for (i = 0; i < exts->nr_actions; i++) {
290 struct tc_action *a = exts->actions[i];
291
292 tcf_action_stats_update(a, bytes, packets, lastuse, true);
293 }
294
295 preempt_enable();
296#endif
297}
298
299
300
301
302
303
304
305static inline bool tcf_exts_has_actions(struct tcf_exts *exts)
306{
307#ifdef CONFIG_NET_CLS_ACT
308 return exts->nr_actions;
309#else
310 return false;
311#endif
312}
313
314
315
316
317
318
319
320
321
322
323
324
325static inline int
326tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts,
327 struct tcf_result *res)
328{
329#ifdef CONFIG_NET_CLS_ACT
330 return tcf_action_exec(skb, exts->actions, exts->nr_actions, res);
331#endif
332 return TC_ACT_OK;
333}
334
335int tcf_exts_validate(struct net *net, struct tcf_proto *tp,
336 struct nlattr **tb, struct nlattr *rate_tlv,
337 struct tcf_exts *exts, bool ovr, bool rtnl_held,
338 struct netlink_ext_ack *extack);
339void tcf_exts_destroy(struct tcf_exts *exts);
340void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src);
341int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts);
342int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts);
343
344
345
346
347struct tcf_pkt_info {
348 unsigned char * ptr;
349 int nexthdr;
350};
351
352#ifdef CONFIG_NET_EMATCH
353
354struct tcf_ematch_ops;
355
356
357
358
359
360
361
362
363
364
365struct tcf_ematch {
366 struct tcf_ematch_ops * ops;
367 unsigned long data;
368 unsigned int datalen;
369 u16 matchid;
370 u16 flags;
371 struct net *net;
372};
373
374static inline int tcf_em_is_container(struct tcf_ematch *em)
375{
376 return !em->ops;
377}
378
379static inline int tcf_em_is_simple(struct tcf_ematch *em)
380{
381 return em->flags & TCF_EM_SIMPLE;
382}
383
384static inline int tcf_em_is_inverted(struct tcf_ematch *em)
385{
386 return em->flags & TCF_EM_INVERT;
387}
388
389static inline int tcf_em_last_match(struct tcf_ematch *em)
390{
391 return (em->flags & TCF_EM_REL_MASK) == TCF_EM_REL_END;
392}
393
394static inline int tcf_em_early_end(struct tcf_ematch *em, int result)
395{
396 if (tcf_em_last_match(em))
397 return 1;
398
399 if (result == 0 && em->flags & TCF_EM_REL_AND)
400 return 1;
401
402 if (result != 0 && em->flags & TCF_EM_REL_OR)
403 return 1;
404
405 return 0;
406}
407
408
409
410
411
412
413
414struct tcf_ematch_tree {
415 struct tcf_ematch_tree_hdr hdr;
416 struct tcf_ematch * matches;
417
418};
419
420
421
422
423
424
425
426
427
428
429
430
431
432struct tcf_ematch_ops {
433 int kind;
434 int datalen;
435 int (*change)(struct net *net, void *,
436 int, struct tcf_ematch *);
437 int (*match)(struct sk_buff *, struct tcf_ematch *,
438 struct tcf_pkt_info *);
439 void (*destroy)(struct tcf_ematch *);
440 int (*dump)(struct sk_buff *, struct tcf_ematch *);
441 struct module *owner;
442 struct list_head link;
443};
444
445int tcf_em_register(struct tcf_ematch_ops *);
446void tcf_em_unregister(struct tcf_ematch_ops *);
447int tcf_em_tree_validate(struct tcf_proto *, struct nlattr *,
448 struct tcf_ematch_tree *);
449void tcf_em_tree_destroy(struct tcf_ematch_tree *);
450int tcf_em_tree_dump(struct sk_buff *, struct tcf_ematch_tree *, int);
451int __tcf_em_tree_match(struct sk_buff *, struct tcf_ematch_tree *,
452 struct tcf_pkt_info *);
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468static inline int tcf_em_tree_match(struct sk_buff *skb,
469 struct tcf_ematch_tree *tree,
470 struct tcf_pkt_info *info)
471{
472 if (tree->hdr.nmatches)
473 return __tcf_em_tree_match(skb, tree, info);
474 else
475 return 1;
476}
477
478#define MODULE_ALIAS_TCF_EMATCH(kind) MODULE_ALIAS("ematch-kind-" __stringify(kind))
479
480#else
481
482struct tcf_ematch_tree {
483};
484
485#define tcf_em_tree_validate(tp, tb, t) ((void)(t), 0)
486#define tcf_em_tree_destroy(t) do { (void)(t); } while(0)
487#define tcf_em_tree_dump(skb, t, tlv) (0)
488#define tcf_em_tree_match(skb, t, info) ((void)(info), 1)
489
490#endif
491
492static inline unsigned char * tcf_get_base_ptr(struct sk_buff *skb, int layer)
493{
494 switch (layer) {
495 case TCF_LAYER_LINK:
496 return skb_mac_header(skb);
497 case TCF_LAYER_NETWORK:
498 return skb_network_header(skb);
499 case TCF_LAYER_TRANSPORT:
500 return skb_transport_header(skb);
501 }
502
503 return NULL;
504}
505
506static inline int tcf_valid_offset(const struct sk_buff *skb,
507 const unsigned char *ptr, const int len)
508{
509 return likely((ptr + len) <= skb_tail_pointer(skb) &&
510 ptr >= skb->head &&
511 (ptr <= (ptr + len)));
512}
513
514static inline int
515tcf_change_indev(struct net *net, struct nlattr *indev_tlv,
516 struct netlink_ext_ack *extack)
517{
518 char indev[IFNAMSIZ];
519 struct net_device *dev;
520
521 if (nla_strlcpy(indev, indev_tlv, IFNAMSIZ) >= IFNAMSIZ) {
522 NL_SET_ERR_MSG(extack, "Interface name too long");
523 return -EINVAL;
524 }
525 dev = __dev_get_by_name(net, indev);
526 if (!dev)
527 return -ENODEV;
528 return dev->ifindex;
529}
530
531static inline bool
532tcf_match_indev(struct sk_buff *skb, int ifindex)
533{
534 if (!ifindex)
535 return true;
536 if (!skb->skb_iif)
537 return false;
538 return ifindex == skb->skb_iif;
539}
540
541int tc_setup_flow_action(struct flow_action *flow_action,
542 const struct tcf_exts *exts);
543int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
544 void *type_data, bool err_stop);
545unsigned int tcf_exts_num_actions(struct tcf_exts *exts);
546
547struct tc_cls_u32_knode {
548 struct tcf_exts *exts;
549 struct tcf_result *res;
550 struct tc_u32_sel *sel;
551 u32 handle;
552 u32 val;
553 u32 mask;
554 u32 link_handle;
555 u8 fshift;
556};
557
558struct tc_cls_u32_hnode {
559 u32 handle;
560 u32 prio;
561 unsigned int divisor;
562};
563
564enum tc_clsu32_command {
565 TC_CLSU32_NEW_KNODE,
566 TC_CLSU32_REPLACE_KNODE,
567 TC_CLSU32_DELETE_KNODE,
568 TC_CLSU32_NEW_HNODE,
569 TC_CLSU32_REPLACE_HNODE,
570 TC_CLSU32_DELETE_HNODE,
571};
572
573struct tc_cls_u32_offload {
574 struct flow_cls_common_offload common;
575
576 enum tc_clsu32_command command;
577 union {
578 struct tc_cls_u32_knode knode;
579 struct tc_cls_u32_hnode hnode;
580 };
581};
582
583static inline bool tc_can_offload(const struct net_device *dev)
584{
585 return dev->features & NETIF_F_HW_TC;
586}
587
588static inline bool tc_can_offload_extack(const struct net_device *dev,
589 struct netlink_ext_ack *extack)
590{
591 bool can = tc_can_offload(dev);
592
593 if (!can)
594 NL_SET_ERR_MSG(extack, "TC offload is disabled on net device");
595
596 return can;
597}
598
599static inline bool
600tc_cls_can_offload_and_chain0(const struct net_device *dev,
601 struct flow_cls_common_offload *common)
602{
603 if (!tc_can_offload_extack(dev, common->extack))
604 return false;
605 if (common->chain_index) {
606 NL_SET_ERR_MSG(common->extack,
607 "Driver supports only offload of chain 0");
608 return false;
609 }
610 return true;
611}
612
613static inline bool tc_skip_hw(u32 flags)
614{
615 return (flags & TCA_CLS_FLAGS_SKIP_HW) ? true : false;
616}
617
618static inline bool tc_skip_sw(u32 flags)
619{
620 return (flags & TCA_CLS_FLAGS_SKIP_SW) ? true : false;
621}
622
623
624static inline bool tc_flags_valid(u32 flags)
625{
626 if (flags & ~(TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW |
627 TCA_CLS_FLAGS_VERBOSE))
628 return false;
629
630 flags &= TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW;
631 if (!(flags ^ (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)))
632 return false;
633
634 return true;
635}
636
637static inline bool tc_in_hw(u32 flags)
638{
639 return (flags & TCA_CLS_FLAGS_IN_HW) ? true : false;
640}
641
642static inline void
643tc_cls_common_offload_init(struct flow_cls_common_offload *cls_common,
644 const struct tcf_proto *tp, u32 flags,
645 struct netlink_ext_ack *extack)
646{
647 cls_common->chain_index = tp->chain->index;
648 cls_common->protocol = tp->protocol;
649 cls_common->prio = tp->prio >> 16;
650 if (tc_skip_sw(flags) || flags & TCA_CLS_FLAGS_VERBOSE)
651 cls_common->extack = extack;
652}
653
654enum tc_matchall_command {
655 TC_CLSMATCHALL_REPLACE,
656 TC_CLSMATCHALL_DESTROY,
657 TC_CLSMATCHALL_STATS,
658};
659
660struct tc_cls_matchall_offload {
661 struct flow_cls_common_offload common;
662 enum tc_matchall_command command;
663 struct flow_rule *rule;
664 struct flow_stats stats;
665 unsigned long cookie;
666};
667
668enum tc_clsbpf_command {
669 TC_CLSBPF_OFFLOAD,
670 TC_CLSBPF_STATS,
671};
672
673struct tc_cls_bpf_offload {
674 struct flow_cls_common_offload common;
675 enum tc_clsbpf_command command;
676 struct tcf_exts *exts;
677 struct bpf_prog *prog;
678 struct bpf_prog *oldprog;
679 const char *name;
680 bool exts_integrated;
681};
682
683struct tc_mqprio_qopt_offload {
684
685 struct tc_mqprio_qopt qopt;
686 u16 mode;
687 u16 shaper;
688 u32 flags;
689 u64 min_rate[TC_QOPT_MAX_QUEUE];
690 u64 max_rate[TC_QOPT_MAX_QUEUE];
691};
692
693
694
695
696struct tc_cookie {
697 u8 *data;
698 u32 len;
699 struct rcu_head rcu;
700};
701
702struct tc_qopt_offload_stats {
703 struct gnet_stats_basic_packed *bstats;
704 struct gnet_stats_queue *qstats;
705};
706
707enum tc_mq_command {
708 TC_MQ_CREATE,
709 TC_MQ_DESTROY,
710 TC_MQ_STATS,
711 TC_MQ_GRAFT,
712};
713
714struct tc_mq_opt_offload_graft_params {
715 unsigned long queue;
716 u32 child_handle;
717};
718
719struct tc_mq_qopt_offload {
720 enum tc_mq_command command;
721 u32 handle;
722 union {
723 struct tc_qopt_offload_stats stats;
724 struct tc_mq_opt_offload_graft_params graft_params;
725 };
726};
727
728enum tc_red_command {
729 TC_RED_REPLACE,
730 TC_RED_DESTROY,
731 TC_RED_STATS,
732 TC_RED_XSTATS,
733 TC_RED_GRAFT,
734};
735
736struct tc_red_qopt_offload_params {
737 u32 min;
738 u32 max;
739 u32 probability;
740 u32 limit;
741 bool is_ecn;
742 bool is_harddrop;
743 struct gnet_stats_queue *qstats;
744};
745
746struct tc_red_qopt_offload {
747 enum tc_red_command command;
748 u32 handle;
749 u32 parent;
750 union {
751 struct tc_red_qopt_offload_params set;
752 struct tc_qopt_offload_stats stats;
753 struct red_stats *xstats;
754 u32 child_handle;
755 };
756};
757
758enum tc_gred_command {
759 TC_GRED_REPLACE,
760 TC_GRED_DESTROY,
761 TC_GRED_STATS,
762};
763
764struct tc_gred_vq_qopt_offload_params {
765 bool present;
766 u32 limit;
767 u32 prio;
768 u32 min;
769 u32 max;
770 bool is_ecn;
771 bool is_harddrop;
772 u32 probability;
773
774 u32 *backlog;
775};
776
777struct tc_gred_qopt_offload_params {
778 bool grio_on;
779 bool wred_on;
780 unsigned int dp_cnt;
781 unsigned int dp_def;
782 struct gnet_stats_queue *qstats;
783 struct tc_gred_vq_qopt_offload_params tab[MAX_DPs];
784};
785
786struct tc_gred_qopt_offload_stats {
787 struct gnet_stats_basic_packed bstats[MAX_DPs];
788 struct gnet_stats_queue qstats[MAX_DPs];
789 struct red_stats *xstats[MAX_DPs];
790};
791
792struct tc_gred_qopt_offload {
793 enum tc_gred_command command;
794 u32 handle;
795 u32 parent;
796 union {
797 struct tc_gred_qopt_offload_params set;
798 struct tc_gred_qopt_offload_stats stats;
799 };
800};
801
802enum tc_prio_command {
803 TC_PRIO_REPLACE,
804 TC_PRIO_DESTROY,
805 TC_PRIO_STATS,
806 TC_PRIO_GRAFT,
807};
808
809struct tc_prio_qopt_offload_params {
810 int bands;
811 u8 priomap[TC_PRIO_MAX + 1];
812
813
814
815
816 struct gnet_stats_queue *qstats;
817};
818
819struct tc_prio_qopt_offload_graft_params {
820 u8 band;
821 u32 child_handle;
822};
823
824struct tc_prio_qopt_offload {
825 enum tc_prio_command command;
826 u32 handle;
827 u32 parent;
828 union {
829 struct tc_prio_qopt_offload_params replace_params;
830 struct tc_qopt_offload_stats stats;
831 struct tc_prio_qopt_offload_graft_params graft_params;
832 };
833};
834
835enum tc_root_command {
836 TC_ROOT_GRAFT,
837};
838
839struct tc_root_qopt_offload {
840 enum tc_root_command command;
841 u32 handle;
842 bool ingress;
843};
844
845#endif
846