1
2#ifndef _NET_NF_TABLES_H
3#define _NET_NF_TABLES_H
4
5#include <asm/unaligned.h>
6#include <linux/list.h>
7#include <linux/netfilter.h>
8#include <linux/netfilter/nfnetlink.h>
9#include <linux/netfilter/x_tables.h>
10#include <linux/netfilter/nf_tables.h>
11#include <linux/u64_stats_sync.h>
12#include <linux/rhashtable.h>
13#include <net/netfilter/nf_flow_table.h>
14#include <net/netlink.h>
15#include <net/flow_offload.h>
16
17#define NFT_MAX_HOOKS (NF_INET_INGRESS + 1)
18
19struct module;
20
21#define NFT_JUMP_STACK_SIZE 16
22
23struct nft_pktinfo {
24 struct sk_buff *skb;
25 bool tprot_set;
26 u8 tprot;
27
28 struct xt_action_param xt;
29};
30
31static inline struct net *nft_net(const struct nft_pktinfo *pkt)
32{
33 return pkt->xt.state->net;
34}
35
36static inline unsigned int nft_hook(const struct nft_pktinfo *pkt)
37{
38 return pkt->xt.state->hook;
39}
40
41static inline u8 nft_pf(const struct nft_pktinfo *pkt)
42{
43 return pkt->xt.state->pf;
44}
45
46static inline const struct net_device *nft_in(const struct nft_pktinfo *pkt)
47{
48 return pkt->xt.state->in;
49}
50
51static inline const struct net_device *nft_out(const struct nft_pktinfo *pkt)
52{
53 return pkt->xt.state->out;
54}
55
56static inline void nft_set_pktinfo(struct nft_pktinfo *pkt,
57 struct sk_buff *skb,
58 const struct nf_hook_state *state)
59{
60 pkt->skb = skb;
61 pkt->xt.state = state;
62}
63
64static inline void nft_set_pktinfo_unspec(struct nft_pktinfo *pkt,
65 struct sk_buff *skb)
66{
67 pkt->tprot_set = false;
68 pkt->tprot = 0;
69 pkt->xt.thoff = 0;
70 pkt->xt.fragoff = 0;
71}
72
73
74
75
76
77
78
79struct nft_verdict {
80 u32 code;
81 struct nft_chain *chain;
82};
83
84struct nft_data {
85 union {
86 u32 data[4];
87 struct nft_verdict verdict;
88 };
89} __attribute__((aligned(__alignof__(u64))));
90
91
92
93
94
95
96
97
98
99struct nft_regs {
100 union {
101 u32 data[20];
102 struct nft_verdict verdict;
103 };
104};
105
106
107
108
109
110
111
112
113static inline void nft_reg_store8(u32 *dreg, u8 val)
114{
115 *dreg = 0;
116 *(u8 *)dreg = val;
117}
118
119static inline u8 nft_reg_load8(const u32 *sreg)
120{
121 return *(u8 *)sreg;
122}
123
124static inline void nft_reg_store16(u32 *dreg, u16 val)
125{
126 *dreg = 0;
127 *(u16 *)dreg = val;
128}
129
130static inline u16 nft_reg_load16(const u32 *sreg)
131{
132 return *(u16 *)sreg;
133}
134
135static inline void nft_reg_store64(u32 *dreg, u64 val)
136{
137 put_unaligned(val, (u64 *)dreg);
138}
139
140static inline u64 nft_reg_load64(const u32 *sreg)
141{
142 return get_unaligned((u64 *)sreg);
143}
144
145static inline void nft_data_copy(u32 *dst, const struct nft_data *src,
146 unsigned int len)
147{
148 if (len % NFT_REG32_SIZE)
149 dst[len / NFT_REG32_SIZE] = 0;
150 memcpy(dst, src, len);
151}
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166struct nft_ctx {
167 struct net *net;
168 struct nft_table *table;
169 struct nft_chain *chain;
170 const struct nlattr * const *nla;
171 u32 portid;
172 u32 seq;
173 u16 flags;
174 u8 family;
175 u8 level;
176 bool report;
177};
178
179struct nft_data_desc {
180 enum nft_data_types type;
181 unsigned int len;
182};
183
184int nft_data_init(const struct nft_ctx *ctx,
185 struct nft_data *data, unsigned int size,
186 struct nft_data_desc *desc, const struct nlattr *nla);
187void nft_data_hold(const struct nft_data *data, enum nft_data_types type);
188void nft_data_release(const struct nft_data *data, enum nft_data_types type);
189int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
190 enum nft_data_types type, unsigned int len);
191
192static inline enum nft_data_types nft_dreg_to_type(enum nft_registers reg)
193{
194 return reg == NFT_REG_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE;
195}
196
197static inline enum nft_registers nft_type_to_reg(enum nft_data_types type)
198{
199 return type == NFT_DATA_VERDICT ? NFT_REG_VERDICT : NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE;
200}
201
202int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest);
203int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg);
204
205int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len);
206int nft_parse_register_store(const struct nft_ctx *ctx,
207 const struct nlattr *attr, u8 *dreg,
208 const struct nft_data *data,
209 enum nft_data_types type, unsigned int len);
210
211
212
213
214
215
216
217
218
219
220
221struct nft_userdata {
222 u8 len;
223 unsigned char data[];
224};
225
226
227
228
229
230
231
232
233struct nft_set_elem {
234 union {
235 u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
236 struct nft_data val;
237 } key;
238 union {
239 u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
240 struct nft_data val;
241 } key_end;
242 union {
243 u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
244 struct nft_data val;
245 } data;
246 void *priv;
247};
248
249struct nft_set;
250struct nft_set_iter {
251 u8 genmask;
252 unsigned int count;
253 unsigned int skip;
254 int err;
255 int (*fn)(const struct nft_ctx *ctx,
256 struct nft_set *set,
257 const struct nft_set_iter *iter,
258 struct nft_set_elem *elem);
259};
260
261
262
263
264
265
266
267
268
269
270
271struct nft_set_desc {
272 unsigned int klen;
273 unsigned int dlen;
274 unsigned int size;
275 u8 field_len[NFT_REG32_COUNT];
276 u8 field_count;
277 bool expr;
278};
279
280
281
282
283
284
285
286
287enum nft_set_class {
288 NFT_SET_CLASS_O_1,
289 NFT_SET_CLASS_O_LOG_N,
290 NFT_SET_CLASS_O_N,
291};
292
293
294
295
296
297
298
299
300
301struct nft_set_estimate {
302 u64 size;
303 enum nft_set_class lookup;
304 enum nft_set_class space;
305};
306
307#define NFT_EXPR_MAXATTR 16
308#define NFT_EXPR_SIZE(size) (sizeof(struct nft_expr) + \
309 ALIGN(size, __alignof__(struct nft_expr)))
310
311
312
313
314
315
316
317struct nft_expr {
318 const struct nft_expr_ops *ops;
319 unsigned char data[]
320 __attribute__((aligned(__alignof__(u64))));
321};
322
323static inline void *nft_expr_priv(const struct nft_expr *expr)
324{
325 return (void *)expr->data;
326}
327
328int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src);
329void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr);
330int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
331 const struct nft_expr *expr);
332
333struct nft_set_ext;
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357struct nft_set_ops {
358 bool (*lookup)(const struct net *net,
359 const struct nft_set *set,
360 const u32 *key,
361 const struct nft_set_ext **ext);
362 bool (*update)(struct nft_set *set,
363 const u32 *key,
364 void *(*new)(struct nft_set *,
365 const struct nft_expr *,
366 struct nft_regs *),
367 const struct nft_expr *expr,
368 struct nft_regs *regs,
369 const struct nft_set_ext **ext);
370 bool (*delete)(const struct nft_set *set,
371 const u32 *key);
372
373 int (*insert)(const struct net *net,
374 const struct nft_set *set,
375 const struct nft_set_elem *elem,
376 struct nft_set_ext **ext);
377 void (*activate)(const struct net *net,
378 const struct nft_set *set,
379 const struct nft_set_elem *elem);
380 void * (*deactivate)(const struct net *net,
381 const struct nft_set *set,
382 const struct nft_set_elem *elem);
383 bool (*flush)(const struct net *net,
384 const struct nft_set *set,
385 void *priv);
386 void (*remove)(const struct net *net,
387 const struct nft_set *set,
388 const struct nft_set_elem *elem);
389 void (*walk)(const struct nft_ctx *ctx,
390 struct nft_set *set,
391 struct nft_set_iter *iter);
392 void * (*get)(const struct net *net,
393 const struct nft_set *set,
394 const struct nft_set_elem *elem,
395 unsigned int flags);
396
397 u64 (*privsize)(const struct nlattr * const nla[],
398 const struct nft_set_desc *desc);
399 bool (*estimate)(const struct nft_set_desc *desc,
400 u32 features,
401 struct nft_set_estimate *est);
402 int (*init)(const struct nft_set *set,
403 const struct nft_set_desc *desc,
404 const struct nlattr * const nla[]);
405 void (*destroy)(const struct nft_set *set);
406 void (*gc_init)(const struct nft_set *set);
407
408 unsigned int elemsize;
409};
410
411
412
413
414
415
416
417struct nft_set_type {
418 const struct nft_set_ops ops;
419 u32 features;
420};
421#define to_set_type(o) container_of(o, struct nft_set_type, ops)
422
423struct nft_set_elem_expr {
424 u8 size;
425 unsigned char data[]
426 __attribute__((aligned(__alignof__(struct nft_expr))));
427};
428
429#define nft_setelem_expr_at(__elem_expr, __offset) \
430 ((struct nft_expr *)&__elem_expr->data[__offset])
431
432#define nft_setelem_expr_foreach(__expr, __elem_expr, __size) \
433 for (__expr = nft_setelem_expr_at(__elem_expr, 0), __size = 0; \
434 __size < (__elem_expr)->size; \
435 __size += (__expr)->ops->size, __expr = ((void *)(__expr)) + (__expr)->ops->size)
436
437#define NFT_SET_EXPR_MAX 2
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470struct nft_set {
471 struct list_head list;
472 struct list_head bindings;
473 struct nft_table *table;
474 possible_net_t net;
475 char *name;
476 u64 handle;
477 u32 ktype;
478 u32 dtype;
479 u32 objtype;
480 u32 size;
481 u8 field_len[NFT_REG32_COUNT];
482 u8 field_count;
483 u32 use;
484 atomic_t nelems;
485 u32 ndeact;
486 u64 timeout;
487 u32 gc_int;
488 u16 policy;
489 u16 udlen;
490 unsigned char *udata;
491
492 const struct nft_set_ops *ops ____cacheline_aligned;
493 u16 flags:14,
494 genmask:2;
495 u8 klen;
496 u8 dlen;
497 u8 num_exprs;
498 struct nft_expr *exprs[NFT_SET_EXPR_MAX];
499 unsigned char data[]
500 __attribute__((aligned(__alignof__(u64))));
501};
502
503static inline bool nft_set_is_anonymous(const struct nft_set *set)
504{
505 return set->flags & NFT_SET_ANONYMOUS;
506}
507
508static inline void *nft_set_priv(const struct nft_set *set)
509{
510 return (void *)set->data;
511}
512
513static inline struct nft_set *nft_set_container_of(const void *priv)
514{
515 return (void *)priv - offsetof(struct nft_set, data);
516}
517
518struct nft_set *nft_set_lookup_global(const struct net *net,
519 const struct nft_table *table,
520 const struct nlattr *nla_set_name,
521 const struct nlattr *nla_set_id,
522 u8 genmask);
523
524static inline unsigned long nft_set_gc_interval(const struct nft_set *set)
525{
526 return set->gc_int ? msecs_to_jiffies(set->gc_int) : HZ;
527}
528
529
530
531
532
533
534
535
536
537
538
539struct nft_set_binding {
540 struct list_head list;
541 const struct nft_chain *chain;
542 u32 flags;
543};
544
545enum nft_trans_phase;
546void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
547 struct nft_set_binding *binding,
548 enum nft_trans_phase phase);
549int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
550 struct nft_set_binding *binding);
551void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set);
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567enum nft_set_extensions {
568 NFT_SET_EXT_KEY,
569 NFT_SET_EXT_KEY_END,
570 NFT_SET_EXT_DATA,
571 NFT_SET_EXT_FLAGS,
572 NFT_SET_EXT_TIMEOUT,
573 NFT_SET_EXT_EXPIRATION,
574 NFT_SET_EXT_USERDATA,
575 NFT_SET_EXT_EXPRESSIONS,
576 NFT_SET_EXT_OBJREF,
577 NFT_SET_EXT_NUM
578};
579
580
581
582
583
584
585
586struct nft_set_ext_type {
587 u8 len;
588 u8 align;
589};
590
591extern const struct nft_set_ext_type nft_set_ext_types[];
592
593
594
595
596
597
598
599struct nft_set_ext_tmpl {
600 u16 len;
601 u8 offset[NFT_SET_EXT_NUM];
602};
603
604
605
606
607
608
609
610
611struct nft_set_ext {
612 u8 genmask;
613 u8 offset[NFT_SET_EXT_NUM];
614 char data[];
615};
616
617static inline void nft_set_ext_prepare(struct nft_set_ext_tmpl *tmpl)
618{
619 memset(tmpl, 0, sizeof(*tmpl));
620 tmpl->len = sizeof(struct nft_set_ext);
621}
622
623static inline void nft_set_ext_add_length(struct nft_set_ext_tmpl *tmpl, u8 id,
624 unsigned int len)
625{
626 tmpl->len = ALIGN(tmpl->len, nft_set_ext_types[id].align);
627 BUG_ON(tmpl->len > U8_MAX);
628 tmpl->offset[id] = tmpl->len;
629 tmpl->len += nft_set_ext_types[id].len + len;
630}
631
632static inline void nft_set_ext_add(struct nft_set_ext_tmpl *tmpl, u8 id)
633{
634 nft_set_ext_add_length(tmpl, id, 0);
635}
636
637static inline void nft_set_ext_init(struct nft_set_ext *ext,
638 const struct nft_set_ext_tmpl *tmpl)
639{
640 memcpy(ext->offset, tmpl->offset, sizeof(ext->offset));
641}
642
643static inline bool __nft_set_ext_exists(const struct nft_set_ext *ext, u8 id)
644{
645 return !!ext->offset[id];
646}
647
648static inline bool nft_set_ext_exists(const struct nft_set_ext *ext, u8 id)
649{
650 return ext && __nft_set_ext_exists(ext, id);
651}
652
653static inline void *nft_set_ext(const struct nft_set_ext *ext, u8 id)
654{
655 return (void *)ext + ext->offset[id];
656}
657
658static inline struct nft_data *nft_set_ext_key(const struct nft_set_ext *ext)
659{
660 return nft_set_ext(ext, NFT_SET_EXT_KEY);
661}
662
663static inline struct nft_data *nft_set_ext_key_end(const struct nft_set_ext *ext)
664{
665 return nft_set_ext(ext, NFT_SET_EXT_KEY_END);
666}
667
668static inline struct nft_data *nft_set_ext_data(const struct nft_set_ext *ext)
669{
670 return nft_set_ext(ext, NFT_SET_EXT_DATA);
671}
672
673static inline u8 *nft_set_ext_flags(const struct nft_set_ext *ext)
674{
675 return nft_set_ext(ext, NFT_SET_EXT_FLAGS);
676}
677
678static inline u64 *nft_set_ext_timeout(const struct nft_set_ext *ext)
679{
680 return nft_set_ext(ext, NFT_SET_EXT_TIMEOUT);
681}
682
683static inline u64 *nft_set_ext_expiration(const struct nft_set_ext *ext)
684{
685 return nft_set_ext(ext, NFT_SET_EXT_EXPIRATION);
686}
687
688static inline struct nft_userdata *nft_set_ext_userdata(const struct nft_set_ext *ext)
689{
690 return nft_set_ext(ext, NFT_SET_EXT_USERDATA);
691}
692
693static inline struct nft_set_elem_expr *nft_set_ext_expr(const struct nft_set_ext *ext)
694{
695 return nft_set_ext(ext, NFT_SET_EXT_EXPRESSIONS);
696}
697
698static inline bool nft_set_elem_expired(const struct nft_set_ext *ext)
699{
700 return nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION) &&
701 time_is_before_eq_jiffies64(*nft_set_ext_expiration(ext));
702}
703
704static inline struct nft_set_ext *nft_set_elem_ext(const struct nft_set *set,
705 void *elem)
706{
707 return elem + set->ops->elemsize;
708}
709
710static inline struct nft_object **nft_set_ext_obj(const struct nft_set_ext *ext)
711{
712 return nft_set_ext(ext, NFT_SET_EXT_OBJREF);
713}
714
715struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
716 const struct nft_set *set,
717 const struct nlattr *attr);
718
719void *nft_set_elem_init(const struct nft_set *set,
720 const struct nft_set_ext_tmpl *tmpl,
721 const u32 *key, const u32 *key_end, const u32 *data,
722 u64 timeout, u64 expiration, gfp_t gfp);
723int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
724 struct nft_expr *expr_array[]);
725void nft_set_elem_destroy(const struct nft_set *set, void *elem,
726 bool destroy_expr);
727
728
729
730
731
732
733
734
735struct nft_set_gc_batch_head {
736 struct rcu_head rcu;
737 const struct nft_set *set;
738 unsigned int cnt;
739};
740
741#define NFT_SET_GC_BATCH_SIZE ((PAGE_SIZE - \
742 sizeof(struct nft_set_gc_batch_head)) / \
743 sizeof(void *))
744
745
746
747
748
749
750
751struct nft_set_gc_batch {
752 struct nft_set_gc_batch_head head;
753 void *elems[NFT_SET_GC_BATCH_SIZE];
754};
755
756struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
757 gfp_t gfp);
758void nft_set_gc_batch_release(struct rcu_head *rcu);
759
760static inline void nft_set_gc_batch_complete(struct nft_set_gc_batch *gcb)
761{
762 if (gcb != NULL)
763 call_rcu(&gcb->head.rcu, nft_set_gc_batch_release);
764}
765
766static inline struct nft_set_gc_batch *
767nft_set_gc_batch_check(const struct nft_set *set, struct nft_set_gc_batch *gcb,
768 gfp_t gfp)
769{
770 if (gcb != NULL) {
771 if (gcb->head.cnt + 1 < ARRAY_SIZE(gcb->elems))
772 return gcb;
773 nft_set_gc_batch_complete(gcb);
774 }
775 return nft_set_gc_batch_alloc(set, gfp);
776}
777
778static inline void nft_set_gc_batch_add(struct nft_set_gc_batch *gcb,
779 void *elem)
780{
781 gcb->elems[gcb->head.cnt++] = elem;
782}
783
784struct nft_expr_ops;
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799struct nft_expr_type {
800 const struct nft_expr_ops *(*select_ops)(const struct nft_ctx *,
801 const struct nlattr * const tb[]);
802 void (*release_ops)(const struct nft_expr_ops *ops);
803 const struct nft_expr_ops *ops;
804 struct list_head list;
805 const char *name;
806 struct module *owner;
807 const struct nla_policy *policy;
808 unsigned int maxattr;
809 u8 family;
810 u8 flags;
811};
812
813#define NFT_EXPR_STATEFUL 0x1
814#define NFT_EXPR_GC 0x2
815
816enum nft_trans_phase {
817 NFT_TRANS_PREPARE,
818 NFT_TRANS_ABORT,
819 NFT_TRANS_COMMIT,
820 NFT_TRANS_RELEASE
821};
822
823struct nft_flow_rule;
824struct nft_offload_ctx;
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840struct nft_expr_ops {
841 void (*eval)(const struct nft_expr *expr,
842 struct nft_regs *regs,
843 const struct nft_pktinfo *pkt);
844 int (*clone)(struct nft_expr *dst,
845 const struct nft_expr *src);
846 unsigned int size;
847
848 int (*init)(const struct nft_ctx *ctx,
849 const struct nft_expr *expr,
850 const struct nlattr * const tb[]);
851 void (*activate)(const struct nft_ctx *ctx,
852 const struct nft_expr *expr);
853 void (*deactivate)(const struct nft_ctx *ctx,
854 const struct nft_expr *expr,
855 enum nft_trans_phase phase);
856 void (*destroy)(const struct nft_ctx *ctx,
857 const struct nft_expr *expr);
858 void (*destroy_clone)(const struct nft_ctx *ctx,
859 const struct nft_expr *expr);
860 int (*dump)(struct sk_buff *skb,
861 const struct nft_expr *expr);
862 int (*validate)(const struct nft_ctx *ctx,
863 const struct nft_expr *expr,
864 const struct nft_data **data);
865 bool (*gc)(struct net *net,
866 const struct nft_expr *expr);
867 int (*offload)(struct nft_offload_ctx *ctx,
868 struct nft_flow_rule *flow,
869 const struct nft_expr *expr);
870 u32 offload_flags;
871 const struct nft_expr_type *type;
872 void *data;
873};
874
875
876
877
878
879
880
881
882
883
884
885struct nft_rule {
886 struct list_head list;
887 u64 handle:42,
888 genmask:2,
889 dlen:12,
890 udata:1;
891 unsigned char data[]
892 __attribute__((aligned(__alignof__(struct nft_expr))));
893};
894
895static inline struct nft_expr *nft_expr_first(const struct nft_rule *rule)
896{
897 return (struct nft_expr *)&rule->data[0];
898}
899
900static inline struct nft_expr *nft_expr_next(const struct nft_expr *expr)
901{
902 return ((void *)expr) + expr->ops->size;
903}
904
905static inline struct nft_expr *nft_expr_last(const struct nft_rule *rule)
906{
907 return (struct nft_expr *)&rule->data[rule->dlen];
908}
909
910static inline bool nft_expr_more(const struct nft_rule *rule,
911 const struct nft_expr *expr)
912{
913 return expr != nft_expr_last(rule) && expr->ops;
914}
915
916static inline struct nft_userdata *nft_userdata(const struct nft_rule *rule)
917{
918 return (void *)&rule->data[rule->dlen];
919}
920
921void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule);
922
923static inline void nft_set_elem_update_expr(const struct nft_set_ext *ext,
924 struct nft_regs *regs,
925 const struct nft_pktinfo *pkt)
926{
927 struct nft_set_elem_expr *elem_expr;
928 struct nft_expr *expr;
929 u32 size;
930
931 if (__nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS)) {
932 elem_expr = nft_set_ext_expr(ext);
933 nft_setelem_expr_foreach(expr, elem_expr, size) {
934 expr->ops->eval(expr, regs, pkt);
935 if (regs->verdict.code == NFT_BREAK)
936 return;
937 }
938 }
939}
940
941
942
943
944
945
946#define nft_rule_for_each_expr(expr, last, rule) \
947 for ((expr) = nft_expr_first(rule), (last) = nft_expr_last(rule); \
948 (expr) != (last); \
949 (expr) = nft_expr_next(expr))
950
951#define NFT_CHAIN_POLICY_UNSET U8_MAX
952
953
954
955
956
957
958
959
960
961
962
963
964
965struct nft_chain {
966 struct nft_rule *__rcu *rules_gen_0;
967 struct nft_rule *__rcu *rules_gen_1;
968 struct list_head rules;
969 struct list_head list;
970 struct rhlist_head rhlhead;
971 struct nft_table *table;
972 u64 handle;
973 u32 use;
974 u8 flags:5,
975 bound:1,
976 genmask:2;
977 char *name;
978 u16 udlen;
979 u8 *udata;
980
981
982 struct nft_rule **rules_next;
983};
984
985int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain);
986
987enum nft_chain_types {
988 NFT_CHAIN_T_DEFAULT = 0,
989 NFT_CHAIN_T_ROUTE,
990 NFT_CHAIN_T_NAT,
991 NFT_CHAIN_T_MAX
992};
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006struct nft_chain_type {
1007 const char *name;
1008 enum nft_chain_types type;
1009 int family;
1010 struct module *owner;
1011 unsigned int hook_mask;
1012 nf_hookfn *hooks[NFT_MAX_HOOKS];
1013 int (*ops_register)(struct net *net, const struct nf_hook_ops *ops);
1014 void (*ops_unregister)(struct net *net, const struct nf_hook_ops *ops);
1015};
1016
1017int nft_chain_validate_dependency(const struct nft_chain *chain,
1018 enum nft_chain_types type);
1019int nft_chain_validate_hooks(const struct nft_chain *chain,
1020 unsigned int hook_flags);
1021
1022static inline bool nft_chain_is_bound(struct nft_chain *chain)
1023{
1024 return (chain->flags & NFT_CHAIN_BINDING) && chain->bound;
1025}
1026
1027void nft_chain_del(struct nft_chain *chain);
1028void nf_tables_chain_destroy(struct nft_ctx *ctx);
1029
1030struct nft_stats {
1031 u64 bytes;
1032 u64 pkts;
1033 struct u64_stats_sync syncp;
1034};
1035
1036struct nft_hook {
1037 struct list_head list;
1038 bool inactive;
1039 struct nf_hook_ops ops;
1040 struct rcu_head rcu;
1041};
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054struct nft_base_chain {
1055 struct nf_hook_ops ops;
1056 struct list_head hook_list;
1057 const struct nft_chain_type *type;
1058 u8 policy;
1059 u8 flags;
1060 struct nft_stats __percpu *stats;
1061 struct nft_chain chain;
1062 struct flow_block flow_block;
1063};
1064
1065static inline struct nft_base_chain *nft_base_chain(const struct nft_chain *chain)
1066{
1067 return container_of(chain, struct nft_base_chain, chain);
1068}
1069
1070static inline bool nft_is_base_chain(const struct nft_chain *chain)
1071{
1072 return chain->flags & NFT_CHAIN_BASE;
1073}
1074
1075int __nft_release_basechain(struct nft_ctx *ctx);
1076
1077unsigned int nft_do_chain(struct nft_pktinfo *pkt, void *priv);
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096struct nft_table {
1097 struct list_head list;
1098 struct rhltable chains_ht;
1099 struct list_head chains;
1100 struct list_head sets;
1101 struct list_head objects;
1102 struct list_head flowtables;
1103 u64 hgenerator;
1104 u64 handle;
1105 u32 use;
1106 u16 family:6,
1107 flags:8,
1108 genmask:2;
1109 u32 nlpid;
1110 char *name;
1111 u16 udlen;
1112 u8 *udata;
1113};
1114
1115static inline bool nft_table_has_owner(const struct nft_table *table)
1116{
1117 return table->flags & NFT_TABLE_F_OWNER;
1118}
1119
1120static inline bool nft_base_chain_netdev(int family, u32 hooknum)
1121{
1122 return family == NFPROTO_NETDEV ||
1123 (family == NFPROTO_INET && hooknum == NF_INET_INGRESS);
1124}
1125
1126void nft_register_chain_type(const struct nft_chain_type *);
1127void nft_unregister_chain_type(const struct nft_chain_type *);
1128
1129int nft_register_expr(struct nft_expr_type *);
1130void nft_unregister_expr(struct nft_expr_type *);
1131
1132int nft_verdict_dump(struct sk_buff *skb, int type,
1133 const struct nft_verdict *v);
1134
1135
1136
1137
1138
1139
1140
1141struct nft_object_hash_key {
1142 const char *name;
1143 const struct nft_table *table;
1144};
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158struct nft_object {
1159 struct list_head list;
1160 struct rhlist_head rhlhead;
1161 struct nft_object_hash_key key;
1162 u32 genmask:2,
1163 use:30;
1164 u64 handle;
1165 u16 udlen;
1166 u8 *udata;
1167
1168 const struct nft_object_ops *ops ____cacheline_aligned;
1169 unsigned char data[]
1170 __attribute__((aligned(__alignof__(u64))));
1171};
1172
1173static inline void *nft_obj_data(const struct nft_object *obj)
1174{
1175 return (void *)obj->data;
1176}
1177
1178#define nft_expr_obj(expr) *((struct nft_object **)nft_expr_priv(expr))
1179
1180struct nft_object *nft_obj_lookup(const struct net *net,
1181 const struct nft_table *table,
1182 const struct nlattr *nla, u32 objtype,
1183 u8 genmask);
1184
1185void nft_obj_notify(struct net *net, const struct nft_table *table,
1186 struct nft_object *obj, u32 portid, u32 seq,
1187 int event, int family, int report, gfp_t gfp);
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200struct nft_object_type {
1201 const struct nft_object_ops *(*select_ops)(const struct nft_ctx *,
1202 const struct nlattr * const tb[]);
1203 const struct nft_object_ops *ops;
1204 struct list_head list;
1205 u32 type;
1206 unsigned int maxattr;
1207 struct module *owner;
1208 const struct nla_policy *policy;
1209};
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221struct nft_object_ops {
1222 void (*eval)(struct nft_object *obj,
1223 struct nft_regs *regs,
1224 const struct nft_pktinfo *pkt);
1225 unsigned int size;
1226 int (*init)(const struct nft_ctx *ctx,
1227 const struct nlattr *const tb[],
1228 struct nft_object *obj);
1229 void (*destroy)(const struct nft_ctx *ctx,
1230 struct nft_object *obj);
1231 int (*dump)(struct sk_buff *skb,
1232 struct nft_object *obj,
1233 bool reset);
1234 void (*update)(struct nft_object *obj,
1235 struct nft_object *newobj);
1236 const struct nft_object_type *type;
1237};
1238
1239int nft_register_obj(struct nft_object_type *obj_type);
1240void nft_unregister_obj(struct nft_object_type *obj_type);
1241
1242#define NFT_NETDEVICE_MAX 256
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259struct nft_flowtable {
1260 struct list_head list;
1261 struct nft_table *table;
1262 char *name;
1263 int hooknum;
1264 int ops_len;
1265 u32 genmask:2,
1266 use:30;
1267 u64 handle;
1268
1269 struct list_head hook_list ____cacheline_aligned;
1270 struct nf_flowtable data;
1271};
1272
1273struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
1274 const struct nlattr *nla,
1275 u8 genmask);
1276
1277void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
1278 struct nft_flowtable *flowtable,
1279 enum nft_trans_phase phase);
1280
1281void nft_register_flowtable_type(struct nf_flowtable_type *type);
1282void nft_unregister_flowtable_type(struct nf_flowtable_type *type);
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296struct nft_traceinfo {
1297 const struct nft_pktinfo *pkt;
1298 const struct nft_base_chain *basechain;
1299 const struct nft_chain *chain;
1300 const struct nft_rule *rule;
1301 const struct nft_verdict *verdict;
1302 enum nft_trace_types type;
1303 bool packet_dumped;
1304 bool trace;
1305};
1306
1307void nft_trace_init(struct nft_traceinfo *info, const struct nft_pktinfo *pkt,
1308 const struct nft_verdict *verdict,
1309 const struct nft_chain *basechain);
1310
1311void nft_trace_notify(struct nft_traceinfo *info);
1312
1313#define MODULE_ALIAS_NFT_CHAIN(family, name) \
1314 MODULE_ALIAS("nft-chain-" __stringify(family) "-" name)
1315
1316#define MODULE_ALIAS_NFT_AF_EXPR(family, name) \
1317 MODULE_ALIAS("nft-expr-" __stringify(family) "-" name)
1318
1319#define MODULE_ALIAS_NFT_EXPR(name) \
1320 MODULE_ALIAS("nft-expr-" name)
1321
1322#define MODULE_ALIAS_NFT_OBJ(type) \
1323 MODULE_ALIAS("nft-obj-" __stringify(type))
1324
1325#if IS_ENABLED(CONFIG_NF_TABLES)
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339static inline unsigned int nft_gencursor_next(const struct net *net)
1340{
1341 return net->nft.gencursor + 1 == 1 ? 1 : 0;
1342}
1343
1344static inline u8 nft_genmask_next(const struct net *net)
1345{
1346 return 1 << nft_gencursor_next(net);
1347}
1348
1349static inline u8 nft_genmask_cur(const struct net *net)
1350{
1351
1352 return 1 << READ_ONCE(net->nft.gencursor);
1353}
1354
1355#define NFT_GENMASK_ANY ((1 << 0) | (1 << 1))
1356
1357
1358
1359
1360
1361
1362#define nft_is_active(__net, __obj) \
1363 (((__obj)->genmask & nft_genmask_cur(__net)) == 0)
1364
1365
1366#define nft_is_active_next(__net, __obj) \
1367 (((__obj)->genmask & nft_genmask_next(__net)) == 0)
1368
1369
1370#define nft_activate_next(__net, __obj) \
1371 (__obj)->genmask = nft_genmask_cur(__net)
1372
1373
1374#define nft_deactivate_next(__net, __obj) \
1375 (__obj)->genmask = nft_genmask_next(__net)
1376
1377
1378#define nft_clear(__net, __obj) \
1379 (__obj)->genmask &= ~nft_genmask_next(__net)
1380#define nft_active_genmask(__obj, __genmask) \
1381 !((__obj)->genmask & __genmask)
1382
1383
1384
1385
1386
1387static inline bool nft_set_elem_active(const struct nft_set_ext *ext,
1388 u8 genmask)
1389{
1390 return !(ext->genmask & genmask);
1391}
1392
1393static inline void nft_set_elem_change_active(const struct net *net,
1394 const struct nft_set *set,
1395 struct nft_set_ext *ext)
1396{
1397 ext->genmask ^= nft_genmask_next(net);
1398}
1399
1400#endif
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412#define NFT_SET_ELEM_BUSY_MASK (1 << 2)
1413
1414#if defined(__LITTLE_ENDIAN_BITFIELD)
1415#define NFT_SET_ELEM_BUSY_BIT 2
1416#elif defined(__BIG_ENDIAN_BITFIELD)
1417#define NFT_SET_ELEM_BUSY_BIT (BITS_PER_LONG - BITS_PER_BYTE + 2)
1418#else
1419#error
1420#endif
1421
1422static inline int nft_set_elem_mark_busy(struct nft_set_ext *ext)
1423{
1424 unsigned long *word = (unsigned long *)ext;
1425
1426 BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);
1427 return test_and_set_bit(NFT_SET_ELEM_BUSY_BIT, word);
1428}
1429
1430static inline void nft_set_elem_clear_busy(struct nft_set_ext *ext)
1431{
1432 unsigned long *word = (unsigned long *)ext;
1433
1434 clear_bit(NFT_SET_ELEM_BUSY_BIT, word);
1435}
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446struct nft_trans {
1447 struct list_head list;
1448 int msg_type;
1449 bool put_net;
1450 struct nft_ctx ctx;
1451 char data[];
1452};
1453
1454struct nft_trans_rule {
1455 struct nft_rule *rule;
1456 struct nft_flow_rule *flow;
1457 u32 rule_id;
1458};
1459
1460#define nft_trans_rule(trans) \
1461 (((struct nft_trans_rule *)trans->data)->rule)
1462#define nft_trans_flow_rule(trans) \
1463 (((struct nft_trans_rule *)trans->data)->flow)
1464#define nft_trans_rule_id(trans) \
1465 (((struct nft_trans_rule *)trans->data)->rule_id)
1466
1467struct nft_trans_set {
1468 struct nft_set *set;
1469 u32 set_id;
1470 bool bound;
1471};
1472
1473#define nft_trans_set(trans) \
1474 (((struct nft_trans_set *)trans->data)->set)
1475#define nft_trans_set_id(trans) \
1476 (((struct nft_trans_set *)trans->data)->set_id)
1477#define nft_trans_set_bound(trans) \
1478 (((struct nft_trans_set *)trans->data)->bound)
1479
1480struct nft_trans_chain {
1481 bool update;
1482 char *name;
1483 struct nft_stats __percpu *stats;
1484 u8 policy;
1485 u32 chain_id;
1486};
1487
1488#define nft_trans_chain_update(trans) \
1489 (((struct nft_trans_chain *)trans->data)->update)
1490#define nft_trans_chain_name(trans) \
1491 (((struct nft_trans_chain *)trans->data)->name)
1492#define nft_trans_chain_stats(trans) \
1493 (((struct nft_trans_chain *)trans->data)->stats)
1494#define nft_trans_chain_policy(trans) \
1495 (((struct nft_trans_chain *)trans->data)->policy)
1496#define nft_trans_chain_id(trans) \
1497 (((struct nft_trans_chain *)trans->data)->chain_id)
1498
1499struct nft_trans_table {
1500 bool update;
1501 bool enable;
1502};
1503
1504#define nft_trans_table_update(trans) \
1505 (((struct nft_trans_table *)trans->data)->update)
1506#define nft_trans_table_enable(trans) \
1507 (((struct nft_trans_table *)trans->data)->enable)
1508
1509struct nft_trans_elem {
1510 struct nft_set *set;
1511 struct nft_set_elem elem;
1512 bool bound;
1513};
1514
1515#define nft_trans_elem_set(trans) \
1516 (((struct nft_trans_elem *)trans->data)->set)
1517#define nft_trans_elem(trans) \
1518 (((struct nft_trans_elem *)trans->data)->elem)
1519#define nft_trans_elem_set_bound(trans) \
1520 (((struct nft_trans_elem *)trans->data)->bound)
1521
1522struct nft_trans_obj {
1523 struct nft_object *obj;
1524 struct nft_object *newobj;
1525 bool update;
1526};
1527
1528#define nft_trans_obj(trans) \
1529 (((struct nft_trans_obj *)trans->data)->obj)
1530#define nft_trans_obj_newobj(trans) \
1531 (((struct nft_trans_obj *)trans->data)->newobj)
1532#define nft_trans_obj_update(trans) \
1533 (((struct nft_trans_obj *)trans->data)->update)
1534
1535struct nft_trans_flowtable {
1536 struct nft_flowtable *flowtable;
1537 bool update;
1538 struct list_head hook_list;
1539 u32 flags;
1540};
1541
1542#define nft_trans_flowtable(trans) \
1543 (((struct nft_trans_flowtable *)trans->data)->flowtable)
1544#define nft_trans_flowtable_update(trans) \
1545 (((struct nft_trans_flowtable *)trans->data)->update)
1546#define nft_trans_flowtable_hooks(trans) \
1547 (((struct nft_trans_flowtable *)trans->data)->hook_list)
1548#define nft_trans_flowtable_flags(trans) \
1549 (((struct nft_trans_flowtable *)trans->data)->flags)
1550
1551int __init nft_chain_filter_init(void);
1552void nft_chain_filter_fini(void);
1553
1554void __init nft_chain_route_init(void);
1555void nft_chain_route_fini(void);
1556
1557void nf_tables_trans_destroy_flush_work(void);
1558
1559int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result);
1560__be64 nf_jiffies64_to_msecs(u64 input);
1561
1562#endif
1563