1
2#ifndef _NET_NF_TABLES_H
3#define _NET_NF_TABLES_H
4
5#include <asm/unaligned.h>
6#include <linux/list.h>
7#include <linux/netfilter.h>
8#include <linux/netfilter/nfnetlink.h>
9#include <linux/netfilter/x_tables.h>
10#include <linux/netfilter/nf_tables.h>
11#include <linux/u64_stats_sync.h>
12#include <linux/rhashtable.h>
13#include <net/netfilter/nf_flow_table.h>
14#include <net/netlink.h>
15#include <net/flow_offload.h>
16#include <net/netns/generic.h>
17
18#define NFT_MAX_HOOKS (NF_INET_INGRESS + 1)
19
20struct module;
21
22#define NFT_JUMP_STACK_SIZE 16
23
24struct nft_pktinfo {
25 struct sk_buff *skb;
26 const struct nf_hook_state *state;
27 bool tprot_set;
28 u8 tprot;
29 u16 fragoff;
30 unsigned int thoff;
31};
32
33static inline struct sock *nft_sk(const struct nft_pktinfo *pkt)
34{
35 return pkt->state->sk;
36}
37
38static inline unsigned int nft_thoff(const struct nft_pktinfo *pkt)
39{
40 return pkt->thoff;
41}
42
43static inline struct net *nft_net(const struct nft_pktinfo *pkt)
44{
45 return pkt->state->net;
46}
47
48static inline unsigned int nft_hook(const struct nft_pktinfo *pkt)
49{
50 return pkt->state->hook;
51}
52
53static inline u8 nft_pf(const struct nft_pktinfo *pkt)
54{
55 return pkt->state->pf;
56}
57
58static inline const struct net_device *nft_in(const struct nft_pktinfo *pkt)
59{
60 return pkt->state->in;
61}
62
63static inline const struct net_device *nft_out(const struct nft_pktinfo *pkt)
64{
65 return pkt->state->out;
66}
67
68static inline void nft_set_pktinfo(struct nft_pktinfo *pkt,
69 struct sk_buff *skb,
70 const struct nf_hook_state *state)
71{
72 pkt->skb = skb;
73 pkt->state = state;
74}
75
76static inline void nft_set_pktinfo_unspec(struct nft_pktinfo *pkt)
77{
78 pkt->tprot_set = false;
79 pkt->tprot = 0;
80 pkt->thoff = 0;
81 pkt->fragoff = 0;
82}
83
84
85
86
87
88
89
90struct nft_verdict {
91 u32 code;
92 struct nft_chain *chain;
93};
94
95struct nft_data {
96 union {
97 u32 data[4];
98 struct nft_verdict verdict;
99 };
100} __attribute__((aligned(__alignof__(u64))));
101
102
103
104
105
106
107
108
109
110struct nft_regs {
111 union {
112 u32 data[20];
113 struct nft_verdict verdict;
114 };
115};
116
117
118
119
120
121
122
123
124static inline void nft_reg_store8(u32 *dreg, u8 val)
125{
126 *dreg = 0;
127 *(u8 *)dreg = val;
128}
129
130static inline u8 nft_reg_load8(const u32 *sreg)
131{
132 return *(u8 *)sreg;
133}
134
135static inline void nft_reg_store16(u32 *dreg, u16 val)
136{
137 *dreg = 0;
138 *(u16 *)dreg = val;
139}
140
141static inline u16 nft_reg_load16(const u32 *sreg)
142{
143 return *(u16 *)sreg;
144}
145
146static inline void nft_reg_store64(u32 *dreg, u64 val)
147{
148 put_unaligned(val, (u64 *)dreg);
149}
150
151static inline u64 nft_reg_load64(const u32 *sreg)
152{
153 return get_unaligned((u64 *)sreg);
154}
155
156static inline void nft_data_copy(u32 *dst, const struct nft_data *src,
157 unsigned int len)
158{
159 if (len % NFT_REG32_SIZE)
160 dst[len / NFT_REG32_SIZE] = 0;
161 memcpy(dst, src, len);
162}
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177struct nft_ctx {
178 struct net *net;
179 struct nft_table *table;
180 struct nft_chain *chain;
181 const struct nlattr * const *nla;
182 u32 portid;
183 u32 seq;
184 u16 flags;
185 u8 family;
186 u8 level;
187 bool report;
188};
189
190struct nft_data_desc {
191 enum nft_data_types type;
192 unsigned int len;
193};
194
195int nft_data_init(const struct nft_ctx *ctx,
196 struct nft_data *data, unsigned int size,
197 struct nft_data_desc *desc, const struct nlattr *nla);
198void nft_data_hold(const struct nft_data *data, enum nft_data_types type);
199void nft_data_release(const struct nft_data *data, enum nft_data_types type);
200int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
201 enum nft_data_types type, unsigned int len);
202
203static inline enum nft_data_types nft_dreg_to_type(enum nft_registers reg)
204{
205 return reg == NFT_REG_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE;
206}
207
208static inline enum nft_registers nft_type_to_reg(enum nft_data_types type)
209{
210 return type == NFT_DATA_VERDICT ? NFT_REG_VERDICT : NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE;
211}
212
213int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest);
214int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg);
215
216int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len);
217int nft_parse_register_store(const struct nft_ctx *ctx,
218 const struct nlattr *attr, u8 *dreg,
219 const struct nft_data *data,
220 enum nft_data_types type, unsigned int len);
221
222
223
224
225
226
227
228
229
230
231
232struct nft_userdata {
233 u8 len;
234 unsigned char data[];
235};
236
237
238
239
240
241
242
243
244struct nft_set_elem {
245 union {
246 u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
247 struct nft_data val;
248 } key;
249 union {
250 u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
251 struct nft_data val;
252 } key_end;
253 union {
254 u32 buf[NFT_DATA_VALUE_MAXLEN / sizeof(u32)];
255 struct nft_data val;
256 } data;
257 void *priv;
258};
259
260struct nft_set;
261struct nft_set_iter {
262 u8 genmask;
263 unsigned int count;
264 unsigned int skip;
265 int err;
266 int (*fn)(const struct nft_ctx *ctx,
267 struct nft_set *set,
268 const struct nft_set_iter *iter,
269 struct nft_set_elem *elem);
270};
271
272
273
274
275
276
277
278
279
280
281
282struct nft_set_desc {
283 unsigned int klen;
284 unsigned int dlen;
285 unsigned int size;
286 u8 field_len[NFT_REG32_COUNT];
287 u8 field_count;
288 bool expr;
289};
290
291
292
293
294
295
296
297
298enum nft_set_class {
299 NFT_SET_CLASS_O_1,
300 NFT_SET_CLASS_O_LOG_N,
301 NFT_SET_CLASS_O_N,
302};
303
304
305
306
307
308
309
310
311
312struct nft_set_estimate {
313 u64 size;
314 enum nft_set_class lookup;
315 enum nft_set_class space;
316};
317
318#define NFT_EXPR_MAXATTR 16
319#define NFT_EXPR_SIZE(size) (sizeof(struct nft_expr) + \
320 ALIGN(size, __alignof__(struct nft_expr)))
321
322
323
324
325
326
327
328struct nft_expr {
329 const struct nft_expr_ops *ops;
330 unsigned char data[]
331 __attribute__((aligned(__alignof__(u64))));
332};
333
334static inline void *nft_expr_priv(const struct nft_expr *expr)
335{
336 return (void *)expr->data;
337}
338
339int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src);
340void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr);
341int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
342 const struct nft_expr *expr);
343
344struct nft_set_ext;
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368struct nft_set_ops {
369 bool (*lookup)(const struct net *net,
370 const struct nft_set *set,
371 const u32 *key,
372 const struct nft_set_ext **ext);
373 bool (*update)(struct nft_set *set,
374 const u32 *key,
375 void *(*new)(struct nft_set *,
376 const struct nft_expr *,
377 struct nft_regs *),
378 const struct nft_expr *expr,
379 struct nft_regs *regs,
380 const struct nft_set_ext **ext);
381 bool (*delete)(const struct nft_set *set,
382 const u32 *key);
383
384 int (*insert)(const struct net *net,
385 const struct nft_set *set,
386 const struct nft_set_elem *elem,
387 struct nft_set_ext **ext);
388 void (*activate)(const struct net *net,
389 const struct nft_set *set,
390 const struct nft_set_elem *elem);
391 void * (*deactivate)(const struct net *net,
392 const struct nft_set *set,
393 const struct nft_set_elem *elem);
394 bool (*flush)(const struct net *net,
395 const struct nft_set *set,
396 void *priv);
397 void (*remove)(const struct net *net,
398 const struct nft_set *set,
399 const struct nft_set_elem *elem);
400 void (*walk)(const struct nft_ctx *ctx,
401 struct nft_set *set,
402 struct nft_set_iter *iter);
403 void * (*get)(const struct net *net,
404 const struct nft_set *set,
405 const struct nft_set_elem *elem,
406 unsigned int flags);
407
408 u64 (*privsize)(const struct nlattr * const nla[],
409 const struct nft_set_desc *desc);
410 bool (*estimate)(const struct nft_set_desc *desc,
411 u32 features,
412 struct nft_set_estimate *est);
413 int (*init)(const struct nft_set *set,
414 const struct nft_set_desc *desc,
415 const struct nlattr * const nla[]);
416 void (*destroy)(const struct nft_set *set);
417 void (*gc_init)(const struct nft_set *set);
418
419 unsigned int elemsize;
420};
421
422
423
424
425
426
427
428struct nft_set_type {
429 const struct nft_set_ops ops;
430 u32 features;
431};
432#define to_set_type(o) container_of(o, struct nft_set_type, ops)
433
434struct nft_set_elem_expr {
435 u8 size;
436 unsigned char data[]
437 __attribute__((aligned(__alignof__(struct nft_expr))));
438};
439
440#define nft_setelem_expr_at(__elem_expr, __offset) \
441 ((struct nft_expr *)&__elem_expr->data[__offset])
442
443#define nft_setelem_expr_foreach(__expr, __elem_expr, __size) \
444 for (__expr = nft_setelem_expr_at(__elem_expr, 0), __size = 0; \
445 __size < (__elem_expr)->size; \
446 __size += (__expr)->ops->size, __expr = ((void *)(__expr)) + (__expr)->ops->size)
447
448#define NFT_SET_EXPR_MAX 2
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481struct nft_set {
482 struct list_head list;
483 struct list_head bindings;
484 struct nft_table *table;
485 possible_net_t net;
486 char *name;
487 u64 handle;
488 u32 ktype;
489 u32 dtype;
490 u32 objtype;
491 u32 size;
492 u8 field_len[NFT_REG32_COUNT];
493 u8 field_count;
494 u32 use;
495 atomic_t nelems;
496 u32 ndeact;
497 u64 timeout;
498 u32 gc_int;
499 u16 policy;
500 u16 udlen;
501 unsigned char *udata;
502
503 const struct nft_set_ops *ops ____cacheline_aligned;
504 u16 flags:14,
505 genmask:2;
506 u8 klen;
507 u8 dlen;
508 u8 num_exprs;
509 struct nft_expr *exprs[NFT_SET_EXPR_MAX];
510 struct list_head catchall_list;
511 unsigned char data[]
512 __attribute__((aligned(__alignof__(u64))));
513};
514
515static inline bool nft_set_is_anonymous(const struct nft_set *set)
516{
517 return set->flags & NFT_SET_ANONYMOUS;
518}
519
520static inline void *nft_set_priv(const struct nft_set *set)
521{
522 return (void *)set->data;
523}
524
525static inline struct nft_set *nft_set_container_of(const void *priv)
526{
527 return (void *)priv - offsetof(struct nft_set, data);
528}
529
530struct nft_set *nft_set_lookup_global(const struct net *net,
531 const struct nft_table *table,
532 const struct nlattr *nla_set_name,
533 const struct nlattr *nla_set_id,
534 u8 genmask);
535
536struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
537 const struct nft_set *set);
538void *nft_set_catchall_gc(const struct nft_set *set);
539
540static inline unsigned long nft_set_gc_interval(const struct nft_set *set)
541{
542 return set->gc_int ? msecs_to_jiffies(set->gc_int) : HZ;
543}
544
545
546
547
548
549
550
551
552
553
554
555struct nft_set_binding {
556 struct list_head list;
557 const struct nft_chain *chain;
558 u32 flags;
559};
560
561enum nft_trans_phase;
562void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
563 struct nft_set_binding *binding,
564 enum nft_trans_phase phase);
565int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
566 struct nft_set_binding *binding);
567void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set);
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583enum nft_set_extensions {
584 NFT_SET_EXT_KEY,
585 NFT_SET_EXT_KEY_END,
586 NFT_SET_EXT_DATA,
587 NFT_SET_EXT_FLAGS,
588 NFT_SET_EXT_TIMEOUT,
589 NFT_SET_EXT_EXPIRATION,
590 NFT_SET_EXT_USERDATA,
591 NFT_SET_EXT_EXPRESSIONS,
592 NFT_SET_EXT_OBJREF,
593 NFT_SET_EXT_NUM
594};
595
596
597
598
599
600
601
602struct nft_set_ext_type {
603 u8 len;
604 u8 align;
605};
606
607extern const struct nft_set_ext_type nft_set_ext_types[];
608
609
610
611
612
613
614
615struct nft_set_ext_tmpl {
616 u16 len;
617 u8 offset[NFT_SET_EXT_NUM];
618};
619
620
621
622
623
624
625
626
627struct nft_set_ext {
628 u8 genmask;
629 u8 offset[NFT_SET_EXT_NUM];
630 char data[];
631};
632
633static inline void nft_set_ext_prepare(struct nft_set_ext_tmpl *tmpl)
634{
635 memset(tmpl, 0, sizeof(*tmpl));
636 tmpl->len = sizeof(struct nft_set_ext);
637}
638
639static inline void nft_set_ext_add_length(struct nft_set_ext_tmpl *tmpl, u8 id,
640 unsigned int len)
641{
642 tmpl->len = ALIGN(tmpl->len, nft_set_ext_types[id].align);
643 BUG_ON(tmpl->len > U8_MAX);
644 tmpl->offset[id] = tmpl->len;
645 tmpl->len += nft_set_ext_types[id].len + len;
646}
647
648static inline void nft_set_ext_add(struct nft_set_ext_tmpl *tmpl, u8 id)
649{
650 nft_set_ext_add_length(tmpl, id, 0);
651}
652
653static inline void nft_set_ext_init(struct nft_set_ext *ext,
654 const struct nft_set_ext_tmpl *tmpl)
655{
656 memcpy(ext->offset, tmpl->offset, sizeof(ext->offset));
657}
658
659static inline bool __nft_set_ext_exists(const struct nft_set_ext *ext, u8 id)
660{
661 return !!ext->offset[id];
662}
663
664static inline bool nft_set_ext_exists(const struct nft_set_ext *ext, u8 id)
665{
666 return ext && __nft_set_ext_exists(ext, id);
667}
668
669static inline void *nft_set_ext(const struct nft_set_ext *ext, u8 id)
670{
671 return (void *)ext + ext->offset[id];
672}
673
674static inline struct nft_data *nft_set_ext_key(const struct nft_set_ext *ext)
675{
676 return nft_set_ext(ext, NFT_SET_EXT_KEY);
677}
678
679static inline struct nft_data *nft_set_ext_key_end(const struct nft_set_ext *ext)
680{
681 return nft_set_ext(ext, NFT_SET_EXT_KEY_END);
682}
683
684static inline struct nft_data *nft_set_ext_data(const struct nft_set_ext *ext)
685{
686 return nft_set_ext(ext, NFT_SET_EXT_DATA);
687}
688
689static inline u8 *nft_set_ext_flags(const struct nft_set_ext *ext)
690{
691 return nft_set_ext(ext, NFT_SET_EXT_FLAGS);
692}
693
694static inline u64 *nft_set_ext_timeout(const struct nft_set_ext *ext)
695{
696 return nft_set_ext(ext, NFT_SET_EXT_TIMEOUT);
697}
698
699static inline u64 *nft_set_ext_expiration(const struct nft_set_ext *ext)
700{
701 return nft_set_ext(ext, NFT_SET_EXT_EXPIRATION);
702}
703
704static inline struct nft_userdata *nft_set_ext_userdata(const struct nft_set_ext *ext)
705{
706 return nft_set_ext(ext, NFT_SET_EXT_USERDATA);
707}
708
709static inline struct nft_set_elem_expr *nft_set_ext_expr(const struct nft_set_ext *ext)
710{
711 return nft_set_ext(ext, NFT_SET_EXT_EXPRESSIONS);
712}
713
714static inline bool nft_set_elem_expired(const struct nft_set_ext *ext)
715{
716 return nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION) &&
717 time_is_before_eq_jiffies64(*nft_set_ext_expiration(ext));
718}
719
720static inline struct nft_set_ext *nft_set_elem_ext(const struct nft_set *set,
721 void *elem)
722{
723 return elem + set->ops->elemsize;
724}
725
726static inline struct nft_object **nft_set_ext_obj(const struct nft_set_ext *ext)
727{
728 return nft_set_ext(ext, NFT_SET_EXT_OBJREF);
729}
730
731struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
732 const struct nft_set *set,
733 const struct nlattr *attr);
734
735void *nft_set_elem_init(const struct nft_set *set,
736 const struct nft_set_ext_tmpl *tmpl,
737 const u32 *key, const u32 *key_end, const u32 *data,
738 u64 timeout, u64 expiration, gfp_t gfp);
739int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
740 struct nft_expr *expr_array[]);
741void nft_set_elem_destroy(const struct nft_set *set, void *elem,
742 bool destroy_expr);
743
744
745
746
747
748
749
750
751struct nft_set_gc_batch_head {
752 struct rcu_head rcu;
753 const struct nft_set *set;
754 unsigned int cnt;
755};
756
757#define NFT_SET_GC_BATCH_SIZE ((PAGE_SIZE - \
758 sizeof(struct nft_set_gc_batch_head)) / \
759 sizeof(void *))
760
761
762
763
764
765
766
767struct nft_set_gc_batch {
768 struct nft_set_gc_batch_head head;
769 void *elems[NFT_SET_GC_BATCH_SIZE];
770};
771
772struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
773 gfp_t gfp);
774void nft_set_gc_batch_release(struct rcu_head *rcu);
775
776static inline void nft_set_gc_batch_complete(struct nft_set_gc_batch *gcb)
777{
778 if (gcb != NULL)
779 call_rcu(&gcb->head.rcu, nft_set_gc_batch_release);
780}
781
782static inline struct nft_set_gc_batch *
783nft_set_gc_batch_check(const struct nft_set *set, struct nft_set_gc_batch *gcb,
784 gfp_t gfp)
785{
786 if (gcb != NULL) {
787 if (gcb->head.cnt + 1 < ARRAY_SIZE(gcb->elems))
788 return gcb;
789 nft_set_gc_batch_complete(gcb);
790 }
791 return nft_set_gc_batch_alloc(set, gfp);
792}
793
794static inline void nft_set_gc_batch_add(struct nft_set_gc_batch *gcb,
795 void *elem)
796{
797 gcb->elems[gcb->head.cnt++] = elem;
798}
799
800struct nft_expr_ops;
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815struct nft_expr_type {
816 const struct nft_expr_ops *(*select_ops)(const struct nft_ctx *,
817 const struct nlattr * const tb[]);
818 void (*release_ops)(const struct nft_expr_ops *ops);
819 const struct nft_expr_ops *ops;
820 struct list_head list;
821 const char *name;
822 struct module *owner;
823 const struct nla_policy *policy;
824 unsigned int maxattr;
825 u8 family;
826 u8 flags;
827};
828
829#define NFT_EXPR_STATEFUL 0x1
830#define NFT_EXPR_GC 0x2
831
832enum nft_trans_phase {
833 NFT_TRANS_PREPARE,
834 NFT_TRANS_ABORT,
835 NFT_TRANS_COMMIT,
836 NFT_TRANS_RELEASE
837};
838
839struct nft_flow_rule;
840struct nft_offload_ctx;
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856struct nft_expr_ops {
857 void (*eval)(const struct nft_expr *expr,
858 struct nft_regs *regs,
859 const struct nft_pktinfo *pkt);
860 int (*clone)(struct nft_expr *dst,
861 const struct nft_expr *src);
862 unsigned int size;
863
864 int (*init)(const struct nft_ctx *ctx,
865 const struct nft_expr *expr,
866 const struct nlattr * const tb[]);
867 void (*activate)(const struct nft_ctx *ctx,
868 const struct nft_expr *expr);
869 void (*deactivate)(const struct nft_ctx *ctx,
870 const struct nft_expr *expr,
871 enum nft_trans_phase phase);
872 void (*destroy)(const struct nft_ctx *ctx,
873 const struct nft_expr *expr);
874 void (*destroy_clone)(const struct nft_ctx *ctx,
875 const struct nft_expr *expr);
876 int (*dump)(struct sk_buff *skb,
877 const struct nft_expr *expr);
878 int (*validate)(const struct nft_ctx *ctx,
879 const struct nft_expr *expr,
880 const struct nft_data **data);
881 bool (*gc)(struct net *net,
882 const struct nft_expr *expr);
883 int (*offload)(struct nft_offload_ctx *ctx,
884 struct nft_flow_rule *flow,
885 const struct nft_expr *expr);
886 void (*offload_stats)(struct nft_expr *expr,
887 const struct flow_stats *stats);
888 u32 offload_flags;
889 const struct nft_expr_type *type;
890 void *data;
891};
892
893
894
895
896
897
898
899
900
901
902
903struct nft_rule {
904 struct list_head list;
905 u64 handle:42,
906 genmask:2,
907 dlen:12,
908 udata:1;
909 unsigned char data[]
910 __attribute__((aligned(__alignof__(struct nft_expr))));
911};
912
913static inline struct nft_expr *nft_expr_first(const struct nft_rule *rule)
914{
915 return (struct nft_expr *)&rule->data[0];
916}
917
918static inline struct nft_expr *nft_expr_next(const struct nft_expr *expr)
919{
920 return ((void *)expr) + expr->ops->size;
921}
922
923static inline struct nft_expr *nft_expr_last(const struct nft_rule *rule)
924{
925 return (struct nft_expr *)&rule->data[rule->dlen];
926}
927
928static inline bool nft_expr_more(const struct nft_rule *rule,
929 const struct nft_expr *expr)
930{
931 return expr != nft_expr_last(rule) && expr->ops;
932}
933
934static inline struct nft_userdata *nft_userdata(const struct nft_rule *rule)
935{
936 return (void *)&rule->data[rule->dlen];
937}
938
939void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule);
940
941static inline void nft_set_elem_update_expr(const struct nft_set_ext *ext,
942 struct nft_regs *regs,
943 const struct nft_pktinfo *pkt)
944{
945 struct nft_set_elem_expr *elem_expr;
946 struct nft_expr *expr;
947 u32 size;
948
949 if (__nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS)) {
950 elem_expr = nft_set_ext_expr(ext);
951 nft_setelem_expr_foreach(expr, elem_expr, size) {
952 expr->ops->eval(expr, regs, pkt);
953 if (regs->verdict.code == NFT_BREAK)
954 return;
955 }
956 }
957}
958
959
960
961
962
963
964#define nft_rule_for_each_expr(expr, last, rule) \
965 for ((expr) = nft_expr_first(rule), (last) = nft_expr_last(rule); \
966 (expr) != (last); \
967 (expr) = nft_expr_next(expr))
968
969#define NFT_CHAIN_POLICY_UNSET U8_MAX
970
971
972
973
974
975
976
977
978
979
980
981
982
983struct nft_chain {
984 struct nft_rule *__rcu *rules_gen_0;
985 struct nft_rule *__rcu *rules_gen_1;
986 struct list_head rules;
987 struct list_head list;
988 struct rhlist_head rhlhead;
989 struct nft_table *table;
990 u64 handle;
991 u32 use;
992 u8 flags:5,
993 bound:1,
994 genmask:2;
995 char *name;
996 u16 udlen;
997 u8 *udata;
998
999
1000 struct nft_rule **rules_next;
1001};
1002
1003int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain);
1004
1005enum nft_chain_types {
1006 NFT_CHAIN_T_DEFAULT = 0,
1007 NFT_CHAIN_T_ROUTE,
1008 NFT_CHAIN_T_NAT,
1009 NFT_CHAIN_T_MAX
1010};
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024struct nft_chain_type {
1025 const char *name;
1026 enum nft_chain_types type;
1027 int family;
1028 struct module *owner;
1029 unsigned int hook_mask;
1030 nf_hookfn *hooks[NFT_MAX_HOOKS];
1031 int (*ops_register)(struct net *net, const struct nf_hook_ops *ops);
1032 void (*ops_unregister)(struct net *net, const struct nf_hook_ops *ops);
1033};
1034
1035int nft_chain_validate_dependency(const struct nft_chain *chain,
1036 enum nft_chain_types type);
1037int nft_chain_validate_hooks(const struct nft_chain *chain,
1038 unsigned int hook_flags);
1039
1040static inline bool nft_chain_is_bound(struct nft_chain *chain)
1041{
1042 return (chain->flags & NFT_CHAIN_BINDING) && chain->bound;
1043}
1044
1045void nft_chain_del(struct nft_chain *chain);
1046void nf_tables_chain_destroy(struct nft_ctx *ctx);
1047
1048struct nft_stats {
1049 u64 bytes;
1050 u64 pkts;
1051 struct u64_stats_sync syncp;
1052};
1053
1054struct nft_hook {
1055 struct list_head list;
1056 bool inactive;
1057 struct nf_hook_ops ops;
1058 struct rcu_head rcu;
1059};
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072struct nft_base_chain {
1073 struct nf_hook_ops ops;
1074 struct list_head hook_list;
1075 const struct nft_chain_type *type;
1076 u8 policy;
1077 u8 flags;
1078 struct nft_stats __percpu *stats;
1079 struct nft_chain chain;
1080 struct flow_block flow_block;
1081};
1082
1083static inline struct nft_base_chain *nft_base_chain(const struct nft_chain *chain)
1084{
1085 return container_of(chain, struct nft_base_chain, chain);
1086}
1087
1088static inline bool nft_is_base_chain(const struct nft_chain *chain)
1089{
1090 return chain->flags & NFT_CHAIN_BASE;
1091}
1092
1093int __nft_release_basechain(struct nft_ctx *ctx);
1094
1095unsigned int nft_do_chain(struct nft_pktinfo *pkt, void *priv);
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114struct nft_table {
1115 struct list_head list;
1116 struct rhltable chains_ht;
1117 struct list_head chains;
1118 struct list_head sets;
1119 struct list_head objects;
1120 struct list_head flowtables;
1121 u64 hgenerator;
1122 u64 handle;
1123 u32 use;
1124 u16 family:6,
1125 flags:8,
1126 genmask:2;
1127 u32 nlpid;
1128 char *name;
1129 u16 udlen;
1130 u8 *udata;
1131};
1132
1133static inline bool nft_table_has_owner(const struct nft_table *table)
1134{
1135 return table->flags & NFT_TABLE_F_OWNER;
1136}
1137
1138static inline bool nft_base_chain_netdev(int family, u32 hooknum)
1139{
1140 return family == NFPROTO_NETDEV ||
1141 (family == NFPROTO_INET && hooknum == NF_INET_INGRESS);
1142}
1143
1144void nft_register_chain_type(const struct nft_chain_type *);
1145void nft_unregister_chain_type(const struct nft_chain_type *);
1146
1147int nft_register_expr(struct nft_expr_type *);
1148void nft_unregister_expr(struct nft_expr_type *);
1149
1150int nft_verdict_dump(struct sk_buff *skb, int type,
1151 const struct nft_verdict *v);
1152
1153
1154
1155
1156
1157
1158
1159struct nft_object_hash_key {
1160 const char *name;
1161 const struct nft_table *table;
1162};
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176struct nft_object {
1177 struct list_head list;
1178 struct rhlist_head rhlhead;
1179 struct nft_object_hash_key key;
1180 u32 genmask:2,
1181 use:30;
1182 u64 handle;
1183 u16 udlen;
1184 u8 *udata;
1185
1186 const struct nft_object_ops *ops ____cacheline_aligned;
1187 unsigned char data[]
1188 __attribute__((aligned(__alignof__(u64))));
1189};
1190
1191static inline void *nft_obj_data(const struct nft_object *obj)
1192{
1193 return (void *)obj->data;
1194}
1195
1196#define nft_expr_obj(expr) *((struct nft_object **)nft_expr_priv(expr))
1197
1198struct nft_object *nft_obj_lookup(const struct net *net,
1199 const struct nft_table *table,
1200 const struct nlattr *nla, u32 objtype,
1201 u8 genmask);
1202
1203void nft_obj_notify(struct net *net, const struct nft_table *table,
1204 struct nft_object *obj, u32 portid, u32 seq,
1205 int event, int family, int report, gfp_t gfp);
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218struct nft_object_type {
1219 const struct nft_object_ops *(*select_ops)(const struct nft_ctx *,
1220 const struct nlattr * const tb[]);
1221 const struct nft_object_ops *ops;
1222 struct list_head list;
1223 u32 type;
1224 unsigned int maxattr;
1225 struct module *owner;
1226 const struct nla_policy *policy;
1227};
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239struct nft_object_ops {
1240 void (*eval)(struct nft_object *obj,
1241 struct nft_regs *regs,
1242 const struct nft_pktinfo *pkt);
1243 unsigned int size;
1244 int (*init)(const struct nft_ctx *ctx,
1245 const struct nlattr *const tb[],
1246 struct nft_object *obj);
1247 void (*destroy)(const struct nft_ctx *ctx,
1248 struct nft_object *obj);
1249 int (*dump)(struct sk_buff *skb,
1250 struct nft_object *obj,
1251 bool reset);
1252 void (*update)(struct nft_object *obj,
1253 struct nft_object *newobj);
1254 const struct nft_object_type *type;
1255};
1256
1257int nft_register_obj(struct nft_object_type *obj_type);
1258void nft_unregister_obj(struct nft_object_type *obj_type);
1259
1260#define NFT_NETDEVICE_MAX 256
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277struct nft_flowtable {
1278 struct list_head list;
1279 struct nft_table *table;
1280 char *name;
1281 int hooknum;
1282 int ops_len;
1283 u32 genmask:2,
1284 use:30;
1285 u64 handle;
1286
1287 struct list_head hook_list ____cacheline_aligned;
1288 struct nf_flowtable data;
1289};
1290
1291struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
1292 const struct nlattr *nla,
1293 u8 genmask);
1294
1295void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
1296 struct nft_flowtable *flowtable,
1297 enum nft_trans_phase phase);
1298
1299void nft_register_flowtable_type(struct nf_flowtable_type *type);
1300void nft_unregister_flowtable_type(struct nf_flowtable_type *type);
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314struct nft_traceinfo {
1315 const struct nft_pktinfo *pkt;
1316 const struct nft_base_chain *basechain;
1317 const struct nft_chain *chain;
1318 const struct nft_rule *rule;
1319 const struct nft_verdict *verdict;
1320 enum nft_trace_types type;
1321 bool packet_dumped;
1322 bool trace;
1323};
1324
1325void nft_trace_init(struct nft_traceinfo *info, const struct nft_pktinfo *pkt,
1326 const struct nft_verdict *verdict,
1327 const struct nft_chain *basechain);
1328
1329void nft_trace_notify(struct nft_traceinfo *info);
1330
1331#define MODULE_ALIAS_NFT_CHAIN(family, name) \
1332 MODULE_ALIAS("nft-chain-" __stringify(family) "-" name)
1333
1334#define MODULE_ALIAS_NFT_AF_EXPR(family, name) \
1335 MODULE_ALIAS("nft-expr-" __stringify(family) "-" name)
1336
1337#define MODULE_ALIAS_NFT_EXPR(name) \
1338 MODULE_ALIAS("nft-expr-" name)
1339
1340#define MODULE_ALIAS_NFT_OBJ(type) \
1341 MODULE_ALIAS("nft-obj-" __stringify(type))
1342
1343#if IS_ENABLED(CONFIG_NF_TABLES)
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357static inline unsigned int nft_gencursor_next(const struct net *net)
1358{
1359 return net->nft.gencursor + 1 == 1 ? 1 : 0;
1360}
1361
1362static inline u8 nft_genmask_next(const struct net *net)
1363{
1364 return 1 << nft_gencursor_next(net);
1365}
1366
1367static inline u8 nft_genmask_cur(const struct net *net)
1368{
1369
1370 return 1 << READ_ONCE(net->nft.gencursor);
1371}
1372
1373#define NFT_GENMASK_ANY ((1 << 0) | (1 << 1))
1374
1375
1376
1377
1378
1379
1380#define nft_is_active(__net, __obj) \
1381 (((__obj)->genmask & nft_genmask_cur(__net)) == 0)
1382
1383
1384#define nft_is_active_next(__net, __obj) \
1385 (((__obj)->genmask & nft_genmask_next(__net)) == 0)
1386
1387
1388#define nft_activate_next(__net, __obj) \
1389 (__obj)->genmask = nft_genmask_cur(__net)
1390
1391
1392#define nft_deactivate_next(__net, __obj) \
1393 (__obj)->genmask = nft_genmask_next(__net)
1394
1395
1396#define nft_clear(__net, __obj) \
1397 (__obj)->genmask &= ~nft_genmask_next(__net)
1398#define nft_active_genmask(__obj, __genmask) \
1399 !((__obj)->genmask & __genmask)
1400
1401
1402
1403
1404
1405static inline bool nft_set_elem_active(const struct nft_set_ext *ext,
1406 u8 genmask)
1407{
1408 return !(ext->genmask & genmask);
1409}
1410
1411static inline void nft_set_elem_change_active(const struct net *net,
1412 const struct nft_set *set,
1413 struct nft_set_ext *ext)
1414{
1415 ext->genmask ^= nft_genmask_next(net);
1416}
1417
1418#endif
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430#define NFT_SET_ELEM_BUSY_MASK (1 << 2)
1431
1432#if defined(__LITTLE_ENDIAN_BITFIELD)
1433#define NFT_SET_ELEM_BUSY_BIT 2
1434#elif defined(__BIG_ENDIAN_BITFIELD)
1435#define NFT_SET_ELEM_BUSY_BIT (BITS_PER_LONG - BITS_PER_BYTE + 2)
1436#else
1437#error
1438#endif
1439
1440static inline int nft_set_elem_mark_busy(struct nft_set_ext *ext)
1441{
1442 unsigned long *word = (unsigned long *)ext;
1443
1444 BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);
1445 return test_and_set_bit(NFT_SET_ELEM_BUSY_BIT, word);
1446}
1447
1448static inline void nft_set_elem_clear_busy(struct nft_set_ext *ext)
1449{
1450 unsigned long *word = (unsigned long *)ext;
1451
1452 clear_bit(NFT_SET_ELEM_BUSY_BIT, word);
1453}
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464struct nft_trans {
1465 struct list_head list;
1466 int msg_type;
1467 bool put_net;
1468 struct nft_ctx ctx;
1469 char data[];
1470};
1471
1472struct nft_trans_rule {
1473 struct nft_rule *rule;
1474 struct nft_flow_rule *flow;
1475 u32 rule_id;
1476};
1477
1478#define nft_trans_rule(trans) \
1479 (((struct nft_trans_rule *)trans->data)->rule)
1480#define nft_trans_flow_rule(trans) \
1481 (((struct nft_trans_rule *)trans->data)->flow)
1482#define nft_trans_rule_id(trans) \
1483 (((struct nft_trans_rule *)trans->data)->rule_id)
1484
1485struct nft_trans_set {
1486 struct nft_set *set;
1487 u32 set_id;
1488 bool bound;
1489};
1490
1491#define nft_trans_set(trans) \
1492 (((struct nft_trans_set *)trans->data)->set)
1493#define nft_trans_set_id(trans) \
1494 (((struct nft_trans_set *)trans->data)->set_id)
1495#define nft_trans_set_bound(trans) \
1496 (((struct nft_trans_set *)trans->data)->bound)
1497
1498struct nft_trans_chain {
1499 bool update;
1500 char *name;
1501 struct nft_stats __percpu *stats;
1502 u8 policy;
1503 u32 chain_id;
1504};
1505
1506#define nft_trans_chain_update(trans) \
1507 (((struct nft_trans_chain *)trans->data)->update)
1508#define nft_trans_chain_name(trans) \
1509 (((struct nft_trans_chain *)trans->data)->name)
1510#define nft_trans_chain_stats(trans) \
1511 (((struct nft_trans_chain *)trans->data)->stats)
1512#define nft_trans_chain_policy(trans) \
1513 (((struct nft_trans_chain *)trans->data)->policy)
1514#define nft_trans_chain_id(trans) \
1515 (((struct nft_trans_chain *)trans->data)->chain_id)
1516
1517struct nft_trans_table {
1518 bool update;
1519};
1520
1521#define nft_trans_table_update(trans) \
1522 (((struct nft_trans_table *)trans->data)->update)
1523
1524struct nft_trans_elem {
1525 struct nft_set *set;
1526 struct nft_set_elem elem;
1527 bool bound;
1528};
1529
1530#define nft_trans_elem_set(trans) \
1531 (((struct nft_trans_elem *)trans->data)->set)
1532#define nft_trans_elem(trans) \
1533 (((struct nft_trans_elem *)trans->data)->elem)
1534#define nft_trans_elem_set_bound(trans) \
1535 (((struct nft_trans_elem *)trans->data)->bound)
1536
1537struct nft_trans_obj {
1538 struct nft_object *obj;
1539 struct nft_object *newobj;
1540 bool update;
1541};
1542
1543#define nft_trans_obj(trans) \
1544 (((struct nft_trans_obj *)trans->data)->obj)
1545#define nft_trans_obj_newobj(trans) \
1546 (((struct nft_trans_obj *)trans->data)->newobj)
1547#define nft_trans_obj_update(trans) \
1548 (((struct nft_trans_obj *)trans->data)->update)
1549
1550struct nft_trans_flowtable {
1551 struct nft_flowtable *flowtable;
1552 bool update;
1553 struct list_head hook_list;
1554 u32 flags;
1555};
1556
1557#define nft_trans_flowtable(trans) \
1558 (((struct nft_trans_flowtable *)trans->data)->flowtable)
1559#define nft_trans_flowtable_update(trans) \
1560 (((struct nft_trans_flowtable *)trans->data)->update)
1561#define nft_trans_flowtable_hooks(trans) \
1562 (((struct nft_trans_flowtable *)trans->data)->hook_list)
1563#define nft_trans_flowtable_flags(trans) \
1564 (((struct nft_trans_flowtable *)trans->data)->flags)
1565
1566int __init nft_chain_filter_init(void);
1567void nft_chain_filter_fini(void);
1568
1569void __init nft_chain_route_init(void);
1570void nft_chain_route_fini(void);
1571
1572void nf_tables_trans_destroy_flush_work(void);
1573
1574int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result);
1575__be64 nf_jiffies64_to_msecs(u64 input);
1576
1577#ifdef CONFIG_MODULES
1578__printf(2, 3) int nft_request_module(struct net *net, const char *fmt, ...);
1579#else
1580static inline int nft_request_module(struct net *net, const char *fmt, ...) { return -ENOENT; }
1581#endif
1582
1583struct nftables_pernet {
1584 struct list_head tables;
1585 struct list_head commit_list;
1586 struct list_head module_list;
1587 struct list_head notify_list;
1588 struct mutex commit_mutex;
1589 unsigned int base_seq;
1590 u8 validate_state;
1591};
1592
1593extern unsigned int nf_tables_net_id;
1594
1595static inline struct nftables_pernet *nft_pernet(const struct net *net)
1596{
1597 return net_generic(net, nf_tables_net_id);
1598}
1599
1600#endif
1601