1
2
3
4
5
6
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/list.h>
12#include <linux/rculist.h>
13#include <linux/skbuff.h>
14#include <linux/netlink.h>
15#include <linux/netfilter.h>
16#include <linux/static_key.h>
17#include <linux/netfilter/nfnetlink.h>
18#include <linux/netfilter/nf_tables.h>
19#include <net/netfilter/nf_tables_core.h>
20#include <net/netfilter/nf_tables.h>
21#include <net/netfilter/nf_log.h>
22#include <net/netfilter/nft_meta.h>
23
24static noinline void __nft_trace_packet(struct nft_traceinfo *info,
25 const struct nft_chain *chain,
26 enum nft_trace_types type)
27{
28 const struct nft_pktinfo *pkt = info->pkt;
29
30 if (!info->trace || !pkt->skb->nf_trace)
31 return;
32
33 info->chain = chain;
34 info->type = type;
35
36 nft_trace_notify(info);
37}
38
39static inline void nft_trace_packet(struct nft_traceinfo *info,
40 const struct nft_chain *chain,
41 const struct nft_rule *rule,
42 enum nft_trace_types type)
43{
44 if (static_branch_unlikely(&nft_trace_enabled)) {
45 info->rule = rule;
46 __nft_trace_packet(info, chain, type);
47 }
48}
49
50static void nft_bitwise_fast_eval(const struct nft_expr *expr,
51 struct nft_regs *regs)
52{
53 const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
54 u32 *src = ®s->data[priv->sreg];
55 u32 *dst = ®s->data[priv->dreg];
56
57 *dst = (*src & priv->mask) ^ priv->xor;
58}
59
60static void nft_cmp_fast_eval(const struct nft_expr *expr,
61 struct nft_regs *regs)
62{
63 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
64
65 if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv)
66 return;
67 regs->verdict.code = NFT_BREAK;
68}
69
70static bool nft_payload_fast_eval(const struct nft_expr *expr,
71 struct nft_regs *regs,
72 const struct nft_pktinfo *pkt)
73{
74 const struct nft_payload *priv = nft_expr_priv(expr);
75 const struct sk_buff *skb = pkt->skb;
76 u32 *dest = ®s->data[priv->dreg];
77 unsigned char *ptr;
78
79 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
80 ptr = skb_network_header(skb);
81 else {
82 if (!pkt->tprot_set)
83 return false;
84 ptr = skb_network_header(skb) + nft_thoff(pkt);
85 }
86
87 ptr += priv->offset;
88
89 if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
90 return false;
91
92 *dest = 0;
93 if (priv->len == 2)
94 *(u16 *)dest = *(u16 *)ptr;
95 else if (priv->len == 4)
96 *(u32 *)dest = *(u32 *)ptr;
97 else
98 *(u8 *)dest = *(u8 *)ptr;
99 return true;
100}
101
102DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
103
104static noinline void nft_update_chain_stats(const struct nft_chain *chain,
105 const struct nft_pktinfo *pkt)
106{
107 struct nft_base_chain *base_chain;
108 struct nft_stats __percpu *pstats;
109 struct nft_stats *stats;
110
111 base_chain = nft_base_chain(chain);
112
113 rcu_read_lock();
114 pstats = READ_ONCE(base_chain->stats);
115 if (pstats) {
116 local_bh_disable();
117 stats = this_cpu_ptr(pstats);
118 u64_stats_update_begin(&stats->syncp);
119 stats->pkts++;
120 stats->bytes += pkt->skb->len;
121 u64_stats_update_end(&stats->syncp);
122 local_bh_enable();
123 }
124 rcu_read_unlock();
125}
126
127struct nft_jumpstack {
128 const struct nft_chain *chain;
129 struct nft_rule *const *rules;
130};
131
132static void expr_call_ops_eval(const struct nft_expr *expr,
133 struct nft_regs *regs,
134 struct nft_pktinfo *pkt)
135{
136#ifdef CONFIG_RETPOLINE
137 unsigned long e = (unsigned long)expr->ops->eval;
138#define X(e, fun) \
139 do { if ((e) == (unsigned long)(fun)) \
140 return fun(expr, regs, pkt); } while (0)
141
142 X(e, nft_payload_eval);
143 X(e, nft_cmp_eval);
144 X(e, nft_meta_get_eval);
145 X(e, nft_lookup_eval);
146 X(e, nft_range_eval);
147 X(e, nft_immediate_eval);
148 X(e, nft_byteorder_eval);
149 X(e, nft_dynset_eval);
150 X(e, nft_rt_get_eval);
151 X(e, nft_bitwise_eval);
152#undef X
153#endif
154 expr->ops->eval(expr, regs, pkt);
155}
156
157unsigned int
158nft_do_chain(struct nft_pktinfo *pkt, void *priv)
159{
160 const struct nft_chain *chain = priv, *basechain = chain;
161 const struct net *net = nft_net(pkt);
162 struct nft_rule *const *rules;
163 const struct nft_rule *rule;
164 const struct nft_expr *expr, *last;
165 struct nft_regs regs;
166 unsigned int stackptr = 0;
167 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
168 bool genbit = READ_ONCE(net->nft.gencursor);
169 struct nft_traceinfo info;
170
171 info.trace = false;
172 if (static_branch_unlikely(&nft_trace_enabled))
173 nft_trace_init(&info, pkt, ®s.verdict, basechain);
174do_chain:
175 if (genbit)
176 rules = rcu_dereference(chain->rules_gen_1);
177 else
178 rules = rcu_dereference(chain->rules_gen_0);
179
180next_rule:
181 rule = *rules;
182 regs.verdict.code = NFT_CONTINUE;
183 for (; *rules ; rules++) {
184 rule = *rules;
185 nft_rule_for_each_expr(expr, last, rule) {
186 if (expr->ops == &nft_cmp_fast_ops)
187 nft_cmp_fast_eval(expr, ®s);
188 else if (expr->ops == &nft_bitwise_fast_ops)
189 nft_bitwise_fast_eval(expr, ®s);
190 else if (expr->ops != &nft_payload_fast_ops ||
191 !nft_payload_fast_eval(expr, ®s, pkt))
192 expr_call_ops_eval(expr, ®s, pkt);
193
194 if (regs.verdict.code != NFT_CONTINUE)
195 break;
196 }
197
198 switch (regs.verdict.code) {
199 case NFT_BREAK:
200 regs.verdict.code = NFT_CONTINUE;
201 continue;
202 case NFT_CONTINUE:
203 nft_trace_packet(&info, chain, rule,
204 NFT_TRACETYPE_RULE);
205 continue;
206 }
207 break;
208 }
209
210 switch (regs.verdict.code & NF_VERDICT_MASK) {
211 case NF_ACCEPT:
212 case NF_DROP:
213 case NF_QUEUE:
214 case NF_STOLEN:
215 nft_trace_packet(&info, chain, rule,
216 NFT_TRACETYPE_RULE);
217 return regs.verdict.code;
218 }
219
220 switch (regs.verdict.code) {
221 case NFT_JUMP:
222 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
223 return NF_DROP;
224 jumpstack[stackptr].chain = chain;
225 jumpstack[stackptr].rules = rules + 1;
226 stackptr++;
227 fallthrough;
228 case NFT_GOTO:
229 nft_trace_packet(&info, chain, rule,
230 NFT_TRACETYPE_RULE);
231
232 chain = regs.verdict.chain;
233 goto do_chain;
234 case NFT_CONTINUE:
235 case NFT_RETURN:
236 nft_trace_packet(&info, chain, rule,
237 NFT_TRACETYPE_RETURN);
238 break;
239 default:
240 WARN_ON(1);
241 }
242
243 if (stackptr > 0) {
244 stackptr--;
245 chain = jumpstack[stackptr].chain;
246 rules = jumpstack[stackptr].rules;
247 goto next_rule;
248 }
249
250 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY);
251
252 if (static_branch_unlikely(&nft_counters_enabled))
253 nft_update_chain_stats(basechain, pkt);
254
255 return nft_base_chain(basechain)->policy;
256}
257EXPORT_SYMBOL_GPL(nft_do_chain);
258
259static struct nft_expr_type *nft_basic_types[] = {
260 &nft_imm_type,
261 &nft_cmp_type,
262 &nft_lookup_type,
263 &nft_bitwise_type,
264 &nft_byteorder_type,
265 &nft_payload_type,
266 &nft_dynset_type,
267 &nft_range_type,
268 &nft_meta_type,
269 &nft_rt_type,
270 &nft_exthdr_type,
271 &nft_last_type,
272};
273
274static struct nft_object_type *nft_basic_objects[] = {
275#ifdef CONFIG_NETWORK_SECMARK
276 &nft_secmark_obj_type,
277#endif
278};
279
280int __init nf_tables_core_module_init(void)
281{
282 int err, i, j = 0;
283
284 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
285 err = nft_register_obj(nft_basic_objects[i]);
286 if (err)
287 goto err;
288 }
289
290 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
291 err = nft_register_expr(nft_basic_types[j]);
292 if (err)
293 goto err;
294 }
295
296 return 0;
297
298err:
299 while (j-- > 0)
300 nft_unregister_expr(nft_basic_types[j]);
301
302 while (i-- > 0)
303 nft_unregister_obj(nft_basic_objects[i]);
304
305 return err;
306}
307
308void nf_tables_core_module_exit(void)
309{
310 int i;
311
312 i = ARRAY_SIZE(nft_basic_types);
313 while (i-- > 0)
314 nft_unregister_expr(nft_basic_types[i]);
315
316 i = ARRAY_SIZE(nft_basic_objects);
317 while (i-- > 0)
318 nft_unregister_obj(nft_basic_objects[i]);
319}
320
