1
2
3
4
5
6
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/list.h>
12#include <linux/rculist.h>
13#include <linux/skbuff.h>
14#include <linux/netlink.h>
15#include <linux/netfilter.h>
16#include <linux/static_key.h>
17#include <linux/netfilter/nfnetlink.h>
18#include <linux/netfilter/nf_tables.h>
19#include <net/netfilter/nf_tables_core.h>
20#include <net/netfilter/nf_tables.h>
21#include <net/netfilter/nf_log.h>
22#include <net/netfilter/nft_meta.h>
23
24static noinline void __nft_trace_packet(struct nft_traceinfo *info,
25 const struct nft_chain *chain,
26 enum nft_trace_types type)
27{
28 const struct nft_pktinfo *pkt = info->pkt;
29
30 if (!info->trace || !pkt->skb->nf_trace)
31 return;
32
33 info->chain = chain;
34 info->type = type;
35
36 nft_trace_notify(info);
37}
38
39static inline void nft_trace_packet(struct nft_traceinfo *info,
40 const struct nft_chain *chain,
41 const struct nft_rule *rule,
42 enum nft_trace_types type)
43{
44 if (static_branch_unlikely(&nft_trace_enabled)) {
45 info->rule = rule;
46 __nft_trace_packet(info, chain, type);
47 }
48}
49
50static void nft_cmp_fast_eval(const struct nft_expr *expr,
51 struct nft_regs *regs)
52{
53 const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
54 u32 mask = nft_cmp_fast_mask(priv->len);
55
56 if ((regs->data[priv->sreg] & mask) == priv->data)
57 return;
58 regs->verdict.code = NFT_BREAK;
59}
60
61static bool nft_payload_fast_eval(const struct nft_expr *expr,
62 struct nft_regs *regs,
63 const struct nft_pktinfo *pkt)
64{
65 const struct nft_payload *priv = nft_expr_priv(expr);
66 const struct sk_buff *skb = pkt->skb;
67 u32 *dest = ®s->data[priv->dreg];
68 unsigned char *ptr;
69
70 if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
71 ptr = skb_network_header(skb);
72 else {
73 if (!pkt->tprot_set)
74 return false;
75 ptr = skb_network_header(skb) + pkt->xt.thoff;
76 }
77
78 ptr += priv->offset;
79
80 if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
81 return false;
82
83 *dest = 0;
84 if (priv->len == 2)
85 *(u16 *)dest = *(u16 *)ptr;
86 else if (priv->len == 4)
87 *(u32 *)dest = *(u32 *)ptr;
88 else
89 *(u8 *)dest = *(u8 *)ptr;
90 return true;
91}
92
93DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
94
95static noinline void nft_update_chain_stats(const struct nft_chain *chain,
96 const struct nft_pktinfo *pkt)
97{
98 struct nft_base_chain *base_chain;
99 struct nft_stats __percpu *pstats;
100 struct nft_stats *stats;
101
102 base_chain = nft_base_chain(chain);
103
104 rcu_read_lock();
105 pstats = READ_ONCE(base_chain->stats);
106 if (pstats) {
107 local_bh_disable();
108 stats = this_cpu_ptr(pstats);
109 u64_stats_update_begin(&stats->syncp);
110 stats->pkts++;
111 stats->bytes += pkt->skb->len;
112 u64_stats_update_end(&stats->syncp);
113 local_bh_enable();
114 }
115 rcu_read_unlock();
116}
117
118struct nft_jumpstack {
119 const struct nft_chain *chain;
120 struct nft_rule *const *rules;
121};
122
123static void expr_call_ops_eval(const struct nft_expr *expr,
124 struct nft_regs *regs,
125 struct nft_pktinfo *pkt)
126{
127#ifdef CONFIG_RETPOLINE
128 unsigned long e = (unsigned long)expr->ops->eval;
129#define X(e, fun) \
130 do { if ((e) == (unsigned long)(fun)) \
131 return fun(expr, regs, pkt); } while (0)
132
133 X(e, nft_payload_eval);
134 X(e, nft_cmp_eval);
135 X(e, nft_meta_get_eval);
136 X(e, nft_lookup_eval);
137 X(e, nft_range_eval);
138 X(e, nft_immediate_eval);
139 X(e, nft_byteorder_eval);
140 X(e, nft_dynset_eval);
141 X(e, nft_rt_get_eval);
142 X(e, nft_bitwise_eval);
143#undef X
144#endif
145 expr->ops->eval(expr, regs, pkt);
146}
147
148unsigned int
149nft_do_chain(struct nft_pktinfo *pkt, void *priv)
150{
151 const struct nft_chain *chain = priv, *basechain = chain;
152 const struct net *net = nft_net(pkt);
153 struct nft_rule *const *rules;
154 const struct nft_rule *rule;
155 const struct nft_expr *expr, *last;
156 struct nft_regs regs;
157 unsigned int stackptr = 0;
158 struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
159 bool genbit = READ_ONCE(net->nft.gencursor);
160 struct nft_traceinfo info;
161
162 info.trace = false;
163 if (static_branch_unlikely(&nft_trace_enabled))
164 nft_trace_init(&info, pkt, ®s.verdict, basechain);
165do_chain:
166 if (genbit)
167 rules = rcu_dereference(chain->rules_gen_1);
168 else
169 rules = rcu_dereference(chain->rules_gen_0);
170
171next_rule:
172 rule = *rules;
173 regs.verdict.code = NFT_CONTINUE;
174 for (; *rules ; rules++) {
175 rule = *rules;
176 nft_rule_for_each_expr(expr, last, rule) {
177 if (expr->ops == &nft_cmp_fast_ops)
178 nft_cmp_fast_eval(expr, ®s);
179 else if (expr->ops != &nft_payload_fast_ops ||
180 !nft_payload_fast_eval(expr, ®s, pkt))
181 expr_call_ops_eval(expr, ®s, pkt);
182
183 if (regs.verdict.code != NFT_CONTINUE)
184 break;
185 }
186
187 switch (regs.verdict.code) {
188 case NFT_BREAK:
189 regs.verdict.code = NFT_CONTINUE;
190 continue;
191 case NFT_CONTINUE:
192 nft_trace_packet(&info, chain, rule,
193 NFT_TRACETYPE_RULE);
194 continue;
195 }
196 break;
197 }
198
199 switch (regs.verdict.code & NF_VERDICT_MASK) {
200 case NF_ACCEPT:
201 case NF_DROP:
202 case NF_QUEUE:
203 case NF_STOLEN:
204 nft_trace_packet(&info, chain, rule,
205 NFT_TRACETYPE_RULE);
206 return regs.verdict.code;
207 }
208
209 switch (regs.verdict.code) {
210 case NFT_JUMP:
211 if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
212 return NF_DROP;
213 jumpstack[stackptr].chain = chain;
214 jumpstack[stackptr].rules = rules + 1;
215 stackptr++;
216
217 case NFT_GOTO:
218 nft_trace_packet(&info, chain, rule,
219 NFT_TRACETYPE_RULE);
220
221 chain = regs.verdict.chain;
222 goto do_chain;
223 case NFT_CONTINUE:
224 case NFT_RETURN:
225 nft_trace_packet(&info, chain, rule,
226 NFT_TRACETYPE_RETURN);
227 break;
228 default:
229 WARN_ON(1);
230 }
231
232 if (stackptr > 0) {
233 stackptr--;
234 chain = jumpstack[stackptr].chain;
235 rules = jumpstack[stackptr].rules;
236 goto next_rule;
237 }
238
239 nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY);
240
241 if (static_branch_unlikely(&nft_counters_enabled))
242 nft_update_chain_stats(basechain, pkt);
243
244 return nft_base_chain(basechain)->policy;
245}
246EXPORT_SYMBOL_GPL(nft_do_chain);
247
248static struct nft_expr_type *nft_basic_types[] = {
249 &nft_imm_type,
250 &nft_cmp_type,
251 &nft_lookup_type,
252 &nft_bitwise_type,
253 &nft_byteorder_type,
254 &nft_payload_type,
255 &nft_dynset_type,
256 &nft_range_type,
257 &nft_meta_type,
258 &nft_rt_type,
259 &nft_exthdr_type,
260};
261
262static struct nft_object_type *nft_basic_objects[] = {
263#ifdef CONFIG_NETWORK_SECMARK
264 &nft_secmark_obj_type,
265#endif
266};
267
268int __init nf_tables_core_module_init(void)
269{
270 int err, i, j = 0;
271
272 for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
273 err = nft_register_obj(nft_basic_objects[i]);
274 if (err)
275 goto err;
276 }
277
278 for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
279 err = nft_register_expr(nft_basic_types[j]);
280 if (err)
281 goto err;
282 }
283
284 return 0;
285
286err:
287 while (j-- > 0)
288 nft_unregister_expr(nft_basic_types[j]);
289
290 while (i-- > 0)
291 nft_unregister_obj(nft_basic_objects[i]);
292
293 return err;
294}
295
296void nf_tables_core_module_exit(void)
297{
298 int i;
299
300 i = ARRAY_SIZE(nft_basic_types);
301 while (i-- > 0)
302 nft_unregister_expr(nft_basic_types[i]);
303
304 i = ARRAY_SIZE(nft_basic_objects);
305 while (i-- > 0)
306 nft_unregister_obj(nft_basic_objects[i]);
307}
308
