1
2
3
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/netfilter.h>
12#include <net/protocol.h>
13#include <linux/init.h>
14#include <linux/skbuff.h>
15#include <linux/wait.h>
16#include <linux/module.h>
17#include <linux/interrupt.h>
18#include <linux/if.h>
19#include <linux/netdevice.h>
20#include <linux/netfilter_ipv6.h>
21#include <linux/inetdevice.h>
22#include <linux/proc_fs.h>
23#include <linux/mutex.h>
24#include <linux/slab.h>
25#include <linux/rcupdate.h>
26#include <net/net_namespace.h>
27#include <net/sock.h>
28
29#include "nf_internals.h"
30
31static DEFINE_MUTEX(afinfo_mutex);
32
33const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly;
34EXPORT_SYMBOL(nf_afinfo);
35const struct nf_ipv6_ops __rcu *nf_ipv6_ops __read_mostly;
36EXPORT_SYMBOL_GPL(nf_ipv6_ops);
37
38DEFINE_PER_CPU(bool, nf_skb_duplicated);
39EXPORT_SYMBOL_GPL(nf_skb_duplicated);
40
41int nf_register_afinfo(const struct nf_afinfo *afinfo)
42{
43 mutex_lock(&afinfo_mutex);
44 RCU_INIT_POINTER(nf_afinfo[afinfo->family], afinfo);
45 mutex_unlock(&afinfo_mutex);
46 return 0;
47}
48EXPORT_SYMBOL_GPL(nf_register_afinfo);
49
50void nf_unregister_afinfo(const struct nf_afinfo *afinfo)
51{
52 mutex_lock(&afinfo_mutex);
53 RCU_INIT_POINTER(nf_afinfo[afinfo->family], NULL);
54 mutex_unlock(&afinfo_mutex);
55 synchronize_rcu();
56}
57EXPORT_SYMBOL_GPL(nf_unregister_afinfo);
58
59#ifdef HAVE_JUMP_LABEL
60struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
61EXPORT_SYMBOL(nf_hooks_needed);
62#endif
63
64static DEFINE_MUTEX(nf_hook_mutex);
65#define nf_entry_dereference(e) \
66 rcu_dereference_protected(e, lockdep_is_held(&nf_hook_mutex))
67
68static struct nf_hook_entry __rcu **nf_hook_entry_head(struct net *net, const struct nf_hook_ops *reg)
69{
70 if (reg->pf != NFPROTO_NETDEV)
71 return net->nf.hooks[reg->pf]+reg->hooknum;
72
73#ifdef CONFIG_NETFILTER_INGRESS
74 if (reg->hooknum == NF_NETDEV_INGRESS) {
75 if (reg->dev && dev_net(reg->dev) == net)
76 return ®->dev->nf_hooks_ingress;
77 }
78#endif
79 return NULL;
80}
81
82int nf_register_net_hook(struct net *net, const struct nf_hook_ops *reg)
83{
84 struct nf_hook_entry __rcu **pp;
85 struct nf_hook_entry *entry, *p;
86
87 if (reg->pf == NFPROTO_NETDEV) {
88#ifndef CONFIG_NETFILTER_INGRESS
89 if (reg->hooknum == NF_NETDEV_INGRESS)
90 return -EOPNOTSUPP;
91#endif
92 if (reg->hooknum != NF_NETDEV_INGRESS ||
93 !reg->dev || dev_net(reg->dev) != net)
94 return -EINVAL;
95 }
96
97 pp = nf_hook_entry_head(net, reg);
98 if (!pp)
99 return -EINVAL;
100
101 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
102 if (!entry)
103 return -ENOMEM;
104
105 entry->orig_ops = reg;
106 entry->ops = *reg;
107 entry->next = NULL;
108
109 mutex_lock(&nf_hook_mutex);
110
111
112 while ((p = nf_entry_dereference(*pp)) != NULL) {
113 if (reg->priority < p->orig_ops->priority)
114 break;
115 pp = &p->next;
116 }
117 rcu_assign_pointer(entry->next, p);
118 rcu_assign_pointer(*pp, entry);
119
120 mutex_unlock(&nf_hook_mutex);
121#ifdef CONFIG_NETFILTER_INGRESS
122 if (reg->pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS)
123 net_inc_ingress_queue();
124#endif
125#ifdef HAVE_JUMP_LABEL
126 static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
127#endif
128 return 0;
129}
130EXPORT_SYMBOL(nf_register_net_hook);
131
132void nf_unregister_net_hook(struct net *net, const struct nf_hook_ops *reg)
133{
134 struct nf_hook_entry __rcu **pp;
135 struct nf_hook_entry *p;
136
137 pp = nf_hook_entry_head(net, reg);
138 if (WARN_ON_ONCE(!pp))
139 return;
140
141 mutex_lock(&nf_hook_mutex);
142 while ((p = nf_entry_dereference(*pp)) != NULL) {
143 if (p->orig_ops == reg) {
144 rcu_assign_pointer(*pp, p->next);
145 break;
146 }
147 pp = &p->next;
148 }
149 mutex_unlock(&nf_hook_mutex);
150 if (!p) {
151 WARN(1, "nf_unregister_net_hook: hook not found!\n");
152 return;
153 }
154#ifdef CONFIG_NETFILTER_INGRESS
155 if (reg->pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS)
156 net_dec_ingress_queue();
157#endif
158#ifdef HAVE_JUMP_LABEL
159 static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
160#endif
161 synchronize_net();
162 nf_queue_nf_hook_drop(net, p);
163
164 synchronize_net();
165 kfree(p);
166}
167EXPORT_SYMBOL(nf_unregister_net_hook);
168
169int nf_register_net_hooks(struct net *net, const struct nf_hook_ops *reg,
170 unsigned int n)
171{
172 unsigned int i;
173 int err = 0;
174
175 for (i = 0; i < n; i++) {
176 err = nf_register_net_hook(net, ®[i]);
177 if (err)
178 goto err;
179 }
180 return err;
181
182err:
183 if (i > 0)
184 nf_unregister_net_hooks(net, reg, i);
185 return err;
186}
187EXPORT_SYMBOL(nf_register_net_hooks);
188
189void nf_unregister_net_hooks(struct net *net, const struct nf_hook_ops *reg,
190 unsigned int n)
191{
192 while (n-- > 0)
193 nf_unregister_net_hook(net, ®[n]);
194}
195EXPORT_SYMBOL(nf_unregister_net_hooks);
196
197static LIST_HEAD(nf_hook_list);
198
199static int _nf_register_hook(struct nf_hook_ops *reg)
200{
201 struct net *net, *last;
202 int ret;
203
204 for_each_net(net) {
205 ret = nf_register_net_hook(net, reg);
206 if (ret && ret != -ENOENT)
207 goto rollback;
208 }
209 list_add_tail(®->list, &nf_hook_list);
210
211 return 0;
212rollback:
213 last = net;
214 for_each_net(net) {
215 if (net == last)
216 break;
217 nf_unregister_net_hook(net, reg);
218 }
219 return ret;
220}
221
222int nf_register_hook(struct nf_hook_ops *reg)
223{
224 int ret;
225
226 rtnl_lock();
227 ret = _nf_register_hook(reg);
228 rtnl_unlock();
229
230 return ret;
231}
232EXPORT_SYMBOL(nf_register_hook);
233
234static void _nf_unregister_hook(struct nf_hook_ops *reg)
235{
236 struct net *net;
237
238 list_del(®->list);
239 for_each_net(net)
240 nf_unregister_net_hook(net, reg);
241}
242
243void nf_unregister_hook(struct nf_hook_ops *reg)
244{
245 rtnl_lock();
246 _nf_unregister_hook(reg);
247 rtnl_unlock();
248}
249EXPORT_SYMBOL(nf_unregister_hook);
250
251int nf_register_hooks(struct nf_hook_ops *reg, unsigned int n)
252{
253 unsigned int i;
254 int err = 0;
255
256 for (i = 0; i < n; i++) {
257 err = nf_register_hook(®[i]);
258 if (err)
259 goto err;
260 }
261 return err;
262
263err:
264 if (i > 0)
265 nf_unregister_hooks(reg, i);
266 return err;
267}
268EXPORT_SYMBOL(nf_register_hooks);
269
270
271int _nf_register_hooks(struct nf_hook_ops *reg, unsigned int n)
272{
273 unsigned int i;
274 int err = 0;
275
276 for (i = 0; i < n; i++) {
277 err = _nf_register_hook(®[i]);
278 if (err)
279 goto err;
280 }
281 return err;
282
283err:
284 if (i > 0)
285 _nf_unregister_hooks(reg, i);
286 return err;
287}
288EXPORT_SYMBOL(_nf_register_hooks);
289
290void nf_unregister_hooks(struct nf_hook_ops *reg, unsigned int n)
291{
292 while (n-- > 0)
293 nf_unregister_hook(®[n]);
294}
295EXPORT_SYMBOL(nf_unregister_hooks);
296
297
298void _nf_unregister_hooks(struct nf_hook_ops *reg, unsigned int n)
299{
300 while (n-- > 0)
301 _nf_unregister_hook(®[n]);
302}
303EXPORT_SYMBOL(_nf_unregister_hooks);
304
305unsigned int nf_iterate(struct sk_buff *skb,
306 struct nf_hook_state *state,
307 struct nf_hook_entry **entryp)
308{
309 unsigned int verdict;
310
311
312
313
314
315 while (*entryp) {
316 if (state->thresh > (*entryp)->ops.priority) {
317 *entryp = rcu_dereference((*entryp)->next);
318 continue;
319 }
320
321
322
323repeat:
324 verdict = (*entryp)->ops.hook((*entryp)->ops.priv, skb, state);
325 if (verdict != NF_ACCEPT) {
326#ifdef CONFIG_NETFILTER_DEBUG
327 if (unlikely((verdict & NF_VERDICT_MASK)
328 > NF_MAX_VERDICT)) {
329 NFDEBUG("Evil return from %p(%u).\n",
330 (*entryp)->ops.hook, state->hook);
331 *entryp = rcu_dereference((*entryp)->next);
332 continue;
333 }
334#endif
335 if (verdict != NF_REPEAT)
336 return verdict;
337 goto repeat;
338 }
339 *entryp = rcu_dereference((*entryp)->next);
340 }
341 return NF_ACCEPT;
342}
343
344
345
346
347int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state)
348{
349 struct nf_hook_entry *entry;
350 unsigned int verdict;
351 int ret = 0;
352
353 entry = rcu_dereference(state->hook_entries);
354next_hook:
355 verdict = nf_iterate(skb, state, &entry);
356 if (verdict == NF_ACCEPT || verdict == NF_STOP) {
357 ret = 1;
358 } else if ((verdict & NF_VERDICT_MASK) == NF_DROP) {
359 kfree_skb(skb);
360 ret = NF_DROP_GETERR(verdict);
361 if (ret == 0)
362 ret = -EPERM;
363 } else if ((verdict & NF_VERDICT_MASK) == NF_QUEUE) {
364 ret = nf_queue(skb, state, &entry, verdict);
365 if (ret == 1 && entry)
366 goto next_hook;
367 }
368 return ret;
369}
370EXPORT_SYMBOL(nf_hook_slow);
371
372
373int skb_make_writable(struct sk_buff *skb, unsigned int writable_len)
374{
375 if (writable_len > skb->len)
376 return 0;
377
378
379 if (!skb_cloned(skb)) {
380 if (writable_len <= skb_headlen(skb))
381 return 1;
382 } else if (skb_clone_writable(skb, writable_len))
383 return 1;
384
385 if (writable_len <= skb_headlen(skb))
386 writable_len = 0;
387 else
388 writable_len -= skb_headlen(skb);
389
390 return !!__pskb_pull_tail(skb, writable_len);
391}
392EXPORT_SYMBOL(skb_make_writable);
393
394
395
396
397struct nfnl_ct_hook __rcu *nfnl_ct_hook __read_mostly;
398EXPORT_SYMBOL_GPL(nfnl_ct_hook);
399
400#if IS_ENABLED(CONFIG_NF_CONNTRACK)
401
402
403
404void (*ip_ct_attach)(struct sk_buff *, const struct sk_buff *)
405 __rcu __read_mostly;
406EXPORT_SYMBOL(ip_ct_attach);
407
408void nf_ct_attach(struct sk_buff *new, const struct sk_buff *skb)
409{
410 void (*attach)(struct sk_buff *, const struct sk_buff *);
411
412 if (skb->nfct) {
413 rcu_read_lock();
414 attach = rcu_dereference(ip_ct_attach);
415 if (attach)
416 attach(new, skb);
417 rcu_read_unlock();
418 }
419}
420EXPORT_SYMBOL(nf_ct_attach);
421
422void (*nf_ct_destroy)(struct nf_conntrack *) __rcu __read_mostly;
423EXPORT_SYMBOL(nf_ct_destroy);
424
425void nf_conntrack_destroy(struct nf_conntrack *nfct)
426{
427 void (*destroy)(struct nf_conntrack *);
428
429 rcu_read_lock();
430 destroy = rcu_dereference(nf_ct_destroy);
431 BUG_ON(destroy == NULL);
432 destroy(nfct);
433 rcu_read_unlock();
434}
435EXPORT_SYMBOL(nf_conntrack_destroy);
436
437
438const struct nf_conntrack_zone nf_ct_zone_dflt = {
439 .id = NF_CT_DEFAULT_ZONE_ID,
440 .dir = NF_CT_DEFAULT_ZONE_DIR,
441};
442EXPORT_SYMBOL_GPL(nf_ct_zone_dflt);
443#endif
444
445#ifdef CONFIG_NF_NAT_NEEDED
446void (*nf_nat_decode_session_hook)(struct sk_buff *, struct flowi *);
447EXPORT_SYMBOL(nf_nat_decode_session_hook);
448#endif
449
450static int nf_register_hook_list(struct net *net)
451{
452 struct nf_hook_ops *elem;
453 int ret;
454
455 rtnl_lock();
456 list_for_each_entry(elem, &nf_hook_list, list) {
457 ret = nf_register_net_hook(net, elem);
458 if (ret && ret != -ENOENT)
459 goto out_undo;
460 }
461 rtnl_unlock();
462 return 0;
463
464out_undo:
465 list_for_each_entry_continue_reverse(elem, &nf_hook_list, list)
466 nf_unregister_net_hook(net, elem);
467 rtnl_unlock();
468 return ret;
469}
470
471static void nf_unregister_hook_list(struct net *net)
472{
473 struct nf_hook_ops *elem;
474
475 rtnl_lock();
476 list_for_each_entry(elem, &nf_hook_list, list)
477 nf_unregister_net_hook(net, elem);
478 rtnl_unlock();
479}
480
481static int __net_init netfilter_net_init(struct net *net)
482{
483 int i, h, ret;
484
485 for (i = 0; i < ARRAY_SIZE(net->nf.hooks); i++) {
486 for (h = 0; h < NF_MAX_HOOKS; h++)
487 RCU_INIT_POINTER(net->nf.hooks[i][h], NULL);
488 }
489
490#ifdef CONFIG_PROC_FS
491 net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter",
492 net->proc_net);
493 if (!net->nf.proc_netfilter) {
494 if (!net_eq(net, &init_net))
495 pr_err("cannot create netfilter proc entry");
496
497 return -ENOMEM;
498 }
499#endif
500 ret = nf_register_hook_list(net);
501 if (ret)
502 remove_proc_entry("netfilter", net->proc_net);
503
504 return ret;
505}
506
507static void __net_exit netfilter_net_exit(struct net *net)
508{
509 nf_unregister_hook_list(net);
510 remove_proc_entry("netfilter", net->proc_net);
511}
512
513static struct pernet_operations netfilter_net_ops = {
514 .init = netfilter_net_init,
515 .exit = netfilter_net_exit,
516};
517
518int __init netfilter_init(void)
519{
520 int ret;
521
522 ret = register_pernet_subsys(&netfilter_net_ops);
523 if (ret < 0)
524 goto err;
525
526 ret = netfilter_log_init();
527 if (ret < 0)
528 goto err_pernet;
529
530 return 0;
531err_pernet:
532 unregister_pernet_subsys(&netfilter_net_ops);
533err:
534 return ret;
535}
536