1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/types.h>
16#include <linux/netfilter.h>
17#include <linux/module.h>
18#include <linux/sched.h>
19#include <linux/skbuff.h>
20#include <linux/proc_fs.h>
21#include <linux/vmalloc.h>
22#include <linux/stddef.h>
23#include <linux/slab.h>
24#include <linux/random.h>
25#include <linux/jhash.h>
26#include <linux/err.h>
27#include <linux/percpu.h>
28#include <linux/moduleparam.h>
29#include <linux/notifier.h>
30#include <linux/kernel.h>
31#include <linux/netdevice.h>
32#include <linux/socket.h>
33#include <linux/mm.h>
34#include <linux/nsproxy.h>
35#include <linux/rculist_nulls.h>
36
37#include <net/netfilter/nf_conntrack.h>
38#include <net/netfilter/nf_conntrack_l3proto.h>
39#include <net/netfilter/nf_conntrack_l4proto.h>
40#include <net/netfilter/nf_conntrack_expect.h>
41#include <net/netfilter/nf_conntrack_helper.h>
42#include <net/netfilter/nf_conntrack_seqadj.h>
43#include <net/netfilter/nf_conntrack_core.h>
44#include <net/netfilter/nf_conntrack_extend.h>
45#include <net/netfilter/nf_conntrack_acct.h>
46#include <net/netfilter/nf_conntrack_ecache.h>
47#include <net/netfilter/nf_conntrack_zones.h>
48#include <net/netfilter/nf_conntrack_timestamp.h>
49#include <net/netfilter/nf_conntrack_timeout.h>
50#include <net/netfilter/nf_conntrack_labels.h>
51#include <net/netfilter/nf_conntrack_synproxy.h>
52#include <net/netfilter/nf_nat.h>
53#include <net/netfilter/nf_nat_core.h>
54#include <net/netfilter/nf_nat_helper.h>
55
56#define NF_CONNTRACK_VERSION "0.5.0"
57
58int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
59 enum nf_nat_manip_type manip,
60 const struct nlattr *attr) __read_mostly;
61EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
62
63__cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
64EXPORT_SYMBOL_GPL(nf_conntrack_locks);
65
66__cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
67EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
68
69static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
70{
71 h1 %= CONNTRACK_LOCKS;
72 h2 %= CONNTRACK_LOCKS;
73 spin_unlock(&nf_conntrack_locks[h1]);
74 if (h1 != h2)
75 spin_unlock(&nf_conntrack_locks[h2]);
76}
77
78
79static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
80 unsigned int h2, unsigned int sequence)
81{
82 h1 %= CONNTRACK_LOCKS;
83 h2 %= CONNTRACK_LOCKS;
84 if (h1 <= h2) {
85 spin_lock(&nf_conntrack_locks[h1]);
86 if (h1 != h2)
87 spin_lock_nested(&nf_conntrack_locks[h2],
88 SINGLE_DEPTH_NESTING);
89 } else {
90 spin_lock(&nf_conntrack_locks[h2]);
91 spin_lock_nested(&nf_conntrack_locks[h1],
92 SINGLE_DEPTH_NESTING);
93 }
94 if (read_seqcount_retry(&net->ct.generation, sequence)) {
95 nf_conntrack_double_unlock(h1, h2);
96 return true;
97 }
98 return false;
99}
100
101static void nf_conntrack_all_lock(void)
102{
103 int i;
104
105 for (i = 0; i < CONNTRACK_LOCKS; i++)
106 spin_lock_nested(&nf_conntrack_locks[i], i);
107}
108
109static void nf_conntrack_all_unlock(void)
110{
111 int i;
112
113 for (i = 0; i < CONNTRACK_LOCKS; i++)
114 spin_unlock(&nf_conntrack_locks[i]);
115}
116
117unsigned int nf_conntrack_htable_size __read_mostly;
118EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
119
120unsigned int nf_conntrack_max __read_mostly;
121EXPORT_SYMBOL_GPL(nf_conntrack_max);
122
123DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
124EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
125
126unsigned int nf_conntrack_hash_rnd __read_mostly;
127EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
128
129static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
130{
131 unsigned int n;
132
133
134
135
136
137 n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
138 return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
139 (((__force __u16)tuple->dst.u.all << 16) |
140 tuple->dst.protonum));
141}
142
143static u32 __hash_bucket(u32 hash, unsigned int size)
144{
145 return reciprocal_scale(hash, size);
146}
147
148static u32 hash_bucket(u32 hash, const struct net *net)
149{
150 return __hash_bucket(hash, net->ct.htable_size);
151}
152
153static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
154 u16 zone, unsigned int size)
155{
156 return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
157}
158
159static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
160 const struct nf_conntrack_tuple *tuple)
161{
162 return __hash_conntrack(tuple, zone, net->ct.htable_size);
163}
164
165bool
166nf_ct_get_tuple(const struct sk_buff *skb,
167 unsigned int nhoff,
168 unsigned int dataoff,
169 u_int16_t l3num,
170 u_int8_t protonum,
171 struct nf_conntrack_tuple *tuple,
172 const struct nf_conntrack_l3proto *l3proto,
173 const struct nf_conntrack_l4proto *l4proto)
174{
175 memset(tuple, 0, sizeof(*tuple));
176
177 tuple->src.l3num = l3num;
178 if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
179 return false;
180
181 tuple->dst.protonum = protonum;
182 tuple->dst.dir = IP_CT_DIR_ORIGINAL;
183
184 return l4proto->pkt_to_tuple(skb, dataoff, tuple);
185}
186EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
187
188bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
189 u_int16_t l3num, struct nf_conntrack_tuple *tuple)
190{
191 struct nf_conntrack_l3proto *l3proto;
192 struct nf_conntrack_l4proto *l4proto;
193 unsigned int protoff;
194 u_int8_t protonum;
195 int ret;
196
197 rcu_read_lock();
198
199 l3proto = __nf_ct_l3proto_find(l3num);
200 ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
201 if (ret != NF_ACCEPT) {
202 rcu_read_unlock();
203 return false;
204 }
205
206 l4proto = __nf_ct_l4proto_find(l3num, protonum);
207
208 ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
209 l3proto, l4proto);
210
211 rcu_read_unlock();
212 return ret;
213}
214EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
215
216bool
217nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
218 const struct nf_conntrack_tuple *orig,
219 const struct nf_conntrack_l3proto *l3proto,
220 const struct nf_conntrack_l4proto *l4proto)
221{
222 memset(inverse, 0, sizeof(*inverse));
223
224 inverse->src.l3num = orig->src.l3num;
225 if (l3proto->invert_tuple(inverse, orig) == 0)
226 return false;
227
228 inverse->dst.dir = !orig->dst.dir;
229
230 inverse->dst.protonum = orig->dst.protonum;
231 return l4proto->invert_tuple(inverse, orig);
232}
233EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
234
235static void
236clean_from_lists(struct nf_conn *ct)
237{
238 pr_debug("clean_from_lists(%p)\n", ct);
239 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
240 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
241
242
243 nf_ct_remove_expectations(ct);
244}
245
246
247static void nf_ct_add_to_dying_list(struct nf_conn *ct)
248{
249 struct ct_pcpu *pcpu;
250
251
252 ct->cpu = smp_processor_id();
253 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
254
255 spin_lock(&pcpu->lock);
256 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
257 &pcpu->dying);
258 spin_unlock(&pcpu->lock);
259}
260
261
262static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
263{
264 struct ct_pcpu *pcpu;
265
266
267 ct->cpu = smp_processor_id();
268 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
269
270 spin_lock(&pcpu->lock);
271 hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
272 &pcpu->unconfirmed);
273 spin_unlock(&pcpu->lock);
274}
275
276
277static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
278{
279 struct ct_pcpu *pcpu;
280
281
282 pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
283
284 spin_lock(&pcpu->lock);
285 BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
286 hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
287 spin_unlock(&pcpu->lock);
288}
289
290static void
291destroy_conntrack(struct nf_conntrack *nfct)
292{
293 struct nf_conn *ct = (struct nf_conn *)nfct;
294 struct net *net = nf_ct_net(ct);
295 struct nf_conntrack_l4proto *l4proto;
296
297 pr_debug("destroy_conntrack(%p)\n", ct);
298 NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
299 NF_CT_ASSERT(!timer_pending(&ct->timeout));
300
301 rcu_read_lock();
302 l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
303 if (l4proto && l4proto->destroy)
304 l4proto->destroy(ct);
305
306 rcu_read_unlock();
307
308 local_bh_disable();
309
310
311
312
313
314 nf_ct_remove_expectations(ct);
315
316 nf_ct_del_from_dying_or_unconfirmed_list(ct);
317
318 NF_CT_STAT_INC(net, delete);
319 local_bh_enable();
320
321 if (ct->master)
322 nf_ct_put(ct->master);
323
324 pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
325 nf_conntrack_free(ct);
326}
327
328static void nf_ct_delete_from_lists(struct nf_conn *ct)
329{
330 struct net *net = nf_ct_net(ct);
331 unsigned int hash, reply_hash;
332 u16 zone = nf_ct_zone(ct);
333 unsigned int sequence;
334
335 nf_ct_helper_destroy(ct);
336
337 local_bh_disable();
338 do {
339 sequence = read_seqcount_begin(&net->ct.generation);
340 hash = hash_conntrack(net, zone,
341 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
342 reply_hash = hash_conntrack(net, zone,
343 &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
344 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
345
346 clean_from_lists(ct);
347 nf_conntrack_double_unlock(hash, reply_hash);
348
349 nf_ct_add_to_dying_list(ct);
350
351 NF_CT_STAT_INC(net, delete_list);
352 local_bh_enable();
353}
354
355bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
356{
357 struct nf_conn_tstamp *tstamp;
358
359 tstamp = nf_conn_tstamp_find(ct);
360 if (tstamp && tstamp->stop == 0)
361 tstamp->stop = ktime_get_real_ns();
362
363 if (nf_ct_is_dying(ct))
364 goto delete;
365
366 if (nf_conntrack_event_report(IPCT_DESTROY, ct,
367 portid, report) < 0) {
368
369 nf_ct_delete_from_lists(ct);
370 nf_conntrack_ecache_delayed_work(nf_ct_net(ct));
371 return false;
372 }
373
374 nf_conntrack_ecache_work(nf_ct_net(ct));
375 set_bit(IPS_DYING_BIT, &ct->status);
376 delete:
377 nf_ct_delete_from_lists(ct);
378 nf_ct_put(ct);
379 return true;
380}
381EXPORT_SYMBOL_GPL(nf_ct_delete);
382
383static void death_by_timeout(unsigned long ul_conntrack)
384{
385 nf_ct_delete((struct nf_conn *)ul_conntrack, 0, 0);
386}
387
388static inline bool
389nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
390 const struct nf_conntrack_tuple *tuple,
391 u16 zone)
392{
393 struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
394
395
396
397
398 return nf_ct_tuple_equal(tuple, &h->tuple) &&
399 nf_ct_zone(ct) == zone &&
400 nf_ct_is_confirmed(ct);
401}
402
403
404
405
406
407
408static struct nf_conntrack_tuple_hash *
409____nf_conntrack_find(struct net *net, u16 zone,
410 const struct nf_conntrack_tuple *tuple, u32 hash)
411{
412 struct nf_conntrack_tuple_hash *h;
413 struct hlist_nulls_node *n;
414 unsigned int bucket = hash_bucket(hash, net);
415
416
417
418
419 local_bh_disable();
420begin:
421 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
422 if (nf_ct_key_equal(h, tuple, zone)) {
423 NF_CT_STAT_INC(net, found);
424 local_bh_enable();
425 return h;
426 }
427 NF_CT_STAT_INC(net, searched);
428 }
429
430
431
432
433
434 if (get_nulls_value(n) != bucket) {
435 NF_CT_STAT_INC(net, search_restart);
436 goto begin;
437 }
438 local_bh_enable();
439
440 return NULL;
441}
442
443
444static struct nf_conntrack_tuple_hash *
445__nf_conntrack_find_get(struct net *net, u16 zone,
446 const struct nf_conntrack_tuple *tuple, u32 hash)
447{
448 struct nf_conntrack_tuple_hash *h;
449 struct nf_conn *ct;
450
451 rcu_read_lock();
452begin:
453 h = ____nf_conntrack_find(net, zone, tuple, hash);
454 if (h) {
455 ct = nf_ct_tuplehash_to_ctrack(h);
456 if (unlikely(nf_ct_is_dying(ct) ||
457 !atomic_inc_not_zero(&ct->ct_general.use)))
458 h = NULL;
459 else {
460 if (unlikely(!nf_ct_key_equal(h, tuple, zone))) {
461 nf_ct_put(ct);
462 goto begin;
463 }
464 }
465 }
466 rcu_read_unlock();
467
468 return h;
469}
470
471struct nf_conntrack_tuple_hash *
472nf_conntrack_find_get(struct net *net, u16 zone,
473 const struct nf_conntrack_tuple *tuple)
474{
475 return __nf_conntrack_find_get(net, zone, tuple,
476 hash_conntrack_raw(tuple, zone));
477}
478EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
479
480static void __nf_conntrack_hash_insert(struct nf_conn *ct,
481 unsigned int hash,
482 unsigned int reply_hash)
483{
484 struct net *net = nf_ct_net(ct);
485
486 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
487 &net->ct.hash[hash]);
488 hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
489 &net->ct.hash[reply_hash]);
490}
491
492int
493nf_conntrack_hash_check_insert(struct nf_conn *ct)
494{
495 struct net *net = nf_ct_net(ct);
496 unsigned int hash, reply_hash;
497 struct nf_conntrack_tuple_hash *h;
498 struct hlist_nulls_node *n;
499 u16 zone;
500 unsigned int sequence;
501
502 zone = nf_ct_zone(ct);
503
504 local_bh_disable();
505 do {
506 sequence = read_seqcount_begin(&net->ct.generation);
507 hash = hash_conntrack(net, zone,
508 &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
509 reply_hash = hash_conntrack(net, zone,
510 &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
511 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
512
513
514 hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
515 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
516 &h->tuple) &&
517 zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
518 goto out;
519 hlist_nulls_for_each_entry(h, n, &net->ct.hash[reply_hash], hnnode)
520 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
521 &h->tuple) &&
522 zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
523 goto out;
524
525 add_timer(&ct->timeout);
526 smp_wmb();
527
528 atomic_set(&ct->ct_general.use, 2);
529 __nf_conntrack_hash_insert(ct, hash, reply_hash);
530 nf_conntrack_double_unlock(hash, reply_hash);
531 NF_CT_STAT_INC(net, insert);
532 local_bh_enable();
533 return 0;
534
535out:
536 nf_conntrack_double_unlock(hash, reply_hash);
537 NF_CT_STAT_INC(net, insert_failed);
538 local_bh_enable();
539 return -EEXIST;
540}
541EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
542
543
544void nf_conntrack_tmpl_insert(struct net *net, struct nf_conn *tmpl)
545{
546 struct ct_pcpu *pcpu;
547
548 __set_bit(IPS_TEMPLATE_BIT, &tmpl->status);
549 __set_bit(IPS_CONFIRMED_BIT, &tmpl->status);
550 nf_conntrack_get(&tmpl->ct_general);
551
552
553 local_bh_disable();
554 tmpl->cpu = smp_processor_id();
555 pcpu = per_cpu_ptr(nf_ct_net(tmpl)->ct.pcpu_lists, tmpl->cpu);
556
557 spin_lock(&pcpu->lock);
558
559 hlist_nulls_add_head_rcu(&tmpl->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
560 &pcpu->tmpl);
561 spin_unlock_bh(&pcpu->lock);
562}
563EXPORT_SYMBOL_GPL(nf_conntrack_tmpl_insert);
564
565
566int
567__nf_conntrack_confirm(struct sk_buff *skb)
568{
569 unsigned int hash, reply_hash;
570 struct nf_conntrack_tuple_hash *h;
571 struct nf_conn *ct;
572 struct nf_conn_help *help;
573 struct nf_conn_tstamp *tstamp;
574 struct hlist_nulls_node *n;
575 enum ip_conntrack_info ctinfo;
576 struct net *net;
577 u16 zone;
578 unsigned int sequence;
579
580 ct = nf_ct_get(skb, &ctinfo);
581 net = nf_ct_net(ct);
582
583
584
585
586
587 if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
588 return NF_ACCEPT;
589
590 zone = nf_ct_zone(ct);
591 local_bh_disable();
592
593 do {
594 sequence = read_seqcount_begin(&net->ct.generation);
595
596 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
597 hash = hash_bucket(hash, net);
598 reply_hash = hash_conntrack(net, zone,
599 &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
600
601 } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
602
603
604
605
606
607
608
609
610
611
612 NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
613 pr_debug("Confirming conntrack %p\n", ct);
614
615
616
617
618
619 nf_ct_del_from_dying_or_unconfirmed_list(ct);
620
621 if (unlikely(nf_ct_is_dying(ct)))
622 goto out;
623
624
625
626
627 hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
628 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
629 &h->tuple) &&
630 zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
631 goto out;
632 hlist_nulls_for_each_entry(h, n, &net->ct.hash[reply_hash], hnnode)
633 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
634 &h->tuple) &&
635 zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
636 goto out;
637
638
639
640
641 ct->timeout.expires += jiffies;
642 add_timer(&ct->timeout);
643 atomic_inc(&ct->ct_general.use);
644 ct->status |= IPS_CONFIRMED;
645
646
647 tstamp = nf_conn_tstamp_find(ct);
648 if (tstamp) {
649 if (skb->tstamp.tv64 == 0)
650 __net_timestamp(skb);
651
652 tstamp->start = ktime_to_ns(skb->tstamp);
653 }
654
655
656
657
658
659 __nf_conntrack_hash_insert(ct, hash, reply_hash);
660 nf_conntrack_double_unlock(hash, reply_hash);
661 NF_CT_STAT_INC(net, insert);
662 local_bh_enable();
663
664 help = nfct_help(ct);
665 if (help && help->helper)
666 nf_conntrack_event_cache(IPCT_HELPER, ct);
667
668 nf_conntrack_event_cache(master_ct(ct) ?
669 IPCT_RELATED : IPCT_NEW, ct);
670 return NF_ACCEPT;
671
672out:
673 nf_ct_add_to_dying_list(ct);
674 nf_conntrack_double_unlock(hash, reply_hash);
675 NF_CT_STAT_INC(net, insert_failed);
676 local_bh_enable();
677 return NF_DROP;
678}
679EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
680
681
682
683int
684nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
685 const struct nf_conn *ignored_conntrack)
686{
687 struct net *net = nf_ct_net(ignored_conntrack);
688 struct nf_conntrack_tuple_hash *h;
689 struct hlist_nulls_node *n;
690 struct nf_conn *ct;
691 u16 zone = nf_ct_zone(ignored_conntrack);
692 unsigned int hash = hash_conntrack(net, zone, tuple);
693
694
695
696
697 rcu_read_lock_bh();
698 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
699 ct = nf_ct_tuplehash_to_ctrack(h);
700 if (ct != ignored_conntrack &&
701 nf_ct_tuple_equal(tuple, &h->tuple) &&
702 nf_ct_zone(ct) == zone) {
703 NF_CT_STAT_INC(net, found);
704 rcu_read_unlock_bh();
705 return 1;
706 }
707 NF_CT_STAT_INC(net, searched);
708 }
709 rcu_read_unlock_bh();
710
711 return 0;
712}
713EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
714
715#define NF_CT_EVICTION_RANGE 8
716
717
718
719static noinline int early_drop(struct net *net, unsigned int _hash)
720{
721
722 struct nf_conntrack_tuple_hash *h;
723 struct nf_conn *ct = NULL, *tmp;
724 struct hlist_nulls_node *n;
725 unsigned int i = 0, cnt = 0;
726 int dropped = 0;
727 unsigned int hash, sequence;
728 spinlock_t *lockp;
729
730 local_bh_disable();
731restart:
732 sequence = read_seqcount_begin(&net->ct.generation);
733 hash = hash_bucket(_hash, net);
734 for (; i < net->ct.htable_size; i++) {
735 lockp = &nf_conntrack_locks[hash % CONNTRACK_LOCKS];
736 spin_lock(lockp);
737 if (read_seqcount_retry(&net->ct.generation, sequence)) {
738 spin_unlock(lockp);
739 goto restart;
740 }
741 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
742 hnnode) {
743 tmp = nf_ct_tuplehash_to_ctrack(h);
744 if (!test_bit(IPS_ASSURED_BIT, &tmp->status) &&
745 !nf_ct_is_dying(tmp) &&
746 atomic_inc_not_zero(&tmp->ct_general.use)) {
747 ct = tmp;
748 break;
749 }
750 cnt++;
751 }
752
753 hash = (hash + 1) % net->ct.htable_size;
754 spin_unlock(lockp);
755
756 if (ct || cnt >= NF_CT_EVICTION_RANGE)
757 break;
758
759 }
760 local_bh_enable();
761
762 if (!ct)
763 return dropped;
764
765 if (del_timer(&ct->timeout)) {
766 if (nf_ct_delete(ct, 0, 0)) {
767 dropped = 1;
768 NF_CT_STAT_INC_ATOMIC(net, early_drop);
769 }
770 }
771 nf_ct_put(ct);
772 return dropped;
773}
774
775void init_nf_conntrack_hash_rnd(void)
776{
777 unsigned int rand;
778
779
780
781
782
783
784 do {
785 get_random_bytes(&rand, sizeof(rand));
786 } while (!rand);
787 cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
788}
789
790static struct nf_conn *
791__nf_conntrack_alloc(struct net *net, u16 zone,
792 const struct nf_conntrack_tuple *orig,
793 const struct nf_conntrack_tuple *repl,
794 gfp_t gfp, u32 hash)
795{
796 struct nf_conn *ct;
797
798 if (unlikely(!nf_conntrack_hash_rnd)) {
799 init_nf_conntrack_hash_rnd();
800
801 hash = hash_conntrack_raw(orig, zone);
802 }
803
804
805 atomic_inc(&net->ct.count);
806
807 if (nf_conntrack_max &&
808 unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
809 if (!early_drop(net, hash)) {
810 atomic_dec(&net->ct.count);
811 net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
812 return ERR_PTR(-ENOMEM);
813 }
814 }
815
816
817
818
819
820 ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
821 if (ct == NULL) {
822 atomic_dec(&net->ct.count);
823 return ERR_PTR(-ENOMEM);
824 }
825 spin_lock_init(&ct->lock);
826 ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
827 ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
828 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
829
830 *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
831 ct->status = 0;
832
833 setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
834 write_pnet(&ct->ct_net, net);
835 memset(&ct->__nfct_init_offset[0], 0,
836 offsetof(struct nf_conn, proto) -
837 offsetof(struct nf_conn, __nfct_init_offset[0]));
838#ifdef CONFIG_NF_CONNTRACK_ZONES
839 if (zone) {
840 struct nf_conntrack_zone *nf_ct_zone;
841
842 nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
843 if (!nf_ct_zone)
844 goto out_free;
845 nf_ct_zone->id = zone;
846 }
847#endif
848
849
850
851 atomic_set(&ct->ct_general.use, 0);
852 return ct;
853
854#ifdef CONFIG_NF_CONNTRACK_ZONES
855out_free:
856 atomic_dec(&net->ct.count);
857 kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
858 return ERR_PTR(-ENOMEM);
859#endif
860}
861
862struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
863 const struct nf_conntrack_tuple *orig,
864 const struct nf_conntrack_tuple *repl,
865 gfp_t gfp)
866{
867 return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
868}
869EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
870
871void nf_conntrack_free(struct nf_conn *ct)
872{
873 struct net *net = nf_ct_net(ct);
874
875
876
877
878 NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0);
879
880 nf_ct_ext_destroy(ct);
881 nf_ct_ext_free(ct);
882 kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
883 smp_mb__before_atomic();
884 atomic_dec(&net->ct.count);
885}
886EXPORT_SYMBOL_GPL(nf_conntrack_free);
887
888
889
890
891static struct nf_conntrack_tuple_hash *
892init_conntrack(struct net *net, struct nf_conn *tmpl,
893 const struct nf_conntrack_tuple *tuple,
894 struct nf_conntrack_l3proto *l3proto,
895 struct nf_conntrack_l4proto *l4proto,
896 struct sk_buff *skb,
897 unsigned int dataoff, u32 hash)
898{
899 struct nf_conn *ct;
900 struct nf_conn_help *help;
901 struct nf_conntrack_tuple repl_tuple;
902 struct nf_conntrack_ecache *ecache;
903 struct nf_conntrack_expect *exp = NULL;
904 u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
905 struct nf_conn_timeout *timeout_ext;
906 unsigned int *timeouts;
907
908 if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
909 pr_debug("Can't invert tuple.\n");
910 return NULL;
911 }
912
913 ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
914 hash);
915 if (IS_ERR(ct))
916 return (struct nf_conntrack_tuple_hash *)ct;
917
918 if (tmpl && nfct_synproxy(tmpl)) {
919 nfct_seqadj_ext_add(ct);
920 nfct_synproxy_ext_add(ct);
921 }
922
923 timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
924 if (timeout_ext)
925 timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
926 else
927 timeouts = l4proto->get_timeouts(net);
928
929 if (!l4proto->new(ct, skb, dataoff, timeouts)) {
930 nf_conntrack_free(ct);
931 pr_debug("init conntrack: can't track with proto module\n");
932 return NULL;
933 }
934
935 if (timeout_ext)
936 nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
937
938 nf_ct_acct_ext_add(ct, GFP_ATOMIC);
939 nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
940 nf_ct_labels_ext_add(ct);
941
942 ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
943 nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
944 ecache ? ecache->expmask : 0,
945 GFP_ATOMIC);
946
947 local_bh_disable();
948 if (net->ct.expect_count) {
949 spin_lock(&nf_conntrack_expect_lock);
950 exp = nf_ct_find_expectation(net, zone, tuple);
951 if (exp) {
952 pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
953 ct, exp);
954
955 __set_bit(IPS_EXPECTED_BIT, &ct->status);
956
957 ct->master = exp->master;
958 if (exp->helper) {
959 help = nf_ct_helper_ext_add(ct, exp->helper,
960 GFP_ATOMIC);
961 if (help)
962 rcu_assign_pointer(help->helper, exp->helper);
963 }
964
965#ifdef CONFIG_NF_CONNTRACK_MARK
966 ct->mark = exp->master->mark;
967#endif
968#ifdef CONFIG_NF_CONNTRACK_SECMARK
969 ct->secmark = exp->master->secmark;
970#endif
971 NF_CT_STAT_INC(net, expect_new);
972 }
973 spin_unlock(&nf_conntrack_expect_lock);
974 }
975 if (!exp) {
976 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
977 NF_CT_STAT_INC(net, new);
978 }
979
980
981 nf_conntrack_get(&ct->ct_general);
982 nf_ct_add_to_unconfirmed_list(ct);
983
984 local_bh_enable();
985
986 if (exp) {
987 if (exp->expectfn)
988 exp->expectfn(ct, exp);
989 nf_ct_expect_put(exp);
990 }
991
992 return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
993}
994
995
996static inline struct nf_conn *
997resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
998 struct sk_buff *skb,
999 unsigned int dataoff,
1000 u_int16_t l3num,
1001 u_int8_t protonum,
1002 struct nf_conntrack_l3proto *l3proto,
1003 struct nf_conntrack_l4proto *l4proto,
1004 int *set_reply,
1005 enum ip_conntrack_info *ctinfo)
1006{
1007 struct nf_conntrack_tuple tuple;
1008 struct nf_conntrack_tuple_hash *h;
1009 struct nf_conn *ct;
1010 u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
1011 u32 hash;
1012
1013 if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1014 dataoff, l3num, protonum, &tuple, l3proto,
1015 l4proto)) {
1016 pr_debug("resolve_normal_ct: Can't get tuple\n");
1017 return NULL;
1018 }
1019
1020
1021 hash = hash_conntrack_raw(&tuple, zone);
1022 h = __nf_conntrack_find_get(net, zone, &tuple, hash);
1023 if (!h) {
1024 h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
1025 skb, dataoff, hash);
1026 if (!h)
1027 return NULL;
1028 if (IS_ERR(h))
1029 return (void *)h;
1030 }
1031 ct = nf_ct_tuplehash_to_ctrack(h);
1032
1033
1034 if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1035 *ctinfo = IP_CT_ESTABLISHED_REPLY;
1036
1037 *set_reply = 1;
1038 } else {
1039
1040 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1041 pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
1042 *ctinfo = IP_CT_ESTABLISHED;
1043 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1044 pr_debug("nf_conntrack_in: related packet for %p\n",
1045 ct);
1046 *ctinfo = IP_CT_RELATED;
1047 } else {
1048 pr_debug("nf_conntrack_in: new packet for %p\n", ct);
1049 *ctinfo = IP_CT_NEW;
1050 }
1051 *set_reply = 0;
1052 }
1053 skb->nfct = &ct->ct_general;
1054 skb->nfctinfo = *ctinfo;
1055 return ct;
1056}
1057
1058unsigned int
1059nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
1060 struct sk_buff *skb)
1061{
1062 struct nf_conn *ct, *tmpl = NULL;
1063 enum ip_conntrack_info ctinfo;
1064 struct nf_conntrack_l3proto *l3proto;
1065 struct nf_conntrack_l4proto *l4proto;
1066 unsigned int *timeouts;
1067 unsigned int dataoff;
1068 u_int8_t protonum;
1069 int set_reply = 0;
1070 int ret;
1071
1072 if (skb->nfct) {
1073
1074 tmpl = (struct nf_conn *)skb->nfct;
1075 if (!nf_ct_is_template(tmpl)) {
1076 NF_CT_STAT_INC_ATOMIC(net, ignore);
1077 return NF_ACCEPT;
1078 }
1079 skb->nfct = NULL;
1080 }
1081
1082
1083 l3proto = __nf_ct_l3proto_find(pf);
1084 ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
1085 &dataoff, &protonum);
1086 if (ret <= 0) {
1087 pr_debug("not prepared to track yet or error occurred\n");
1088 NF_CT_STAT_INC_ATOMIC(net, error);
1089 NF_CT_STAT_INC_ATOMIC(net, invalid);
1090 ret = -ret;
1091 goto out;
1092 }
1093
1094 l4proto = __nf_ct_l4proto_find(pf, protonum);
1095
1096
1097
1098
1099 if (l4proto->error != NULL) {
1100 ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
1101 pf, hooknum);
1102 if (ret <= 0) {
1103 NF_CT_STAT_INC_ATOMIC(net, error);
1104 NF_CT_STAT_INC_ATOMIC(net, invalid);
1105 ret = -ret;
1106 goto out;
1107 }
1108
1109 if (skb->nfct)
1110 goto out;
1111 }
1112
1113 ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
1114 l3proto, l4proto, &set_reply, &ctinfo);
1115 if (!ct) {
1116
1117 NF_CT_STAT_INC_ATOMIC(net, invalid);
1118 ret = NF_ACCEPT;
1119 goto out;
1120 }
1121
1122 if (IS_ERR(ct)) {
1123
1124 NF_CT_STAT_INC_ATOMIC(net, drop);
1125 ret = NF_DROP;
1126 goto out;
1127 }
1128
1129 NF_CT_ASSERT(skb->nfct);
1130
1131
1132 timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1133
1134 ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1135 if (ret <= 0) {
1136
1137
1138 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1139 nf_conntrack_put(skb->nfct);
1140 skb->nfct = NULL;
1141 NF_CT_STAT_INC_ATOMIC(net, invalid);
1142 if (ret == -NF_DROP)
1143 NF_CT_STAT_INC_ATOMIC(net, drop);
1144 ret = -ret;
1145 goto out;
1146 }
1147
1148 if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1149 nf_conntrack_event_cache(IPCT_REPLY, ct);
1150out:
1151 if (tmpl) {
1152
1153
1154
1155 if (ret == NF_REPEAT)
1156 skb->nfct = (struct nf_conntrack *)tmpl;
1157 else
1158 nf_ct_put(tmpl);
1159 }
1160
1161 return ret;
1162}
1163EXPORT_SYMBOL_GPL(nf_conntrack_in);
1164
1165bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1166 const struct nf_conntrack_tuple *orig)
1167{
1168 bool ret;
1169
1170 rcu_read_lock();
1171 ret = nf_ct_invert_tuple(inverse, orig,
1172 __nf_ct_l3proto_find(orig->src.l3num),
1173 __nf_ct_l4proto_find(orig->src.l3num,
1174 orig->dst.protonum));
1175 rcu_read_unlock();
1176 return ret;
1177}
1178EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1179
1180
1181
1182void nf_conntrack_alter_reply(struct nf_conn *ct,
1183 const struct nf_conntrack_tuple *newreply)
1184{
1185 struct nf_conn_help *help = nfct_help(ct);
1186
1187
1188 NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1189
1190 pr_debug("Altering reply tuple of %p to ", ct);
1191 nf_ct_dump_tuple(newreply);
1192
1193 ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1194 if (ct->master || (help && !hlist_empty(&help->expectations)))
1195 return;
1196
1197 rcu_read_lock();
1198 __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1199 rcu_read_unlock();
1200}
1201EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1202
1203
1204void __nf_ct_refresh_acct(struct nf_conn *ct,
1205 enum ip_conntrack_info ctinfo,
1206 const struct sk_buff *skb,
1207 unsigned long extra_jiffies,
1208 int do_acct)
1209{
1210 NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1211 NF_CT_ASSERT(skb);
1212
1213
1214 if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1215 goto acct;
1216
1217
1218 if (!nf_ct_is_confirmed(ct)) {
1219 ct->timeout.expires = extra_jiffies;
1220 } else {
1221 unsigned long newtime = jiffies + extra_jiffies;
1222
1223
1224
1225
1226 if (newtime - ct->timeout.expires >= HZ)
1227 mod_timer_pending(&ct->timeout, newtime);
1228 }
1229
1230acct:
1231 if (do_acct) {
1232 struct nf_conn_acct *acct;
1233
1234 acct = nf_conn_acct_find(ct);
1235 if (acct) {
1236 struct nf_conn_counter *counter = acct->counter;
1237
1238 atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
1239 atomic64_add(skb->len, &counter[CTINFO2DIR(ctinfo)].bytes);
1240 }
1241 }
1242}
1243EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1244
1245bool __nf_ct_kill_acct(struct nf_conn *ct,
1246 enum ip_conntrack_info ctinfo,
1247 const struct sk_buff *skb,
1248 int do_acct)
1249{
1250 if (do_acct) {
1251 struct nf_conn_acct *acct;
1252
1253 acct = nf_conn_acct_find(ct);
1254 if (acct) {
1255 struct nf_conn_counter *counter = acct->counter;
1256
1257 atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
1258 atomic64_add(skb->len - skb_network_offset(skb),
1259 &counter[CTINFO2DIR(ctinfo)].bytes);
1260 }
1261 }
1262
1263 if (del_timer(&ct->timeout)) {
1264 ct->timeout.function((unsigned long)ct);
1265 return true;
1266 }
1267 return false;
1268}
1269EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
1270
1271#ifdef CONFIG_NF_CONNTRACK_ZONES
1272static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
1273 .len = sizeof(struct nf_conntrack_zone),
1274 .align = __alignof__(struct nf_conntrack_zone),
1275 .id = NF_CT_EXT_ZONE,
1276};
1277#endif
1278
1279#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1280
1281#include <linux/netfilter/nfnetlink.h>
1282#include <linux/netfilter/nfnetlink_conntrack.h>
1283#include <linux/mutex.h>
1284
1285
1286
1287
1288int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1289 const struct nf_conntrack_tuple *tuple)
1290{
1291 if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1292 nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1293 goto nla_put_failure;
1294 return 0;
1295
1296nla_put_failure:
1297 return -1;
1298}
1299EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1300
1301const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1302 [CTA_PROTO_SRC_PORT] = { .type = NLA_U16 },
1303 [CTA_PROTO_DST_PORT] = { .type = NLA_U16 },
1304};
1305EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1306
1307int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1308 struct nf_conntrack_tuple *t)
1309{
1310 if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1311 return -EINVAL;
1312
1313 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1314 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1315
1316 return 0;
1317}
1318EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1319
1320int nf_ct_port_nlattr_tuple_size(void)
1321{
1322 return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1323}
1324EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1325#endif
1326
1327
1328static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
1329{
1330 struct nf_conn *ct;
1331 enum ip_conntrack_info ctinfo;
1332
1333
1334 ct = nf_ct_get(skb, &ctinfo);
1335 if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1336 ctinfo = IP_CT_RELATED_REPLY;
1337 else
1338 ctinfo = IP_CT_RELATED;
1339
1340
1341 nskb->nfct = &ct->ct_general;
1342 nskb->nfctinfo = ctinfo;
1343 nf_conntrack_get(nskb->nfct);
1344}
1345
1346
1347static struct nf_conn *
1348get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1349 void *data, unsigned int *bucket)
1350{
1351 struct nf_conntrack_tuple_hash *h;
1352 struct nf_conn *ct;
1353 struct hlist_nulls_node *n;
1354 int cpu;
1355 spinlock_t *lockp;
1356
1357 for (; *bucket < net->ct.htable_size; (*bucket)++) {
1358 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
1359 local_bh_disable();
1360 spin_lock(lockp);
1361 if (*bucket < net->ct.htable_size) {
1362 hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
1363 if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1364 continue;
1365 ct = nf_ct_tuplehash_to_ctrack(h);
1366 if (iter(ct, data))
1367 goto found;
1368 }
1369 }
1370 spin_unlock(lockp);
1371 local_bh_enable();
1372 }
1373
1374 for_each_possible_cpu(cpu) {
1375 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1376
1377 spin_lock_bh(&pcpu->lock);
1378 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
1379 ct = nf_ct_tuplehash_to_ctrack(h);
1380 if (iter(ct, data))
1381 set_bit(IPS_DYING_BIT, &ct->status);
1382 }
1383 spin_unlock_bh(&pcpu->lock);
1384 }
1385 return NULL;
1386found:
1387 atomic_inc(&ct->ct_general.use);
1388 spin_unlock(lockp);
1389 local_bh_enable();
1390 return ct;
1391}
1392
1393void nf_ct_iterate_cleanup(struct net *net,
1394 int (*iter)(struct nf_conn *i, void *data),
1395 void *data, u32 portid, int report)
1396{
1397 struct nf_conn *ct;
1398 unsigned int bucket = 0;
1399
1400 while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1401
1402 if (del_timer(&ct->timeout))
1403 nf_ct_delete(ct, portid, report);
1404
1405
1406
1407 nf_ct_put(ct);
1408 }
1409}
1410EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1411
1412static int kill_all(struct nf_conn *i, void *data)
1413{
1414 return 1;
1415}
1416
1417void nf_ct_free_hashtable(void *hash, unsigned int size)
1418{
1419 if (is_vmalloc_addr(hash))
1420 vfree(hash);
1421 else
1422 free_pages((unsigned long)hash,
1423 get_order(sizeof(struct hlist_head) * size));
1424}
1425EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1426
1427static int untrack_refs(void)
1428{
1429 int cnt = 0, cpu;
1430
1431 for_each_possible_cpu(cpu) {
1432 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1433
1434 cnt += atomic_read(&ct->ct_general.use) - 1;
1435 }
1436 return cnt;
1437}
1438
1439void nf_conntrack_cleanup_start(void)
1440{
1441 RCU_INIT_POINTER(ip_ct_attach, NULL);
1442}
1443
1444void nf_conntrack_cleanup_end(void)
1445{
1446 RCU_INIT_POINTER(nf_ct_destroy, NULL);
1447 while (untrack_refs() > 0)
1448 schedule();
1449
1450#ifdef CONFIG_NF_CONNTRACK_ZONES
1451 nf_ct_extend_unregister(&nf_ct_zone_extend);
1452#endif
1453 nf_conntrack_proto_fini();
1454 nf_conntrack_seqadj_fini();
1455 nf_conntrack_labels_fini();
1456 nf_conntrack_helper_fini();
1457 nf_conntrack_timeout_fini();
1458 nf_conntrack_ecache_fini();
1459 nf_conntrack_tstamp_fini();
1460 nf_conntrack_acct_fini();
1461 nf_conntrack_expect_fini();
1462}
1463
1464
1465
1466
1467
1468void nf_conntrack_cleanup_net(struct net *net)
1469{
1470 LIST_HEAD(single);
1471
1472 list_add(&net->exit_list, &single);
1473 nf_conntrack_cleanup_net_list(&single);
1474}
1475
1476void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
1477{
1478 int busy;
1479 struct net *net;
1480
1481
1482
1483
1484
1485
1486 synchronize_net();
1487i_see_dead_people:
1488 busy = 0;
1489 list_for_each_entry(net, net_exit_list, exit_list) {
1490 nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0);
1491 if (atomic_read(&net->ct.count) != 0)
1492 busy = 1;
1493 }
1494 if (busy) {
1495 schedule();
1496 goto i_see_dead_people;
1497 }
1498
1499 list_for_each_entry(net, net_exit_list, exit_list) {
1500 nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1501 nf_conntrack_proto_pernet_fini(net);
1502 nf_conntrack_helper_pernet_fini(net);
1503 nf_conntrack_ecache_pernet_fini(net);
1504 nf_conntrack_tstamp_pernet_fini(net);
1505 nf_conntrack_acct_pernet_fini(net);
1506 nf_conntrack_expect_pernet_fini(net);
1507 kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1508 kfree(net->ct.slabname);
1509 free_percpu(net->ct.stat);
1510 free_percpu(net->ct.pcpu_lists);
1511 }
1512}
1513
1514void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1515{
1516 struct hlist_nulls_head *hash;
1517 unsigned int nr_slots, i;
1518 size_t sz;
1519
1520 BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1521 nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1522 sz = nr_slots * sizeof(struct hlist_nulls_head);
1523 hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1524 get_order(sz));
1525 if (!hash) {
1526 printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
1527 hash = vzalloc(sz);
1528 }
1529
1530 if (hash && nulls)
1531 for (i = 0; i < nr_slots; i++)
1532 INIT_HLIST_NULLS_HEAD(&hash[i], i);
1533
1534 return hash;
1535}
1536EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1537
1538int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1539{
1540 int i, bucket, rc;
1541 unsigned int hashsize, old_size;
1542 struct hlist_nulls_head *hash, *old_hash;
1543 struct nf_conntrack_tuple_hash *h;
1544 struct nf_conn *ct;
1545
1546 if (current->nsproxy->net_ns != &init_net)
1547 return -EOPNOTSUPP;
1548
1549
1550 if (!nf_conntrack_htable_size)
1551 return param_set_uint(val, kp);
1552
1553 rc = kstrtouint(val, 0, &hashsize);
1554 if (rc)
1555 return rc;
1556 if (!hashsize)
1557 return -EINVAL;
1558
1559 hash = nf_ct_alloc_hashtable(&hashsize, 1);
1560 if (!hash)
1561 return -ENOMEM;
1562
1563 local_bh_disable();
1564 nf_conntrack_all_lock();
1565 write_seqcount_begin(&init_net.ct.generation);
1566
1567
1568
1569
1570
1571
1572
1573 for (i = 0; i < init_net.ct.htable_size; i++) {
1574 while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
1575 h = hlist_nulls_entry(init_net.ct.hash[i].first,
1576 struct nf_conntrack_tuple_hash, hnnode);
1577 ct = nf_ct_tuplehash_to_ctrack(h);
1578 hlist_nulls_del_rcu(&h->hnnode);
1579 bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
1580 hashsize);
1581 hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1582 }
1583 }
1584 old_size = init_net.ct.htable_size;
1585 old_hash = init_net.ct.hash;
1586
1587 init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
1588 init_net.ct.hash = hash;
1589
1590 write_seqcount_end(&init_net.ct.generation);
1591 nf_conntrack_all_unlock();
1592 local_bh_enable();
1593
1594 nf_ct_free_hashtable(old_hash, old_size);
1595 return 0;
1596}
1597EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1598
1599module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1600 &nf_conntrack_htable_size, 0600);
1601
1602void nf_ct_untracked_status_or(unsigned long bits)
1603{
1604 int cpu;
1605
1606 for_each_possible_cpu(cpu)
1607 per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1608}
1609EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1610
1611int nf_conntrack_init_start(void)
1612{
1613 int max_factor = 8;
1614 int i, ret, cpu;
1615
1616 for (i = 0; i < CONNTRACK_LOCKS; i++)
1617 spin_lock_init(&nf_conntrack_locks[i]);
1618
1619 if (!nf_conntrack_htable_size) {
1620
1621
1622
1623
1624
1625 nf_conntrack_htable_size
1626 = (((totalram_pages << PAGE_SHIFT) / 16384)
1627 / sizeof(struct hlist_head));
1628 if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
1629 nf_conntrack_htable_size = 65536;
1630 else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1631 nf_conntrack_htable_size = 16384;
1632 if (nf_conntrack_htable_size < 32)
1633 nf_conntrack_htable_size = 32;
1634
1635
1636
1637
1638
1639 max_factor = 4;
1640 }
1641 nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1642
1643 printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1644 NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1645 nf_conntrack_max);
1646
1647 ret = nf_conntrack_expect_init();
1648 if (ret < 0)
1649 goto err_expect;
1650
1651 ret = nf_conntrack_acct_init();
1652 if (ret < 0)
1653 goto err_acct;
1654
1655 ret = nf_conntrack_tstamp_init();
1656 if (ret < 0)
1657 goto err_tstamp;
1658
1659 ret = nf_conntrack_ecache_init();
1660 if (ret < 0)
1661 goto err_ecache;
1662
1663 ret = nf_conntrack_timeout_init();
1664 if (ret < 0)
1665 goto err_timeout;
1666
1667 ret = nf_conntrack_helper_init();
1668 if (ret < 0)
1669 goto err_helper;
1670
1671 ret = nf_conntrack_labels_init();
1672 if (ret < 0)
1673 goto err_labels;
1674
1675 ret = nf_conntrack_seqadj_init();
1676 if (ret < 0)
1677 goto err_seqadj;
1678
1679#ifdef CONFIG_NF_CONNTRACK_ZONES
1680 ret = nf_ct_extend_register(&nf_ct_zone_extend);
1681 if (ret < 0)
1682 goto err_extend;
1683#endif
1684 ret = nf_conntrack_proto_init();
1685 if (ret < 0)
1686 goto err_proto;
1687
1688
1689 for_each_possible_cpu(cpu) {
1690 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1691 write_pnet(&ct->ct_net, &init_net);
1692 atomic_set(&ct->ct_general.use, 1);
1693 }
1694
1695 nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
1696 return 0;
1697
1698err_proto:
1699#ifdef CONFIG_NF_CONNTRACK_ZONES
1700 nf_ct_extend_unregister(&nf_ct_zone_extend);
1701err_extend:
1702#endif
1703 nf_conntrack_seqadj_fini();
1704err_seqadj:
1705 nf_conntrack_labels_fini();
1706err_labels:
1707 nf_conntrack_helper_fini();
1708err_helper:
1709 nf_conntrack_timeout_fini();
1710err_timeout:
1711 nf_conntrack_ecache_fini();
1712err_ecache:
1713 nf_conntrack_tstamp_fini();
1714err_tstamp:
1715 nf_conntrack_acct_fini();
1716err_acct:
1717 nf_conntrack_expect_fini();
1718err_expect:
1719 return ret;
1720}
1721
1722void nf_conntrack_init_end(void)
1723{
1724
1725 RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1726 RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1727}
1728
1729
1730
1731
1732#define UNCONFIRMED_NULLS_VAL ((1<<30)+0)
1733#define DYING_NULLS_VAL ((1<<30)+1)
1734#define TEMPLATE_NULLS_VAL ((1<<30)+2)
1735
1736int nf_conntrack_init_net(struct net *net)
1737{
1738 int ret = -ENOMEM;
1739 int cpu;
1740
1741 atomic_set(&net->ct.count, 0);
1742 seqcount_init(&net->ct.generation);
1743
1744 net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
1745 if (!net->ct.pcpu_lists)
1746 goto err_stat;
1747
1748 for_each_possible_cpu(cpu) {
1749 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1750
1751 spin_lock_init(&pcpu->lock);
1752 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
1753 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
1754 INIT_HLIST_NULLS_HEAD(&pcpu->tmpl, TEMPLATE_NULLS_VAL);
1755 }
1756
1757 net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1758 if (!net->ct.stat)
1759 goto err_pcpu_lists;
1760
1761 net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
1762 if (!net->ct.slabname)
1763 goto err_slabname;
1764
1765 net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
1766 sizeof(struct nf_conn), 0,
1767 SLAB_DESTROY_BY_RCU, NULL);
1768 if (!net->ct.nf_conntrack_cachep) {
1769 printk(KERN_ERR "Unable to create nf_conn slab cache\n");
1770 goto err_cache;
1771 }
1772
1773 net->ct.htable_size = nf_conntrack_htable_size;
1774 net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
1775 if (!net->ct.hash) {
1776 printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
1777 goto err_hash;
1778 }
1779 ret = nf_conntrack_expect_pernet_init(net);
1780 if (ret < 0)
1781 goto err_expect;
1782 ret = nf_conntrack_acct_pernet_init(net);
1783 if (ret < 0)
1784 goto err_acct;
1785 ret = nf_conntrack_tstamp_pernet_init(net);
1786 if (ret < 0)
1787 goto err_tstamp;
1788 ret = nf_conntrack_ecache_pernet_init(net);
1789 if (ret < 0)
1790 goto err_ecache;
1791 ret = nf_conntrack_helper_pernet_init(net);
1792 if (ret < 0)
1793 goto err_helper;
1794 ret = nf_conntrack_proto_pernet_init(net);
1795 if (ret < 0)
1796 goto err_proto;
1797 return 0;
1798
1799err_proto:
1800 nf_conntrack_helper_pernet_fini(net);
1801err_helper:
1802 nf_conntrack_ecache_pernet_fini(net);
1803err_ecache:
1804 nf_conntrack_tstamp_pernet_fini(net);
1805err_tstamp:
1806 nf_conntrack_acct_pernet_fini(net);
1807err_acct:
1808 nf_conntrack_expect_pernet_fini(net);
1809err_expect:
1810 nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1811err_hash:
1812 kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1813err_cache:
1814 kfree(net->ct.slabname);
1815err_slabname:
1816 free_percpu(net->ct.stat);
1817err_pcpu_lists:
1818 free_percpu(net->ct.pcpu_lists);
1819err_stat:
1820 return ret;
1821}
1822