1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#define KMSG_COMPONENT "IPVS"
28#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
29
30#include <linux/module.h>
31#include <linux/kernel.h>
32#include <linux/ip.h>
33#include <linux/tcp.h>
34#include <linux/sctp.h>
35#include <linux/icmp.h>
36#include <linux/slab.h>
37
38#include <net/ip.h>
39#include <net/tcp.h>
40#include <net/udp.h>
41#include <net/icmp.h>
42#include <net/route.h>
43#include <net/ip6_checksum.h>
44#include <net/netns/generic.h>
45
46#include <linux/netfilter.h>
47#include <linux/netfilter_ipv4.h>
48
49#ifdef CONFIG_IP_VS_IPV6
50#include <net/ipv6.h>
51#include <linux/netfilter_ipv6.h>
52#include <net/ip6_route.h>
53#endif
54
55#include <net/ip_vs.h>
56
57
58EXPORT_SYMBOL(register_ip_vs_scheduler);
59EXPORT_SYMBOL(unregister_ip_vs_scheduler);
60EXPORT_SYMBOL(ip_vs_proto_name);
61EXPORT_SYMBOL(ip_vs_conn_new);
62EXPORT_SYMBOL(ip_vs_conn_in_get);
63EXPORT_SYMBOL(ip_vs_conn_out_get);
64#ifdef CONFIG_IP_VS_PROTO_TCP
65EXPORT_SYMBOL(ip_vs_tcp_conn_listen);
66#endif
67EXPORT_SYMBOL(ip_vs_conn_put);
68#ifdef CONFIG_IP_VS_DEBUG
69EXPORT_SYMBOL(ip_vs_get_debug_level);
70#endif
71EXPORT_SYMBOL(ip_vs_new_conn_out);
72
73static unsigned int ip_vs_net_id __read_mostly;
74
75static atomic_t ipvs_netns_cnt = ATOMIC_INIT(0);
76
77
78#define icmp_id(icmph) (((icmph)->un).echo.id)
79#define icmpv6_id(icmph) (icmph->icmp6_dataun.u_echo.identifier)
80
81const char *ip_vs_proto_name(unsigned int proto)
82{
83 static char buf[20];
84
85 switch (proto) {
86 case IPPROTO_IP:
87 return "IP";
88 case IPPROTO_UDP:
89 return "UDP";
90 case IPPROTO_TCP:
91 return "TCP";
92 case IPPROTO_SCTP:
93 return "SCTP";
94 case IPPROTO_ICMP:
95 return "ICMP";
96#ifdef CONFIG_IP_VS_IPV6
97 case IPPROTO_ICMPV6:
98 return "ICMPv6";
99#endif
100 default:
101 sprintf(buf, "IP_%u", proto);
102 return buf;
103 }
104}
105
106void ip_vs_init_hash_table(struct list_head *table, int rows)
107{
108 while (--rows >= 0)
109 INIT_LIST_HEAD(&table[rows]);
110}
111
112static inline void
113ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
114{
115 struct ip_vs_dest *dest = cp->dest;
116 struct netns_ipvs *ipvs = cp->ipvs;
117
118 if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
119 struct ip_vs_cpu_stats *s;
120 struct ip_vs_service *svc;
121
122 local_bh_disable();
123
124 s = this_cpu_ptr(dest->stats.cpustats);
125 u64_stats_update_begin(&s->syncp);
126 s->cnt.inpkts++;
127 s->cnt.inbytes += skb->len;
128 u64_stats_update_end(&s->syncp);
129
130 svc = rcu_dereference(dest->svc);
131 s = this_cpu_ptr(svc->stats.cpustats);
132 u64_stats_update_begin(&s->syncp);
133 s->cnt.inpkts++;
134 s->cnt.inbytes += skb->len;
135 u64_stats_update_end(&s->syncp);
136
137 s = this_cpu_ptr(ipvs->tot_stats.cpustats);
138 u64_stats_update_begin(&s->syncp);
139 s->cnt.inpkts++;
140 s->cnt.inbytes += skb->len;
141 u64_stats_update_end(&s->syncp);
142
143 local_bh_enable();
144 }
145}
146
147
148static inline void
149ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
150{
151 struct ip_vs_dest *dest = cp->dest;
152 struct netns_ipvs *ipvs = cp->ipvs;
153
154 if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
155 struct ip_vs_cpu_stats *s;
156 struct ip_vs_service *svc;
157
158 local_bh_disable();
159
160 s = this_cpu_ptr(dest->stats.cpustats);
161 u64_stats_update_begin(&s->syncp);
162 s->cnt.outpkts++;
163 s->cnt.outbytes += skb->len;
164 u64_stats_update_end(&s->syncp);
165
166 svc = rcu_dereference(dest->svc);
167 s = this_cpu_ptr(svc->stats.cpustats);
168 u64_stats_update_begin(&s->syncp);
169 s->cnt.outpkts++;
170 s->cnt.outbytes += skb->len;
171 u64_stats_update_end(&s->syncp);
172
173 s = this_cpu_ptr(ipvs->tot_stats.cpustats);
174 u64_stats_update_begin(&s->syncp);
175 s->cnt.outpkts++;
176 s->cnt.outbytes += skb->len;
177 u64_stats_update_end(&s->syncp);
178
179 local_bh_enable();
180 }
181}
182
183
184static inline void
185ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc)
186{
187 struct netns_ipvs *ipvs = svc->ipvs;
188 struct ip_vs_cpu_stats *s;
189
190 local_bh_disable();
191
192 s = this_cpu_ptr(cp->dest->stats.cpustats);
193 u64_stats_update_begin(&s->syncp);
194 s->cnt.conns++;
195 u64_stats_update_end(&s->syncp);
196
197 s = this_cpu_ptr(svc->stats.cpustats);
198 u64_stats_update_begin(&s->syncp);
199 s->cnt.conns++;
200 u64_stats_update_end(&s->syncp);
201
202 s = this_cpu_ptr(ipvs->tot_stats.cpustats);
203 u64_stats_update_begin(&s->syncp);
204 s->cnt.conns++;
205 u64_stats_update_end(&s->syncp);
206
207 local_bh_enable();
208}
209
210
211static inline void
212ip_vs_set_state(struct ip_vs_conn *cp, int direction,
213 const struct sk_buff *skb,
214 struct ip_vs_proto_data *pd)
215{
216 if (likely(pd->pp->state_transition))
217 pd->pp->state_transition(cp, direction, skb, pd);
218}
219
220static inline int
221ip_vs_conn_fill_param_persist(const struct ip_vs_service *svc,
222 struct sk_buff *skb, int protocol,
223 const union nf_inet_addr *caddr, __be16 cport,
224 const union nf_inet_addr *vaddr, __be16 vport,
225 struct ip_vs_conn_param *p)
226{
227 ip_vs_conn_fill_param(svc->ipvs, svc->af, protocol, caddr, cport, vaddr,
228 vport, p);
229 p->pe = rcu_dereference(svc->pe);
230 if (p->pe && p->pe->fill_param)
231 return p->pe->fill_param(p, skb);
232
233 return 0;
234}
235
236
237
238
239
240
241
242
243static struct ip_vs_conn *
244ip_vs_sched_persist(struct ip_vs_service *svc,
245 struct sk_buff *skb, __be16 src_port, __be16 dst_port,
246 int *ignored, struct ip_vs_iphdr *iph)
247{
248 struct ip_vs_conn *cp = NULL;
249 struct ip_vs_dest *dest;
250 struct ip_vs_conn *ct;
251 __be16 dport = 0;
252 unsigned int flags;
253 struct ip_vs_conn_param param;
254 const union nf_inet_addr fwmark = { .ip = htonl(svc->fwmark) };
255 union nf_inet_addr snet;
256
257 const union nf_inet_addr *src_addr, *dst_addr;
258
259 if (likely(!ip_vs_iph_inverse(iph))) {
260 src_addr = &iph->saddr;
261 dst_addr = &iph->daddr;
262 } else {
263 src_addr = &iph->daddr;
264 dst_addr = &iph->saddr;
265 }
266
267
268
269#ifdef CONFIG_IP_VS_IPV6
270 if (svc->af == AF_INET6)
271 ipv6_addr_prefix(&snet.in6, &src_addr->in6,
272 (__force __u32) svc->netmask);
273 else
274#endif
275 snet.ip = src_addr->ip & svc->netmask;
276
277 IP_VS_DBG_BUF(6, "p-schedule: src %s:%u dest %s:%u "
278 "mnet %s\n",
279 IP_VS_DBG_ADDR(svc->af, src_addr), ntohs(src_port),
280 IP_VS_DBG_ADDR(svc->af, dst_addr), ntohs(dst_port),
281 IP_VS_DBG_ADDR(svc->af, &snet));
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296 {
297 int protocol = iph->protocol;
298 const union nf_inet_addr *vaddr = dst_addr;
299 __be16 vport = 0;
300
301 if (dst_port == svc->port) {
302
303
304
305
306
307 if (svc->port != FTPPORT)
308 vport = dst_port;
309 } else {
310
311
312
313
314
315
316
317 if (svc->fwmark) {
318 protocol = IPPROTO_IP;
319 vaddr = &fwmark;
320 }
321 }
322
323 if (ip_vs_conn_fill_param_persist(svc, skb, protocol, &snet, 0,
324 vaddr, vport, ¶m) < 0) {
325 *ignored = -1;
326 return NULL;
327 }
328 }
329
330
331 ct = ip_vs_ct_in_get(¶m);
332 if (!ct || !ip_vs_check_template(ct, NULL)) {
333 struct ip_vs_scheduler *sched;
334
335
336
337
338
339
340 sched = rcu_dereference(svc->scheduler);
341 if (sched) {
342
343 smp_rmb();
344 dest = sched->schedule(svc, skb, iph);
345 } else {
346 dest = NULL;
347 }
348 if (!dest) {
349 IP_VS_DBG(1, "p-schedule: no dest found.\n");
350 kfree(param.pe_data);
351 *ignored = 0;
352 return NULL;
353 }
354
355 if (dst_port == svc->port && svc->port != FTPPORT)
356 dport = dest->port;
357
358
359
360
361
362 ct = ip_vs_conn_new(¶m, dest->af, &dest->addr, dport,
363 IP_VS_CONN_F_TEMPLATE, dest, skb->mark);
364 if (ct == NULL) {
365 kfree(param.pe_data);
366 *ignored = -1;
367 return NULL;
368 }
369
370 ct->timeout = svc->timeout;
371 } else {
372
373 dest = ct->dest;
374 kfree(param.pe_data);
375 }
376
377 dport = dst_port;
378 if (dport == svc->port && dest->port)
379 dport = dest->port;
380
381 flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
382 && iph->protocol == IPPROTO_UDP) ?
383 IP_VS_CONN_F_ONE_PACKET : 0;
384
385
386
387
388 ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol, src_addr,
389 src_port, dst_addr, dst_port, ¶m);
390
391 cp = ip_vs_conn_new(¶m, dest->af, &dest->addr, dport, flags, dest,
392 skb->mark);
393 if (cp == NULL) {
394 ip_vs_conn_put(ct);
395 *ignored = -1;
396 return NULL;
397 }
398
399
400
401
402 ip_vs_control_add(cp, ct);
403 ip_vs_conn_put(ct);
404
405 ip_vs_conn_stats(cp, svc);
406 return cp;
407}
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431struct ip_vs_conn *
432ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb,
433 struct ip_vs_proto_data *pd, int *ignored,
434 struct ip_vs_iphdr *iph)
435{
436 struct ip_vs_protocol *pp = pd->pp;
437 struct ip_vs_conn *cp = NULL;
438 struct ip_vs_scheduler *sched;
439 struct ip_vs_dest *dest;
440 __be16 _ports[2], *pptr, cport, vport;
441 const void *caddr, *vaddr;
442 unsigned int flags;
443
444 *ignored = 1;
445
446
447
448 pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports);
449 if (pptr == NULL)
450 return NULL;
451
452 if (likely(!ip_vs_iph_inverse(iph))) {
453 cport = pptr[0];
454 caddr = &iph->saddr;
455 vport = pptr[1];
456 vaddr = &iph->daddr;
457 } else {
458 cport = pptr[1];
459 caddr = &iph->daddr;
460 vport = pptr[0];
461 vaddr = &iph->saddr;
462 }
463
464
465
466
467
468
469
470 if (cport == FTPDATA) {
471 IP_VS_DBG_PKT(12, svc->af, pp, skb, iph->off,
472 "Not scheduling FTPDATA");
473 return NULL;
474 }
475
476
477
478
479 if ((!skb->dev || skb->dev->flags & IFF_LOOPBACK)) {
480 iph->hdr_flags ^= IP_VS_HDR_INVERSE;
481 cp = pp->conn_in_get(svc->ipvs, svc->af, skb, iph);
482 iph->hdr_flags ^= IP_VS_HDR_INVERSE;
483
484 if (cp) {
485 IP_VS_DBG_PKT(12, svc->af, pp, skb, iph->off,
486 "Not scheduling reply for existing"
487 " connection");
488 __ip_vs_conn_put(cp);
489 return NULL;
490 }
491 }
492
493
494
495
496 if (svc->flags & IP_VS_SVC_F_PERSISTENT)
497 return ip_vs_sched_persist(svc, skb, cport, vport, ignored,
498 iph);
499
500 *ignored = 0;
501
502
503
504
505 if (!svc->fwmark && vport != svc->port) {
506 if (!svc->port)
507 pr_err("Schedule: port zero only supported "
508 "in persistent services, "
509 "check your ipvs configuration\n");
510 return NULL;
511 }
512
513 sched = rcu_dereference(svc->scheduler);
514 if (sched) {
515
516 smp_rmb();
517 dest = sched->schedule(svc, skb, iph);
518 } else {
519 dest = NULL;
520 }
521 if (dest == NULL) {
522 IP_VS_DBG(1, "Schedule: no dest found.\n");
523 return NULL;
524 }
525
526 flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
527 && iph->protocol == IPPROTO_UDP) ?
528 IP_VS_CONN_F_ONE_PACKET : 0;
529
530
531
532
533 {
534 struct ip_vs_conn_param p;
535
536 ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
537 caddr, cport, vaddr, vport, &p);
538 cp = ip_vs_conn_new(&p, dest->af, &dest->addr,
539 dest->port ? dest->port : vport,
540 flags, dest, skb->mark);
541 if (!cp) {
542 *ignored = -1;
543 return NULL;
544 }
545 }
546
547 IP_VS_DBG_BUF(6, "Schedule fwd:%c c:%s:%u v:%s:%u "
548 "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
549 ip_vs_fwd_tag(cp),
550 IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
551 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
552 IP_VS_DBG_ADDR(cp->daf, &cp->daddr), ntohs(cp->dport),
553 cp->flags, refcount_read(&cp->refcnt));
554
555 ip_vs_conn_stats(cp, svc);
556 return cp;
557}
558
559static inline int ip_vs_addr_is_unicast(struct net *net, int af,
560 union nf_inet_addr *addr)
561{
562#ifdef CONFIG_IP_VS_IPV6
563 if (af == AF_INET6)
564 return ipv6_addr_type(&addr->in6) & IPV6_ADDR_UNICAST;
565#endif
566 return (inet_addr_type(net, addr->ip) == RTN_UNICAST);
567}
568
569
570
571
572
573
574int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
575 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph)
576{
577 __be16 _ports[2], *pptr, dport;
578 struct netns_ipvs *ipvs = svc->ipvs;
579 struct net *net = ipvs->net;
580
581 pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports);
582 if (!pptr)
583 return NF_DROP;
584 dport = likely(!ip_vs_iph_inverse(iph)) ? pptr[1] : pptr[0];
585
586
587
588
589 if (sysctl_cache_bypass(ipvs) && svc->fwmark &&
590 !(iph->hdr_flags & (IP_VS_HDR_INVERSE | IP_VS_HDR_ICMP)) &&
591 ip_vs_addr_is_unicast(net, svc->af, &iph->daddr)) {
592 int ret;
593 struct ip_vs_conn *cp;
594 unsigned int flags = (svc->flags & IP_VS_SVC_F_ONEPACKET &&
595 iph->protocol == IPPROTO_UDP) ?
596 IP_VS_CONN_F_ONE_PACKET : 0;
597 union nf_inet_addr daddr = { .all = { 0, 0, 0, 0 } };
598
599
600 IP_VS_DBG(6, "%s(): create a cache_bypass entry\n", __func__);
601 {
602 struct ip_vs_conn_param p;
603 ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
604 &iph->saddr, pptr[0],
605 &iph->daddr, pptr[1], &p);
606 cp = ip_vs_conn_new(&p, svc->af, &daddr, 0,
607 IP_VS_CONN_F_BYPASS | flags,
608 NULL, skb->mark);
609 if (!cp)
610 return NF_DROP;
611 }
612
613
614 ip_vs_in_stats(cp, skb);
615
616
617 ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
618
619
620 ret = cp->packet_xmit(skb, cp, pd->pp, iph);
621
622
623 if ((cp->flags & IP_VS_CONN_F_ONE_PACKET) && cp->control)
624 atomic_inc(&cp->control->in_pkts);
625 else
626 atomic_inc(&cp->in_pkts);
627 ip_vs_conn_put(cp);
628 return ret;
629 }
630
631
632
633
634
635
636
637 if (svc->port == FTPPORT && dport != FTPPORT)
638 return NF_ACCEPT;
639
640 if (unlikely(ip_vs_iph_icmp(iph)))
641 return NF_DROP;
642
643
644
645
646
647
648
649
650#ifdef CONFIG_IP_VS_IPV6
651 if (svc->af == AF_INET6) {
652 if (!skb->dev)
653 skb->dev = net->loopback_dev;
654 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
655 } else
656#endif
657 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
658
659 return NF_DROP;
660}
661
662#ifdef CONFIG_SYSCTL
663
664static int sysctl_snat_reroute(struct netns_ipvs *ipvs)
665{
666 return ipvs->sysctl_snat_reroute;
667}
668
669static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs)
670{
671 return ipvs->sysctl_nat_icmp_send;
672}
673
674static int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs)
675{
676 return ipvs->sysctl_expire_nodest_conn;
677}
678
679#else
680
681static int sysctl_snat_reroute(struct netns_ipvs *ipvs) { return 0; }
682static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs) { return 0; }
683static int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) { return 0; }
684
685#endif
686
687__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset)
688{
689 return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0));
690}
691
692static inline enum ip_defrag_users ip_vs_defrag_user(unsigned int hooknum)
693{
694 if (NF_INET_LOCAL_IN == hooknum)
695 return IP_DEFRAG_VS_IN;
696 if (NF_INET_FORWARD == hooknum)
697 return IP_DEFRAG_VS_FWD;
698 return IP_DEFRAG_VS_OUT;
699}
700
701static inline int ip_vs_gather_frags(struct netns_ipvs *ipvs,
702 struct sk_buff *skb, u_int32_t user)
703{
704 int err;
705
706 local_bh_disable();
707 err = ip_defrag(ipvs->net, skb, user);
708 local_bh_enable();
709 if (!err)
710 ip_send_check(ip_hdr(skb));
711
712 return err;
713}
714
715static int ip_vs_route_me_harder(struct netns_ipvs *ipvs, int af,
716 struct sk_buff *skb, unsigned int hooknum)
717{
718 if (!sysctl_snat_reroute(ipvs))
719 return 0;
720
721 if (NF_INET_LOCAL_IN == hooknum)
722 return 0;
723#ifdef CONFIG_IP_VS_IPV6
724 if (af == AF_INET6) {
725 struct dst_entry *dst = skb_dst(skb);
726
727 if (dst->dev && !(dst->dev->flags & IFF_LOOPBACK) &&
728 ip6_route_me_harder(ipvs->net, skb) != 0)
729 return 1;
730 } else
731#endif
732 if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL) &&
733 ip_route_me_harder(ipvs->net, skb, RTN_LOCAL) != 0)
734 return 1;
735
736 return 0;
737}
738
739
740
741
742
743void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
744 struct ip_vs_conn *cp, int inout)
745{
746 struct iphdr *iph = ip_hdr(skb);
747 unsigned int icmp_offset = iph->ihl*4;
748 struct icmphdr *icmph = (struct icmphdr *)(skb_network_header(skb) +
749 icmp_offset);
750 struct iphdr *ciph = (struct iphdr *)(icmph + 1);
751
752 if (inout) {
753 iph->saddr = cp->vaddr.ip;
754 ip_send_check(iph);
755 ciph->daddr = cp->vaddr.ip;
756 ip_send_check(ciph);
757 } else {
758 iph->daddr = cp->daddr.ip;
759 ip_send_check(iph);
760 ciph->saddr = cp->daddr.ip;
761 ip_send_check(ciph);
762 }
763
764
765 if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol ||
766 IPPROTO_SCTP == ciph->protocol) {
767 __be16 *ports = (void *)ciph + ciph->ihl*4;
768
769 if (inout)
770 ports[1] = cp->vport;
771 else
772 ports[0] = cp->dport;
773 }
774
775
776 icmph->checksum = 0;
777 icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
778 skb->ip_summed = CHECKSUM_UNNECESSARY;
779
780 if (inout)
781 IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
782 "Forwarding altered outgoing ICMP");
783 else
784 IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
785 "Forwarding altered incoming ICMP");
786}
787
788#ifdef CONFIG_IP_VS_IPV6
789void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp,
790 struct ip_vs_conn *cp, int inout)
791{
792 struct ipv6hdr *iph = ipv6_hdr(skb);
793 unsigned int icmp_offset = 0;
794 unsigned int offs = 0;
795 int protocol;
796 struct icmp6hdr *icmph;
797 struct ipv6hdr *ciph;
798 unsigned short fragoffs;
799
800 ipv6_find_hdr(skb, &icmp_offset, IPPROTO_ICMPV6, &fragoffs, NULL);
801 icmph = (struct icmp6hdr *)(skb_network_header(skb) + icmp_offset);
802 offs = icmp_offset + sizeof(struct icmp6hdr);
803 ciph = (struct ipv6hdr *)(skb_network_header(skb) + offs);
804
805 protocol = ipv6_find_hdr(skb, &offs, -1, &fragoffs, NULL);
806
807 if (inout) {
808 iph->saddr = cp->vaddr.in6;
809 ciph->daddr = cp->vaddr.in6;
810 } else {
811 iph->daddr = cp->daddr.in6;
812 ciph->saddr = cp->daddr.in6;
813 }
814
815
816 if (!fragoffs && (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
817 IPPROTO_SCTP == protocol)) {
818 __be16 *ports = (void *)(skb_network_header(skb) + offs);
819
820 IP_VS_DBG(11, "%s() changed port %d to %d\n", __func__,
821 ntohs(inout ? ports[1] : ports[0]),
822 ntohs(inout ? cp->vport : cp->dport));
823 if (inout)
824 ports[1] = cp->vport;
825 else
826 ports[0] = cp->dport;
827 }
828
829
830 icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr,
831 skb->len - icmp_offset,
832 IPPROTO_ICMPV6, 0);
833 skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset;
834 skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
835 skb->ip_summed = CHECKSUM_PARTIAL;
836
837 if (inout)
838 IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
839 (void *)ciph - (void *)iph,
840 "Forwarding altered outgoing ICMPv6");
841 else
842 IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
843 (void *)ciph - (void *)iph,
844 "Forwarding altered incoming ICMPv6");
845}
846#endif
847
848
849
850
851static int handle_response_icmp(int af, struct sk_buff *skb,
852 union nf_inet_addr *snet,
853 __u8 protocol, struct ip_vs_conn *cp,
854 struct ip_vs_protocol *pp,
855 unsigned int offset, unsigned int ihl,
856 unsigned int hooknum)
857{
858 unsigned int verdict = NF_DROP;
859
860 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
861 goto ignore_cp;
862
863
864 if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
865
866 IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n",
867 IP_VS_DBG_ADDR(af, snet));
868 goto out;
869 }
870
871 if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
872 IPPROTO_SCTP == protocol)
873 offset += 2 * sizeof(__u16);
874 if (!skb_make_writable(skb, offset))
875 goto out;
876
877#ifdef CONFIG_IP_VS_IPV6
878 if (af == AF_INET6)
879 ip_vs_nat_icmp_v6(skb, pp, cp, 1);
880 else
881#endif
882 ip_vs_nat_icmp(skb, pp, cp, 1);
883
884 if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum))
885 goto out;
886
887
888 ip_vs_out_stats(cp, skb);
889
890 skb->ipvs_property = 1;
891 if (!(cp->flags & IP_VS_CONN_F_NFCT))
892 ip_vs_notrack(skb);
893 else
894 ip_vs_update_conntrack(skb, cp, 0);
895
896ignore_cp:
897 verdict = NF_ACCEPT;
898
899out:
900 __ip_vs_conn_put(cp);
901
902 return verdict;
903}
904
905
906
907
908
909
910static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb,
911 int *related, unsigned int hooknum)
912{
913 struct iphdr *iph;
914 struct icmphdr _icmph, *ic;
915 struct iphdr _ciph, *cih;
916 struct ip_vs_iphdr ciph;
917 struct ip_vs_conn *cp;
918 struct ip_vs_protocol *pp;
919 unsigned int offset, ihl;
920 union nf_inet_addr snet;
921
922 *related = 1;
923
924
925 if (ip_is_fragment(ip_hdr(skb))) {
926 if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum)))
927 return NF_STOLEN;
928 }
929
930 iph = ip_hdr(skb);
931 offset = ihl = iph->ihl * 4;
932 ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
933 if (ic == NULL)
934 return NF_DROP;
935
936 IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n",
937 ic->type, ntohs(icmp_id(ic)),
938 &iph->saddr, &iph->daddr);
939
940
941
942
943
944
945
946
947 if ((ic->type != ICMP_DEST_UNREACH) &&
948 (ic->type != ICMP_SOURCE_QUENCH) &&
949 (ic->type != ICMP_TIME_EXCEEDED)) {
950 *related = 0;
951 return NF_ACCEPT;
952 }
953
954
955 offset += sizeof(_icmph);
956 cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
957 if (cih == NULL)
958 return NF_ACCEPT;
959
960 pp = ip_vs_proto_get(cih->protocol);
961 if (!pp)
962 return NF_ACCEPT;
963
964
965 if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
966 pp->dont_defrag))
967 return NF_ACCEPT;
968
969 IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
970 "Checking outgoing ICMP for");
971
972 ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph);
973
974
975 cp = pp->conn_out_get(ipvs, AF_INET, skb, &ciph);
976 if (!cp)
977 return NF_ACCEPT;
978
979 snet.ip = iph->saddr;
980 return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp,
981 pp, ciph.len, ihl, hooknum);
982}
983
984#ifdef CONFIG_IP_VS_IPV6
985static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
986 int *related, unsigned int hooknum,
987 struct ip_vs_iphdr *ipvsh)
988{
989 struct icmp6hdr _icmph, *ic;
990 struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};
991 struct ip_vs_conn *cp;
992 struct ip_vs_protocol *pp;
993 union nf_inet_addr snet;
994 unsigned int offset;
995
996 *related = 1;
997 ic = frag_safe_skb_hp(skb, ipvsh->len, sizeof(_icmph), &_icmph);
998 if (ic == NULL)
999 return NF_DROP;
1000
1001
1002
1003
1004
1005
1006
1007
1008 if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
1009 *related = 0;
1010 return NF_ACCEPT;
1011 }
1012
1013
1014
1015 if (ipvsh->flags & IP6_FH_F_FRAG)
1016 return NF_DROP;
1017
1018 IP_VS_DBG(8, "Outgoing ICMPv6 (%d,%d) %pI6c->%pI6c\n",
1019 ic->icmp6_type, ntohs(icmpv6_id(ic)),
1020 &ipvsh->saddr, &ipvsh->daddr);
1021
1022 if (!ip_vs_fill_iph_skb_icmp(AF_INET6, skb, ipvsh->len + sizeof(_icmph),
1023 true, &ciph))
1024 return NF_ACCEPT;
1025
1026 pp = ip_vs_proto_get(ciph.protocol);
1027 if (!pp)
1028 return NF_ACCEPT;
1029
1030
1031 cp = pp->conn_out_get(ipvs, AF_INET6, skb, &ciph);
1032 if (!cp)
1033 return NF_ACCEPT;
1034
1035 snet.in6 = ciph.saddr.in6;
1036 offset = ciph.len;
1037 return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp,
1038 pp, offset, sizeof(struct ipv6hdr),
1039 hooknum);
1040}
1041#endif
1042
1043
1044
1045
1046static inline int is_sctp_abort(const struct sk_buff *skb, int nh_len)
1047{
1048 struct sctp_chunkhdr *sch, schunk;
1049 sch = skb_header_pointer(skb, nh_len + sizeof(struct sctphdr),
1050 sizeof(schunk), &schunk);
1051 if (sch == NULL)
1052 return 0;
1053 if (sch->type == SCTP_CID_ABORT)
1054 return 1;
1055 return 0;
1056}
1057
1058static inline int is_tcp_reset(const struct sk_buff *skb, int nh_len)
1059{
1060 struct tcphdr _tcph, *th;
1061
1062 th = skb_header_pointer(skb, nh_len, sizeof(_tcph), &_tcph);
1063 if (th == NULL)
1064 return 0;
1065 return th->rst;
1066}
1067
1068static inline bool is_new_conn(const struct sk_buff *skb,
1069 struct ip_vs_iphdr *iph)
1070{
1071 switch (iph->protocol) {
1072 case IPPROTO_TCP: {
1073 struct tcphdr _tcph, *th;
1074
1075 th = skb_header_pointer(skb, iph->len, sizeof(_tcph), &_tcph);
1076 if (th == NULL)
1077 return false;
1078 return th->syn;
1079 }
1080 case IPPROTO_SCTP: {
1081 struct sctp_chunkhdr *sch, schunk;
1082
1083 sch = skb_header_pointer(skb, iph->len + sizeof(struct sctphdr),
1084 sizeof(schunk), &schunk);
1085 if (sch == NULL)
1086 return false;
1087 return sch->type == SCTP_CID_INIT;
1088 }
1089 default:
1090 return false;
1091 }
1092}
1093
1094static inline bool is_new_conn_expected(const struct ip_vs_conn *cp,
1095 int conn_reuse_mode)
1096{
1097
1098 if (cp->control)
1099 return false;
1100
1101 switch (cp->protocol) {
1102 case IPPROTO_TCP:
1103 return (cp->state == IP_VS_TCP_S_TIME_WAIT) ||
1104 (cp->state == IP_VS_TCP_S_CLOSE) ||
1105 ((conn_reuse_mode & 2) &&
1106 (cp->state == IP_VS_TCP_S_FIN_WAIT) &&
1107 (cp->flags & IP_VS_CONN_F_NOOUTPUT));
1108 case IPPROTO_SCTP:
1109 return cp->state == IP_VS_SCTP_S_CLOSED;
1110 default:
1111 return false;
1112 }
1113}
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc,
1124 struct ip_vs_dest *dest,
1125 struct sk_buff *skb,
1126 const struct ip_vs_iphdr *iph,
1127 __be16 dport,
1128 __be16 cport)
1129{
1130 struct ip_vs_conn_param param;
1131 struct ip_vs_conn *ct = NULL, *cp = NULL;
1132 const union nf_inet_addr *vaddr, *daddr, *caddr;
1133 union nf_inet_addr snet;
1134 __be16 vport;
1135 unsigned int flags;
1136
1137 EnterFunction(12);
1138 vaddr = &svc->addr;
1139 vport = svc->port;
1140 daddr = &iph->saddr;
1141 caddr = &iph->daddr;
1142
1143
1144 if (svc->fwmark)
1145 return NULL;
1146 if (!vport || !dport)
1147 return NULL;
1148
1149
1150 if (svc->flags & IP_VS_SVC_F_PERSISTENT) {
1151
1152#ifdef CONFIG_IP_VS_IPV6
1153 if (svc->af == AF_INET6)
1154 ipv6_addr_prefix(&snet.in6, &caddr->in6,
1155 (__force __u32)svc->netmask);
1156 else
1157#endif
1158 snet.ip = caddr->ip & svc->netmask;
1159
1160 if (ip_vs_conn_fill_param_persist(svc, skb, iph->protocol,
1161 &snet, 0, vaddr,
1162 vport, ¶m) < 0)
1163 return NULL;
1164 ct = ip_vs_ct_in_get(¶m);
1165
1166 if (!ct || !ip_vs_check_template(ct, dest)) {
1167 ct = ip_vs_conn_new(¶m, dest->af, daddr, dport,
1168 IP_VS_CONN_F_TEMPLATE, dest, 0);
1169 if (!ct) {
1170 kfree(param.pe_data);
1171 return NULL;
1172 }
1173 ct->timeout = svc->timeout;
1174 } else {
1175 kfree(param.pe_data);
1176 }
1177 }
1178
1179
1180 flags = ((svc->flags & IP_VS_SVC_F_ONEPACKET) &&
1181 iph->protocol == IPPROTO_UDP) ? IP_VS_CONN_F_ONE_PACKET : 0;
1182
1183 ip_vs_conn_fill_param(svc->ipvs, svc->af, iph->protocol,
1184 caddr, cport, vaddr, vport, ¶m);
1185 cp = ip_vs_conn_new(¶m, dest->af, daddr, dport, flags, dest, 0);
1186 if (!cp) {
1187 if (ct)
1188 ip_vs_conn_put(ct);
1189 return NULL;
1190 }
1191 if (ct) {
1192 ip_vs_control_add(cp, ct);
1193 ip_vs_conn_put(ct);
1194 }
1195 ip_vs_conn_stats(cp, svc);
1196
1197
1198 IP_VS_DBG_BUF(6, "New connection RS-initiated:%c c:%s:%u v:%s:%u "
1199 "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
1200 ip_vs_fwd_tag(cp),
1201 IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
1202 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
1203 IP_VS_DBG_ADDR(cp->af, &cp->daddr), ntohs(cp->dport),
1204 cp->flags, refcount_read(&cp->refcnt));
1205 LeaveFunction(12);
1206 return cp;
1207}
1208
1209
1210
1211
1212
1213
1214
1215
1216static struct ip_vs_conn *__ip_vs_rs_conn_out(unsigned int hooknum,
1217 struct netns_ipvs *ipvs,
1218 int af, struct sk_buff *skb,
1219 const struct ip_vs_iphdr *iph)
1220{
1221 struct ip_vs_dest *dest;
1222 struct ip_vs_conn *cp = NULL;
1223 __be16 _ports[2], *pptr;
1224
1225 if (hooknum == NF_INET_LOCAL_IN)
1226 return NULL;
1227
1228 pptr = frag_safe_skb_hp(skb, iph->len,
1229 sizeof(_ports), _ports);
1230 if (!pptr)
1231 return NULL;
1232
1233 dest = ip_vs_find_real_service(ipvs, af, iph->protocol,
1234 &iph->saddr, pptr[0]);
1235 if (dest) {
1236 struct ip_vs_service *svc;
1237 struct ip_vs_pe *pe;
1238
1239 svc = rcu_dereference(dest->svc);
1240 if (svc) {
1241 pe = rcu_dereference(svc->pe);
1242 if (pe && pe->conn_out)
1243 cp = pe->conn_out(svc, dest, skb, iph,
1244 pptr[0], pptr[1]);
1245 }
1246 }
1247
1248 return cp;
1249}
1250
1251
1252
1253static unsigned int
1254handle_response(int af, struct sk_buff *skb, struct ip_vs_proto_data *pd,
1255 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph,
1256 unsigned int hooknum)
1257{
1258 struct ip_vs_protocol *pp = pd->pp;
1259
1260 IP_VS_DBG_PKT(11, af, pp, skb, iph->off, "Outgoing packet");
1261
1262 if (!skb_make_writable(skb, iph->len))
1263 goto drop;
1264
1265
1266 if (pp->snat_handler && !pp->snat_handler(skb, pp, cp, iph))
1267 goto drop;
1268
1269#ifdef CONFIG_IP_VS_IPV6
1270 if (af == AF_INET6)
1271 ipv6_hdr(skb)->saddr = cp->vaddr.in6;
1272 else
1273#endif
1274 {
1275 ip_hdr(skb)->saddr = cp->vaddr.ip;
1276 ip_send_check(ip_hdr(skb));
1277 }
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294 if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum))
1295 goto drop;
1296
1297 IP_VS_DBG_PKT(10, af, pp, skb, iph->off, "After SNAT");
1298
1299 ip_vs_out_stats(cp, skb);
1300 ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd);
1301 skb->ipvs_property = 1;
1302 if (!(cp->flags & IP_VS_CONN_F_NFCT))
1303 ip_vs_notrack(skb);
1304 else
1305 ip_vs_update_conntrack(skb, cp, 0);
1306 ip_vs_conn_put(cp);
1307
1308 LeaveFunction(11);
1309 return NF_ACCEPT;
1310
1311drop:
1312 ip_vs_conn_put(cp);
1313 kfree_skb(skb);
1314 LeaveFunction(11);
1315 return NF_STOLEN;
1316}
1317
1318
1319
1320
1321static unsigned int
1322ip_vs_out(struct netns_ipvs *ipvs, unsigned int hooknum, struct sk_buff *skb, int af)
1323{
1324 struct ip_vs_iphdr iph;
1325 struct ip_vs_protocol *pp;
1326 struct ip_vs_proto_data *pd;
1327 struct ip_vs_conn *cp;
1328 struct sock *sk;
1329
1330 EnterFunction(11);
1331
1332
1333 if (skb->ipvs_property)
1334 return NF_ACCEPT;
1335
1336 sk = skb_to_full_sk(skb);
1337
1338 if (unlikely(sk && hooknum == NF_INET_LOCAL_OUT &&
1339 af == AF_INET)) {
1340
1341 if (sk->sk_family == PF_INET && inet_sk(sk)->nodefrag)
1342 return NF_ACCEPT;
1343 }
1344
1345 if (unlikely(!skb_dst(skb)))
1346 return NF_ACCEPT;
1347
1348 if (!ipvs->enable)
1349 return NF_ACCEPT;
1350
1351 ip_vs_fill_iph_skb(af, skb, false, &iph);
1352#ifdef CONFIG_IP_VS_IPV6
1353 if (af == AF_INET6) {
1354 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1355 int related;
1356 int verdict = ip_vs_out_icmp_v6(ipvs, skb, &related,
1357 hooknum, &iph);
1358
1359 if (related)
1360 return verdict;
1361 }
1362 } else
1363#endif
1364 if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1365 int related;
1366 int verdict = ip_vs_out_icmp(ipvs, skb, &related, hooknum);
1367
1368 if (related)
1369 return verdict;
1370 }
1371
1372 pd = ip_vs_proto_data_get(ipvs, iph.protocol);
1373 if (unlikely(!pd))
1374 return NF_ACCEPT;
1375 pp = pd->pp;
1376
1377
1378#ifdef CONFIG_IP_VS_IPV6
1379 if (af == AF_INET)
1380#endif
1381 if (unlikely(ip_is_fragment(ip_hdr(skb)) && !pp->dont_defrag)) {
1382 if (ip_vs_gather_frags(ipvs, skb,
1383 ip_vs_defrag_user(hooknum)))
1384 return NF_STOLEN;
1385
1386 ip_vs_fill_iph_skb(AF_INET, skb, false, &iph);
1387 }
1388
1389
1390
1391
1392 cp = pp->conn_out_get(ipvs, af, skb, &iph);
1393
1394 if (likely(cp)) {
1395 if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
1396 goto ignore_cp;
1397 return handle_response(af, skb, pd, cp, &iph, hooknum);
1398 }
1399
1400
1401 if (atomic_read(&ipvs->conn_out_counter)) {
1402
1403
1404
1405
1406
1407 if (pp->protocol == IPPROTO_UDP) {
1408 cp = __ip_vs_rs_conn_out(hooknum, ipvs, af, skb, &iph);
1409 if (likely(cp))
1410 return handle_response(af, skb, pd, cp, &iph,
1411 hooknum);
1412 }
1413 }
1414
1415 if (sysctl_nat_icmp_send(ipvs) &&
1416 (pp->protocol == IPPROTO_TCP ||
1417 pp->protocol == IPPROTO_UDP ||
1418 pp->protocol == IPPROTO_SCTP)) {
1419 __be16 _ports[2], *pptr;
1420
1421 pptr = frag_safe_skb_hp(skb, iph.len,
1422 sizeof(_ports), _ports);
1423 if (pptr == NULL)
1424 return NF_ACCEPT;
1425 if (ip_vs_has_real_service(ipvs, af, iph.protocol, &iph.saddr,
1426 pptr[0])) {
1427
1428
1429
1430
1431
1432 if ((iph.protocol != IPPROTO_TCP &&
1433 iph.protocol != IPPROTO_SCTP)
1434 || ((iph.protocol == IPPROTO_TCP
1435 && !is_tcp_reset(skb, iph.len))
1436 || (iph.protocol == IPPROTO_SCTP
1437 && !is_sctp_abort(skb,
1438 iph.len)))) {
1439#ifdef CONFIG_IP_VS_IPV6
1440 if (af == AF_INET6) {
1441 if (!skb->dev)
1442 skb->dev = ipvs->net->loopback_dev;
1443 icmpv6_send(skb,
1444 ICMPV6_DEST_UNREACH,
1445 ICMPV6_PORT_UNREACH,
1446 0);
1447 } else
1448#endif
1449 icmp_send(skb,
1450 ICMP_DEST_UNREACH,
1451 ICMP_PORT_UNREACH, 0);
1452 return NF_DROP;
1453 }
1454 }
1455 }
1456
1457out:
1458 IP_VS_DBG_PKT(12, af, pp, skb, iph.off,
1459 "ip_vs_out: packet continues traversal as normal");
1460 return NF_ACCEPT;
1461
1462ignore_cp:
1463 __ip_vs_conn_put(cp);
1464 goto out;
1465}
1466
1467
1468
1469
1470
1471
1472static unsigned int
1473ip_vs_reply4(void *priv, struct sk_buff *skb,
1474 const struct nf_hook_state *state)
1475{
1476 return ip_vs_out(net_ipvs(state->net), state->hook, skb, AF_INET);
1477}
1478
1479
1480
1481
1482
1483static unsigned int
1484ip_vs_local_reply4(void *priv, struct sk_buff *skb,
1485 const struct nf_hook_state *state)
1486{
1487 return ip_vs_out(net_ipvs(state->net), state->hook, skb, AF_INET);
1488}
1489
1490#ifdef CONFIG_IP_VS_IPV6
1491
1492
1493
1494
1495
1496
1497static unsigned int
1498ip_vs_reply6(void *priv, struct sk_buff *skb,
1499 const struct nf_hook_state *state)
1500{
1501 return ip_vs_out(net_ipvs(state->net), state->hook, skb, AF_INET6);
1502}
1503
1504
1505
1506
1507
1508static unsigned int
1509ip_vs_local_reply6(void *priv, struct sk_buff *skb,
1510 const struct nf_hook_state *state)
1511{
1512 return ip_vs_out(net_ipvs(state->net), state->hook, skb, AF_INET6);
1513}
1514
1515#endif
1516
1517static unsigned int
1518ip_vs_try_to_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb,
1519 struct ip_vs_proto_data *pd,
1520 int *verdict, struct ip_vs_conn **cpp,
1521 struct ip_vs_iphdr *iph)
1522{
1523 struct ip_vs_protocol *pp = pd->pp;
1524
1525 if (!iph->fragoffs) {
1526
1527
1528
1529
1530
1531 if (!pp->conn_schedule(ipvs, af, skb, pd, verdict, cpp, iph))
1532 return 0;
1533 }
1534
1535 if (unlikely(!*cpp)) {
1536
1537 IP_VS_DBG_PKT(12, af, pp, skb, iph->off,
1538 "ip_vs_in: packet continues traversal as normal");
1539 if (iph->fragoffs) {
1540
1541
1542
1543 IP_VS_DBG_RL("Unhandled frag, load nf_defrag_ipv6\n");
1544 IP_VS_DBG_PKT(7, af, pp, skb, iph->off,
1545 "unhandled fragment");
1546 }
1547 *verdict = NF_ACCEPT;
1548 return 0;
1549 }
1550
1551 return 1;
1552}
1553
1554
1555
1556
1557
1558
1559
1560static int
1561ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related,
1562 unsigned int hooknum)
1563{
1564 struct iphdr *iph;
1565 struct icmphdr _icmph, *ic;
1566 struct iphdr _ciph, *cih;
1567 struct ip_vs_iphdr ciph;
1568 struct ip_vs_conn *cp;
1569 struct ip_vs_protocol *pp;
1570 struct ip_vs_proto_data *pd;
1571 unsigned int offset, offset2, ihl, verdict;
1572 bool ipip, new_cp = false;
1573
1574 *related = 1;
1575
1576
1577 if (ip_is_fragment(ip_hdr(skb))) {
1578 if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum)))
1579 return NF_STOLEN;
1580 }
1581
1582 iph = ip_hdr(skb);
1583 offset = ihl = iph->ihl * 4;
1584 ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
1585 if (ic == NULL)
1586 return NF_DROP;
1587
1588 IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n",
1589 ic->type, ntohs(icmp_id(ic)),
1590 &iph->saddr, &iph->daddr);
1591
1592
1593
1594
1595
1596
1597
1598
1599 if ((ic->type != ICMP_DEST_UNREACH) &&
1600 (ic->type != ICMP_SOURCE_QUENCH) &&
1601 (ic->type != ICMP_TIME_EXCEEDED)) {
1602 *related = 0;
1603 return NF_ACCEPT;
1604 }
1605
1606
1607 offset += sizeof(_icmph);
1608 cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1609 if (cih == NULL)
1610 return NF_ACCEPT;
1611
1612
1613 ipip = false;
1614 if (cih->protocol == IPPROTO_IPIP) {
1615 if (unlikely(cih->frag_off & htons(IP_OFFSET)))
1616 return NF_ACCEPT;
1617
1618 if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL))
1619 return NF_ACCEPT;
1620 offset += cih->ihl * 4;
1621 cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1622 if (cih == NULL)
1623 return NF_ACCEPT;
1624 ipip = true;
1625 }
1626
1627 pd = ip_vs_proto_data_get(ipvs, cih->protocol);
1628 if (!pd)
1629 return NF_ACCEPT;
1630 pp = pd->pp;
1631
1632
1633 if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
1634 pp->dont_defrag))
1635 return NF_ACCEPT;
1636
1637 IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
1638 "Checking incoming ICMP for");
1639
1640 offset2 = offset;
1641 ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !ipip, &ciph);
1642 offset = ciph.len;
1643
1644
1645
1646
1647 cp = pp->conn_in_get(ipvs, AF_INET, skb, &ciph);
1648
1649 if (!cp) {
1650 int v;
1651
1652 if (!sysctl_schedule_icmp(ipvs))
1653 return NF_ACCEPT;
1654
1655 if (!ip_vs_try_to_schedule(ipvs, AF_INET, skb, pd, &v, &cp, &ciph))
1656 return v;
1657 new_cp = true;
1658 }
1659
1660 verdict = NF_DROP;
1661
1662
1663 if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
1664
1665 IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n",
1666 &iph->saddr);
1667 goto out;
1668 }
1669
1670 if (ipip) {
1671 __be32 info = ic->un.gateway;
1672 __u8 type = ic->type;
1673 __u8 code = ic->code;
1674
1675
1676 if (ic->type == ICMP_DEST_UNREACH &&
1677 ic->code == ICMP_FRAG_NEEDED) {
1678 struct ip_vs_dest *dest = cp->dest;
1679 u32 mtu = ntohs(ic->un.frag.mtu);
1680 __be16 frag_off = cih->frag_off;
1681
1682
1683 if (pskb_pull(skb, ihl + sizeof(_icmph)) == NULL)
1684 goto ignore_ipip;
1685 offset2 -= ihl + sizeof(_icmph);
1686 skb_reset_network_header(skb);
1687 IP_VS_DBG(12, "ICMP for IPIP %pI4->%pI4: mtu=%u\n",
1688 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, mtu);
1689 ipv4_update_pmtu(skb, ipvs->net,
1690 mtu, 0, 0, 0, 0);
1691
1692 if (!(frag_off & htons(IP_DF)))
1693 goto ignore_ipip;
1694
1695 if (dest) {
1696 struct ip_vs_dest_dst *dest_dst;
1697
1698 dest_dst = rcu_dereference(dest->dest_dst);
1699 if (dest_dst)
1700 mtu = dst_mtu(dest_dst->dst_cache);
1701 }
1702 if (mtu > 68 + sizeof(struct iphdr))
1703 mtu -= sizeof(struct iphdr);
1704 info = htonl(mtu);
1705 }
1706
1707
1708
1709 if (pskb_pull(skb, offset2) == NULL)
1710 goto ignore_ipip;
1711 skb_reset_network_header(skb);
1712 IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n",
1713 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1714 type, code, ntohl(info));
1715 icmp_send(skb, type, code, info);
1716
1717 ip_vs_out_stats(cp, skb);
1718
1719ignore_ipip:
1720 consume_skb(skb);
1721 verdict = NF_STOLEN;
1722 goto out;
1723 }
1724
1725
1726 ip_vs_in_stats(cp, skb);
1727 if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol ||
1728 IPPROTO_SCTP == cih->protocol)
1729 offset += 2 * sizeof(__u16);
1730 verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph);
1731
1732out:
1733 if (likely(!new_cp))
1734 __ip_vs_conn_put(cp);
1735 else
1736 ip_vs_conn_put(cp);
1737
1738 return verdict;
1739}
1740
1741#ifdef CONFIG_IP_VS_IPV6
1742static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
1743 int *related, unsigned int hooknum,
1744 struct ip_vs_iphdr *iph)
1745{
1746 struct icmp6hdr _icmph, *ic;
1747 struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};
1748 struct ip_vs_conn *cp;
1749 struct ip_vs_protocol *pp;
1750 struct ip_vs_proto_data *pd;
1751 unsigned int offset, verdict;
1752 bool new_cp = false;
1753
1754 *related = 1;
1755
1756 ic = frag_safe_skb_hp(skb, iph->len, sizeof(_icmph), &_icmph);
1757 if (ic == NULL)
1758 return NF_DROP;
1759
1760
1761
1762
1763
1764
1765
1766
1767 if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
1768 *related = 0;
1769 return NF_ACCEPT;
1770 }
1771
1772
1773
1774 if (iph->flags & IP6_FH_F_FRAG)
1775 return NF_DROP;
1776
1777 IP_VS_DBG(8, "Incoming ICMPv6 (%d,%d) %pI6c->%pI6c\n",
1778 ic->icmp6_type, ntohs(icmpv6_id(ic)),
1779 &iph->saddr, &iph->daddr);
1780
1781 offset = iph->len + sizeof(_icmph);
1782 if (!ip_vs_fill_iph_skb_icmp(AF_INET6, skb, offset, true, &ciph))
1783 return NF_ACCEPT;
1784
1785 pd = ip_vs_proto_data_get(ipvs, ciph.protocol);
1786 if (!pd)
1787 return NF_ACCEPT;
1788 pp = pd->pp;
1789
1790
1791 if (ciph.fragoffs)
1792 return NF_ACCEPT;
1793
1794 IP_VS_DBG_PKT(11, AF_INET6, pp, skb, offset,
1795 "Checking incoming ICMPv6 for");
1796
1797
1798
1799
1800 cp = pp->conn_in_get(ipvs, AF_INET6, skb, &ciph);
1801
1802 if (!cp) {
1803 int v;
1804
1805 if (!sysctl_schedule_icmp(ipvs))
1806 return NF_ACCEPT;
1807
1808 if (!ip_vs_try_to_schedule(ipvs, AF_INET6, skb, pd, &v, &cp, &ciph))
1809 return v;
1810
1811 new_cp = true;
1812 }
1813
1814
1815 if ((hooknum == NF_INET_LOCAL_OUT) &&
1816 (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)) {
1817 verdict = NF_ACCEPT;
1818 goto out;
1819 }
1820
1821
1822 ip_vs_in_stats(cp, skb);
1823
1824
1825 offset = ciph.len;
1826 if (IPPROTO_TCP == ciph.protocol || IPPROTO_UDP == ciph.protocol ||
1827 IPPROTO_SCTP == ciph.protocol)
1828 offset += 2 * sizeof(__u16);
1829
1830 verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, offset, hooknum, &ciph);
1831
1832out:
1833 if (likely(!new_cp))
1834 __ip_vs_conn_put(cp);
1835 else
1836 ip_vs_conn_put(cp);
1837
1838 return verdict;
1839}
1840#endif
1841
1842
1843
1844
1845
1846
1847static unsigned int
1848ip_vs_in(struct netns_ipvs *ipvs, unsigned int hooknum, struct sk_buff *skb, int af)
1849{
1850 struct ip_vs_iphdr iph;
1851 struct ip_vs_protocol *pp;
1852 struct ip_vs_proto_data *pd;
1853 struct ip_vs_conn *cp;
1854 int ret, pkts;
1855 int conn_reuse_mode;
1856 struct sock *sk;
1857
1858
1859 if (skb->ipvs_property)
1860 return NF_ACCEPT;
1861
1862
1863
1864
1865
1866
1867 if (unlikely((skb->pkt_type != PACKET_HOST &&
1868 hooknum != NF_INET_LOCAL_OUT) ||
1869 !skb_dst(skb))) {
1870 ip_vs_fill_iph_skb(af, skb, false, &iph);
1871 IP_VS_DBG_BUF(12, "packet type=%d proto=%d daddr=%s"
1872 " ignored in hook %u\n",
1873 skb->pkt_type, iph.protocol,
1874 IP_VS_DBG_ADDR(af, &iph.daddr), hooknum);
1875 return NF_ACCEPT;
1876 }
1877
1878 if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
1879 return NF_ACCEPT;
1880
1881 ip_vs_fill_iph_skb(af, skb, false, &iph);
1882
1883
1884 sk = skb_to_full_sk(skb);
1885 if (unlikely(sk && hooknum == NF_INET_LOCAL_OUT &&
1886 af == AF_INET)) {
1887
1888 if (sk->sk_family == PF_INET && inet_sk(sk)->nodefrag)
1889 return NF_ACCEPT;
1890 }
1891
1892#ifdef CONFIG_IP_VS_IPV6
1893 if (af == AF_INET6) {
1894 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1895 int related;
1896 int verdict = ip_vs_in_icmp_v6(ipvs, skb, &related,
1897 hooknum, &iph);
1898
1899 if (related)
1900 return verdict;
1901 }
1902 } else
1903#endif
1904 if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1905 int related;
1906 int verdict = ip_vs_in_icmp(ipvs, skb, &related,
1907 hooknum);
1908
1909 if (related)
1910 return verdict;
1911 }
1912
1913
1914 pd = ip_vs_proto_data_get(ipvs, iph.protocol);
1915 if (unlikely(!pd)) {
1916
1917
1918
1919
1920 if (sysctl_ignore_tunneled(ipvs))
1921 skb->ipvs_property = 1;
1922
1923 return NF_ACCEPT;
1924 }
1925 pp = pd->pp;
1926
1927
1928
1929 cp = pp->conn_in_get(ipvs, af, skb, &iph);
1930
1931 conn_reuse_mode = sysctl_conn_reuse_mode(ipvs);
1932 if (conn_reuse_mode && !iph.fragoffs && is_new_conn(skb, &iph) && cp) {
1933 bool uses_ct = false, resched = false;
1934
1935 if (unlikely(sysctl_expire_nodest_conn(ipvs)) && cp->dest &&
1936 unlikely(!atomic_read(&cp->dest->weight))) {
1937 resched = true;
1938 uses_ct = ip_vs_conn_uses_conntrack(cp, skb);
1939 } else if (is_new_conn_expected(cp, conn_reuse_mode)) {
1940 uses_ct = ip_vs_conn_uses_conntrack(cp, skb);
1941 if (!atomic_read(&cp->n_control)) {
1942 resched = true;
1943 } else {
1944
1945
1946
1947
1948 resched = !uses_ct;
1949 }
1950 }
1951
1952 if (resched) {
1953 if (!atomic_read(&cp->n_control))
1954 ip_vs_conn_expire_now(cp);
1955 __ip_vs_conn_put(cp);
1956 if (uses_ct)
1957 return NF_DROP;
1958 cp = NULL;
1959 }
1960 }
1961
1962 if (unlikely(!cp)) {
1963 int v;
1964
1965 if (!ip_vs_try_to_schedule(ipvs, af, skb, pd, &v, &cp, &iph))
1966 return v;
1967 }
1968
1969 IP_VS_DBG_PKT(11, af, pp, skb, iph.off, "Incoming packet");
1970
1971
1972 if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
1973
1974
1975 if (sysctl_expire_nodest_conn(ipvs)) {
1976
1977 ip_vs_conn_expire_now(cp);
1978 }
1979
1980
1981 __ip_vs_conn_put(cp);
1982 return NF_DROP;
1983 }
1984
1985 ip_vs_in_stats(cp, skb);
1986 ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
1987 if (cp->packet_xmit)
1988 ret = cp->packet_xmit(skb, cp, pp, &iph);
1989
1990 else {
1991 IP_VS_DBG_RL("warning: packet_xmit is null");
1992 ret = NF_ACCEPT;
1993 }
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004 if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
2005 pkts = sysctl_sync_threshold(ipvs);
2006 else
2007 pkts = atomic_add_return(1, &cp->in_pkts);
2008
2009 if (ipvs->sync_state & IP_VS_STATE_MASTER)
2010 ip_vs_sync_conn(ipvs, cp, pkts);
2011 else if ((cp->flags & IP_VS_CONN_F_ONE_PACKET) && cp->control)
2012
2013 atomic_inc(&cp->control->in_pkts);
2014
2015 ip_vs_conn_put(cp);
2016 return ret;
2017}
2018
2019
2020
2021
2022
2023static unsigned int
2024ip_vs_remote_request4(void *priv, struct sk_buff *skb,
2025 const struct nf_hook_state *state)
2026{
2027 return ip_vs_in(net_ipvs(state->net), state->hook, skb, AF_INET);
2028}
2029
2030
2031
2032
2033
2034static unsigned int
2035ip_vs_local_request4(void *priv, struct sk_buff *skb,
2036 const struct nf_hook_state *state)
2037{
2038 return ip_vs_in(net_ipvs(state->net), state->hook, skb, AF_INET);
2039}
2040
2041#ifdef CONFIG_IP_VS_IPV6
2042
2043
2044
2045
2046
2047static unsigned int
2048ip_vs_remote_request6(void *priv, struct sk_buff *skb,
2049 const struct nf_hook_state *state)
2050{
2051 return ip_vs_in(net_ipvs(state->net), state->hook, skb, AF_INET6);
2052}
2053
2054
2055
2056
2057
2058static unsigned int
2059ip_vs_local_request6(void *priv, struct sk_buff *skb,
2060 const struct nf_hook_state *state)
2061{
2062 return ip_vs_in(net_ipvs(state->net), state->hook, skb, AF_INET6);
2063}
2064
2065#endif
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077static unsigned int
2078ip_vs_forward_icmp(void *priv, struct sk_buff *skb,
2079 const struct nf_hook_state *state)
2080{
2081 int r;
2082 struct netns_ipvs *ipvs = net_ipvs(state->net);
2083
2084 if (ip_hdr(skb)->protocol != IPPROTO_ICMP)
2085 return NF_ACCEPT;
2086
2087
2088 if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
2089 return NF_ACCEPT;
2090
2091 return ip_vs_in_icmp(ipvs, skb, &r, state->hook);
2092}
2093
2094#ifdef CONFIG_IP_VS_IPV6
2095static unsigned int
2096ip_vs_forward_icmp_v6(void *priv, struct sk_buff *skb,
2097 const struct nf_hook_state *state)
2098{
2099 int r;
2100 struct netns_ipvs *ipvs = net_ipvs(state->net);
2101 struct ip_vs_iphdr iphdr;
2102
2103 ip_vs_fill_iph_skb(AF_INET6, skb, false, &iphdr);
2104 if (iphdr.protocol != IPPROTO_ICMPV6)
2105 return NF_ACCEPT;
2106
2107
2108 if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
2109 return NF_ACCEPT;
2110
2111 return ip_vs_in_icmp_v6(ipvs, skb, &r, state->hook, &iphdr);
2112}
2113#endif
2114
2115
2116static const struct nf_hook_ops ip_vs_ops[] = {
2117
2118 {
2119 .hook = ip_vs_reply4,
2120 .pf = NFPROTO_IPV4,
2121 .hooknum = NF_INET_LOCAL_IN,
2122 .priority = NF_IP_PRI_NAT_SRC - 2,
2123 },
2124
2125
2126
2127 {
2128 .hook = ip_vs_remote_request4,
2129 .pf = NFPROTO_IPV4,
2130 .hooknum = NF_INET_LOCAL_IN,
2131 .priority = NF_IP_PRI_NAT_SRC - 1,
2132 },
2133
2134 {
2135 .hook = ip_vs_local_reply4,
2136 .pf = NFPROTO_IPV4,
2137 .hooknum = NF_INET_LOCAL_OUT,
2138 .priority = NF_IP_PRI_NAT_DST + 1,
2139 },
2140
2141 {
2142 .hook = ip_vs_local_request4,
2143 .pf = NFPROTO_IPV4,
2144 .hooknum = NF_INET_LOCAL_OUT,
2145 .priority = NF_IP_PRI_NAT_DST + 2,
2146 },
2147
2148
2149 {
2150 .hook = ip_vs_forward_icmp,
2151 .pf = NFPROTO_IPV4,
2152 .hooknum = NF_INET_FORWARD,
2153 .priority = 99,
2154 },
2155
2156 {
2157 .hook = ip_vs_reply4,
2158 .pf = NFPROTO_IPV4,
2159 .hooknum = NF_INET_FORWARD,
2160 .priority = 100,
2161 },
2162#ifdef CONFIG_IP_VS_IPV6
2163
2164 {
2165 .hook = ip_vs_reply6,
2166 .pf = NFPROTO_IPV6,
2167 .hooknum = NF_INET_LOCAL_IN,
2168 .priority = NF_IP6_PRI_NAT_SRC - 2,
2169 },
2170
2171
2172
2173 {
2174 .hook = ip_vs_remote_request6,
2175 .pf = NFPROTO_IPV6,
2176 .hooknum = NF_INET_LOCAL_IN,
2177 .priority = NF_IP6_PRI_NAT_SRC - 1,
2178 },
2179
2180 {
2181 .hook = ip_vs_local_reply6,
2182 .pf = NFPROTO_IPV6,
2183 .hooknum = NF_INET_LOCAL_OUT,
2184 .priority = NF_IP6_PRI_NAT_DST + 1,
2185 },
2186
2187 {
2188 .hook = ip_vs_local_request6,
2189 .pf = NFPROTO_IPV6,
2190 .hooknum = NF_INET_LOCAL_OUT,
2191 .priority = NF_IP6_PRI_NAT_DST + 2,
2192 },
2193
2194
2195 {
2196 .hook = ip_vs_forward_icmp_v6,
2197 .pf = NFPROTO_IPV6,
2198 .hooknum = NF_INET_FORWARD,
2199 .priority = 99,
2200 },
2201
2202 {
2203 .hook = ip_vs_reply6,
2204 .pf = NFPROTO_IPV6,
2205 .hooknum = NF_INET_FORWARD,
2206 .priority = 100,
2207 },
2208#endif
2209};
2210
2211
2212
2213static int __net_init __ip_vs_init(struct net *net)
2214{
2215 struct netns_ipvs *ipvs;
2216 int ret;
2217
2218 ipvs = net_generic(net, ip_vs_net_id);
2219 if (ipvs == NULL)
2220 return -ENOMEM;
2221
2222
2223 ipvs->enable = 0;
2224 ipvs->net = net;
2225
2226 ipvs->gen = atomic_read(&ipvs_netns_cnt);
2227 atomic_inc(&ipvs_netns_cnt);
2228 net->ipvs = ipvs;
2229
2230 if (ip_vs_estimator_net_init(ipvs) < 0)
2231 goto estimator_fail;
2232
2233 if (ip_vs_control_net_init(ipvs) < 0)
2234 goto control_fail;
2235
2236 if (ip_vs_protocol_net_init(ipvs) < 0)
2237 goto protocol_fail;
2238
2239 if (ip_vs_app_net_init(ipvs) < 0)
2240 goto app_fail;
2241
2242 if (ip_vs_conn_net_init(ipvs) < 0)
2243 goto conn_fail;
2244
2245 if (ip_vs_sync_net_init(ipvs) < 0)
2246 goto sync_fail;
2247
2248 ret = nf_register_net_hooks(net, ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
2249 if (ret < 0)
2250 goto hook_fail;
2251
2252 return 0;
2253
2254
2255
2256
2257hook_fail:
2258 ip_vs_sync_net_cleanup(ipvs);
2259sync_fail:
2260 ip_vs_conn_net_cleanup(ipvs);
2261conn_fail:
2262 ip_vs_app_net_cleanup(ipvs);
2263app_fail:
2264 ip_vs_protocol_net_cleanup(ipvs);
2265protocol_fail:
2266 ip_vs_control_net_cleanup(ipvs);
2267control_fail:
2268 ip_vs_estimator_net_cleanup(ipvs);
2269estimator_fail:
2270 net->ipvs = NULL;
2271 return -ENOMEM;
2272}
2273
2274static void __net_exit __ip_vs_cleanup(struct net *net)
2275{
2276 struct netns_ipvs *ipvs = net_ipvs(net);
2277
2278 nf_unregister_net_hooks(net, ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
2279 ip_vs_service_net_cleanup(ipvs);
2280 ip_vs_conn_net_cleanup(ipvs);
2281 ip_vs_app_net_cleanup(ipvs);
2282 ip_vs_protocol_net_cleanup(ipvs);
2283 ip_vs_control_net_cleanup(ipvs);
2284 ip_vs_estimator_net_cleanup(ipvs);
2285 IP_VS_DBG(2, "ipvs netns %d released\n", ipvs->gen);
2286 net->ipvs = NULL;
2287}
2288
2289static void __net_exit __ip_vs_dev_cleanup(struct net *net)
2290{
2291 struct netns_ipvs *ipvs = net_ipvs(net);
2292 EnterFunction(2);
2293 ipvs->enable = 0;
2294 smp_wmb();
2295 ip_vs_sync_net_cleanup(ipvs);
2296 LeaveFunction(2);
2297}
2298
2299static struct pernet_operations ipvs_core_ops = {
2300 .init = __ip_vs_init,
2301 .exit = __ip_vs_cleanup,
2302 .id = &ip_vs_net_id,
2303 .size = sizeof(struct netns_ipvs),
2304};
2305
2306static struct pernet_operations ipvs_core_dev_ops = {
2307 .exit = __ip_vs_dev_cleanup,
2308};
2309
2310
2311
2312
2313static int __init ip_vs_init(void)
2314{
2315 int ret;
2316
2317 ret = ip_vs_control_init();
2318 if (ret < 0) {
2319 pr_err("can't setup control.\n");
2320 goto exit;
2321 }
2322
2323 ip_vs_protocol_init();
2324
2325 ret = ip_vs_conn_init();
2326 if (ret < 0) {
2327 pr_err("can't setup connection table.\n");
2328 goto cleanup_protocol;
2329 }
2330
2331 ret = register_pernet_subsys(&ipvs_core_ops);
2332 if (ret < 0)
2333 goto cleanup_conn;
2334
2335 ret = register_pernet_device(&ipvs_core_dev_ops);
2336 if (ret < 0)
2337 goto cleanup_sub;
2338
2339 ret = ip_vs_register_nl_ioctl();
2340 if (ret < 0) {
2341 pr_err("can't register netlink/ioctl.\n");
2342 goto cleanup_dev;
2343 }
2344
2345 pr_info("ipvs loaded.\n");
2346
2347 return ret;
2348
2349cleanup_dev:
2350 unregister_pernet_device(&ipvs_core_dev_ops);
2351cleanup_sub:
2352 unregister_pernet_subsys(&ipvs_core_ops);
2353cleanup_conn:
2354 ip_vs_conn_cleanup();
2355cleanup_protocol:
2356 ip_vs_protocol_cleanup();
2357 ip_vs_control_cleanup();
2358exit:
2359 return ret;
2360}
2361
2362static void __exit ip_vs_cleanup(void)
2363{
2364 ip_vs_unregister_nl_ioctl();
2365 unregister_pernet_device(&ipvs_core_dev_ops);
2366 unregister_pernet_subsys(&ipvs_core_ops);
2367 ip_vs_conn_cleanup();
2368 ip_vs_protocol_cleanup();
2369 ip_vs_control_cleanup();
2370 pr_info("ipvs unloaded.\n");
2371}
2372
2373module_init(ip_vs_init);
2374module_exit(ip_vs_cleanup);
2375MODULE_LICENSE("GPL");
2376