1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#define KMSG_COMPONENT "IPVS"
41#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
42
43#include <linux/ip.h>
44#include <linux/module.h>
45#include <linux/kernel.h>
46#include <linux/skbuff.h>
47#include <linux/jiffies.h>
48#include <linux/list.h>
49#include <linux/slab.h>
50
51
52#include <linux/fs.h>
53#include <linux/sysctl.h>
54#include <net/net_namespace.h>
55
56#include <net/ip_vs.h>
57
58
59
60
61
62
63#define CHECK_EXPIRE_INTERVAL (60*HZ)
64#define ENTRY_TIMEOUT (6*60*HZ)
65
66#define DEFAULT_EXPIRATION (24*60*60*HZ)
67
68
69
70
71
72
73
74#define COUNT_FOR_FULL_EXPIRATION 30
75
76
77
78
79#ifndef CONFIG_IP_VS_LBLCR_TAB_BITS
80#define CONFIG_IP_VS_LBLCR_TAB_BITS 10
81#endif
82#define IP_VS_LBLCR_TAB_BITS CONFIG_IP_VS_LBLCR_TAB_BITS
83#define IP_VS_LBLCR_TAB_SIZE (1 << IP_VS_LBLCR_TAB_BITS)
84#define IP_VS_LBLCR_TAB_MASK (IP_VS_LBLCR_TAB_SIZE - 1)
85
86
87
88
89
90struct ip_vs_dest_set_elem {
91 struct list_head list;
92 struct ip_vs_dest __rcu *dest;
93 struct rcu_head rcu_head;
94};
95
96struct ip_vs_dest_set {
97 atomic_t size;
98 unsigned long lastmod;
99 struct list_head list;
100};
101
102
103static void ip_vs_dest_set_insert(struct ip_vs_dest_set *set,
104 struct ip_vs_dest *dest, bool check)
105{
106 struct ip_vs_dest_set_elem *e;
107
108 if (check) {
109 list_for_each_entry(e, &set->list, list) {
110 struct ip_vs_dest *d;
111
112 d = rcu_dereference_protected(e->dest, 1);
113 if (d == dest)
114
115 return;
116 }
117 }
118
119 e = kmalloc(sizeof(*e), GFP_ATOMIC);
120 if (e == NULL)
121 return;
122
123 ip_vs_dest_hold(dest);
124 RCU_INIT_POINTER(e->dest, dest);
125
126 list_add_rcu(&e->list, &set->list);
127 atomic_inc(&set->size);
128
129 set->lastmod = jiffies;
130}
131
132static void
133ip_vs_dest_set_erase(struct ip_vs_dest_set *set, struct ip_vs_dest *dest)
134{
135 struct ip_vs_dest_set_elem *e;
136
137 list_for_each_entry(e, &set->list, list) {
138 struct ip_vs_dest *d;
139
140 d = rcu_dereference_protected(e->dest, 1);
141 if (d == dest) {
142
143 atomic_dec(&set->size);
144 set->lastmod = jiffies;
145 ip_vs_dest_put(dest);
146 list_del_rcu(&e->list);
147 kfree_rcu(e, rcu_head);
148 break;
149 }
150 }
151}
152
153static void ip_vs_dest_set_eraseall(struct ip_vs_dest_set *set)
154{
155 struct ip_vs_dest_set_elem *e, *ep;
156
157 list_for_each_entry_safe(e, ep, &set->list, list) {
158 struct ip_vs_dest *d;
159
160 d = rcu_dereference_protected(e->dest, 1);
161
162
163
164
165 ip_vs_dest_put(d);
166 list_del_rcu(&e->list);
167 kfree_rcu(e, rcu_head);
168 }
169}
170
171
172static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set)
173{
174 register struct ip_vs_dest_set_elem *e;
175 struct ip_vs_dest *dest, *least;
176 int loh, doh;
177
178 if (set == NULL)
179 return NULL;
180
181
182 list_for_each_entry_rcu(e, &set->list, list) {
183 least = rcu_dereference(e->dest);
184 if (least->flags & IP_VS_DEST_F_OVERLOAD)
185 continue;
186
187 if ((atomic_read(&least->weight) > 0)
188 && (least->flags & IP_VS_DEST_F_AVAILABLE)) {
189 loh = ip_vs_dest_conn_overhead(least);
190 goto nextstage;
191 }
192 }
193 return NULL;
194
195
196 nextstage:
197 list_for_each_entry_continue_rcu(e, &set->list, list) {
198 dest = rcu_dereference(e->dest);
199 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
200 continue;
201
202 doh = ip_vs_dest_conn_overhead(dest);
203 if ((loh * atomic_read(&dest->weight) >
204 doh * atomic_read(&least->weight))
205 && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
206 least = dest;
207 loh = doh;
208 }
209 }
210
211 IP_VS_DBG_BUF(6, "%s(): server %s:%d "
212 "activeconns %d refcnt %d weight %d overhead %d\n",
213 __func__,
214 IP_VS_DBG_ADDR(least->af, &least->addr),
215 ntohs(least->port),
216 atomic_read(&least->activeconns),
217 atomic_read(&least->refcnt),
218 atomic_read(&least->weight), loh);
219 return least;
220}
221
222
223
224static inline struct ip_vs_dest *ip_vs_dest_set_max(struct ip_vs_dest_set *set)
225{
226 register struct ip_vs_dest_set_elem *e;
227 struct ip_vs_dest *dest, *most;
228 int moh, doh;
229
230 if (set == NULL)
231 return NULL;
232
233
234 list_for_each_entry(e, &set->list, list) {
235 most = rcu_dereference_protected(e->dest, 1);
236 if (atomic_read(&most->weight) > 0) {
237 moh = ip_vs_dest_conn_overhead(most);
238 goto nextstage;
239 }
240 }
241 return NULL;
242
243
244 nextstage:
245 list_for_each_entry_continue(e, &set->list, list) {
246 dest = rcu_dereference_protected(e->dest, 1);
247 doh = ip_vs_dest_conn_overhead(dest);
248
249 if ((moh * atomic_read(&dest->weight) <
250 doh * atomic_read(&most->weight))
251 && (atomic_read(&dest->weight) > 0)) {
252 most = dest;
253 moh = doh;
254 }
255 }
256
257 IP_VS_DBG_BUF(6, "%s(): server %s:%d "
258 "activeconns %d refcnt %d weight %d overhead %d\n",
259 __func__,
260 IP_VS_DBG_ADDR(most->af, &most->addr), ntohs(most->port),
261 atomic_read(&most->activeconns),
262 atomic_read(&most->refcnt),
263 atomic_read(&most->weight), moh);
264 return most;
265}
266
267
268
269
270
271
272struct ip_vs_lblcr_entry {
273 struct hlist_node list;
274 int af;
275 union nf_inet_addr addr;
276 struct ip_vs_dest_set set;
277 unsigned long lastuse;
278 struct rcu_head rcu_head;
279};
280
281
282
283
284
285struct ip_vs_lblcr_table {
286 struct rcu_head rcu_head;
287 struct hlist_head bucket[IP_VS_LBLCR_TAB_SIZE];
288 atomic_t entries;
289 int max_size;
290 struct timer_list periodic_timer;
291 int rover;
292 int counter;
293 bool dead;
294};
295
296
297#ifdef CONFIG_SYSCTL
298
299
300
301
302static ctl_table vs_vars_table[] = {
303 {
304 .procname = "lblcr_expiration",
305 .data = NULL,
306 .maxlen = sizeof(int),
307 .mode = 0644,
308 .proc_handler = proc_dointvec_jiffies,
309 },
310 { }
311};
312#endif
313
314static inline void ip_vs_lblcr_free(struct ip_vs_lblcr_entry *en)
315{
316 hlist_del_rcu(&en->list);
317 ip_vs_dest_set_eraseall(&en->set);
318 kfree_rcu(en, rcu_head);
319}
320
321
322
323
324
325static inline unsigned int
326ip_vs_lblcr_hashkey(int af, const union nf_inet_addr *addr)
327{
328 __be32 addr_fold = addr->ip;
329
330#ifdef CONFIG_IP_VS_IPV6
331 if (af == AF_INET6)
332 addr_fold = addr->ip6[0]^addr->ip6[1]^
333 addr->ip6[2]^addr->ip6[3];
334#endif
335 return (ntohl(addr_fold)*2654435761UL) & IP_VS_LBLCR_TAB_MASK;
336}
337
338
339
340
341
342
343static void
344ip_vs_lblcr_hash(struct ip_vs_lblcr_table *tbl, struct ip_vs_lblcr_entry *en)
345{
346 unsigned int hash = ip_vs_lblcr_hashkey(en->af, &en->addr);
347
348 hlist_add_head_rcu(&en->list, &tbl->bucket[hash]);
349 atomic_inc(&tbl->entries);
350}
351
352
353
354static inline struct ip_vs_lblcr_entry *
355ip_vs_lblcr_get(int af, struct ip_vs_lblcr_table *tbl,
356 const union nf_inet_addr *addr)
357{
358 unsigned int hash = ip_vs_lblcr_hashkey(af, addr);
359 struct ip_vs_lblcr_entry *en;
360
361 hlist_for_each_entry_rcu(en, &tbl->bucket[hash], list)
362 if (ip_vs_addr_equal(af, &en->addr, addr))
363 return en;
364
365 return NULL;
366}
367
368
369
370
371
372
373static inline struct ip_vs_lblcr_entry *
374ip_vs_lblcr_new(struct ip_vs_lblcr_table *tbl, const union nf_inet_addr *daddr,
375 struct ip_vs_dest *dest)
376{
377 struct ip_vs_lblcr_entry *en;
378
379 en = ip_vs_lblcr_get(dest->af, tbl, daddr);
380 if (!en) {
381 en = kmalloc(sizeof(*en), GFP_ATOMIC);
382 if (!en)
383 return NULL;
384
385 en->af = dest->af;
386 ip_vs_addr_copy(dest->af, &en->addr, daddr);
387 en->lastuse = jiffies;
388
389
390 atomic_set(&(en->set.size), 0);
391 INIT_LIST_HEAD(&en->set.list);
392
393 ip_vs_dest_set_insert(&en->set, dest, false);
394
395 ip_vs_lblcr_hash(tbl, en);
396 return en;
397 }
398
399 ip_vs_dest_set_insert(&en->set, dest, true);
400
401 return en;
402}
403
404
405
406
407
408static void ip_vs_lblcr_flush(struct ip_vs_service *svc)
409{
410 struct ip_vs_lblcr_table *tbl = svc->sched_data;
411 int i;
412 struct ip_vs_lblcr_entry *en;
413 struct hlist_node *next;
414
415 spin_lock_bh(&svc->sched_lock);
416 tbl->dead = 1;
417 for (i=0; i<IP_VS_LBLCR_TAB_SIZE; i++) {
418 hlist_for_each_entry_safe(en, next, &tbl->bucket[i], list) {
419 ip_vs_lblcr_free(en);
420 }
421 }
422 spin_unlock_bh(&svc->sched_lock);
423}
424
425static int sysctl_lblcr_expiration(struct ip_vs_service *svc)
426{
427#ifdef CONFIG_SYSCTL
428 struct netns_ipvs *ipvs = net_ipvs(svc->net);
429 return ipvs->sysctl_lblcr_expiration;
430#else
431 return DEFAULT_EXPIRATION;
432#endif
433}
434
435static inline void ip_vs_lblcr_full_check(struct ip_vs_service *svc)
436{
437 struct ip_vs_lblcr_table *tbl = svc->sched_data;
438 unsigned long now = jiffies;
439 int i, j;
440 struct ip_vs_lblcr_entry *en;
441 struct hlist_node *next;
442
443 for (i=0, j=tbl->rover; i<IP_VS_LBLCR_TAB_SIZE; i++) {
444 j = (j + 1) & IP_VS_LBLCR_TAB_MASK;
445
446 spin_lock(&svc->sched_lock);
447 hlist_for_each_entry_safe(en, next, &tbl->bucket[j], list) {
448 if (time_after(en->lastuse +
449 sysctl_lblcr_expiration(svc), now))
450 continue;
451
452 ip_vs_lblcr_free(en);
453 atomic_dec(&tbl->entries);
454 }
455 spin_unlock(&svc->sched_lock);
456 }
457 tbl->rover = j;
458}
459
460
461
462
463
464
465
466
467
468
469
470
471
472static void ip_vs_lblcr_check_expire(unsigned long data)
473{
474 struct ip_vs_service *svc = (struct ip_vs_service *) data;
475 struct ip_vs_lblcr_table *tbl = svc->sched_data;
476 unsigned long now = jiffies;
477 int goal;
478 int i, j;
479 struct ip_vs_lblcr_entry *en;
480 struct hlist_node *next;
481
482 if ((tbl->counter % COUNT_FOR_FULL_EXPIRATION) == 0) {
483
484 ip_vs_lblcr_full_check(svc);
485 tbl->counter = 1;
486 goto out;
487 }
488
489 if (atomic_read(&tbl->entries) <= tbl->max_size) {
490 tbl->counter++;
491 goto out;
492 }
493
494 goal = (atomic_read(&tbl->entries) - tbl->max_size)*4/3;
495 if (goal > tbl->max_size/2)
496 goal = tbl->max_size/2;
497
498 for (i=0, j=tbl->rover; i<IP_VS_LBLCR_TAB_SIZE; i++) {
499 j = (j + 1) & IP_VS_LBLCR_TAB_MASK;
500
501 spin_lock(&svc->sched_lock);
502 hlist_for_each_entry_safe(en, next, &tbl->bucket[j], list) {
503 if (time_before(now, en->lastuse+ENTRY_TIMEOUT))
504 continue;
505
506 ip_vs_lblcr_free(en);
507 atomic_dec(&tbl->entries);
508 goal--;
509 }
510 spin_unlock(&svc->sched_lock);
511 if (goal <= 0)
512 break;
513 }
514 tbl->rover = j;
515
516 out:
517 mod_timer(&tbl->periodic_timer, jiffies+CHECK_EXPIRE_INTERVAL);
518}
519
520static int ip_vs_lblcr_init_svc(struct ip_vs_service *svc)
521{
522 int i;
523 struct ip_vs_lblcr_table *tbl;
524
525
526
527
528 tbl = kmalloc(sizeof(*tbl), GFP_KERNEL);
529 if (tbl == NULL)
530 return -ENOMEM;
531
532 svc->sched_data = tbl;
533 IP_VS_DBG(6, "LBLCR hash table (memory=%Zdbytes) allocated for "
534 "current service\n", sizeof(*tbl));
535
536
537
538
539 for (i=0; i<IP_VS_LBLCR_TAB_SIZE; i++) {
540 INIT_HLIST_HEAD(&tbl->bucket[i]);
541 }
542 tbl->max_size = IP_VS_LBLCR_TAB_SIZE*16;
543 tbl->rover = 0;
544 tbl->counter = 1;
545 tbl->dead = 0;
546
547
548
549
550 setup_timer(&tbl->periodic_timer, ip_vs_lblcr_check_expire,
551 (unsigned long)svc);
552 mod_timer(&tbl->periodic_timer, jiffies + CHECK_EXPIRE_INTERVAL);
553
554 return 0;
555}
556
557
558static void ip_vs_lblcr_done_svc(struct ip_vs_service *svc)
559{
560 struct ip_vs_lblcr_table *tbl = svc->sched_data;
561
562
563 del_timer_sync(&tbl->periodic_timer);
564
565
566 ip_vs_lblcr_flush(svc);
567
568
569 kfree_rcu(tbl, rcu_head);
570 IP_VS_DBG(6, "LBLCR hash table (memory=%Zdbytes) released\n",
571 sizeof(*tbl));
572}
573
574
575static inline struct ip_vs_dest *
576__ip_vs_lblcr_schedule(struct ip_vs_service *svc)
577{
578 struct ip_vs_dest *dest, *least;
579 int loh, doh;
580
581
582
583
584
585
586
587
588
589
590
591
592
593 list_for_each_entry_rcu(dest, &svc->destinations, n_list) {
594 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
595 continue;
596
597 if (atomic_read(&dest->weight) > 0) {
598 least = dest;
599 loh = ip_vs_dest_conn_overhead(least);
600 goto nextstage;
601 }
602 }
603 return NULL;
604
605
606
607
608 nextstage:
609 list_for_each_entry_continue_rcu(dest, &svc->destinations, n_list) {
610 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
611 continue;
612
613 doh = ip_vs_dest_conn_overhead(dest);
614 if (loh * atomic_read(&dest->weight) >
615 doh * atomic_read(&least->weight)) {
616 least = dest;
617 loh = doh;
618 }
619 }
620
621 IP_VS_DBG_BUF(6, "LBLCR: server %s:%d "
622 "activeconns %d refcnt %d weight %d overhead %d\n",
623 IP_VS_DBG_ADDR(least->af, &least->addr),
624 ntohs(least->port),
625 atomic_read(&least->activeconns),
626 atomic_read(&least->refcnt),
627 atomic_read(&least->weight), loh);
628
629 return least;
630}
631
632
633
634
635
636
637static inline int
638is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc)
639{
640 if (atomic_read(&dest->activeconns) > atomic_read(&dest->weight)) {
641 struct ip_vs_dest *d;
642
643 list_for_each_entry_rcu(d, &svc->destinations, n_list) {
644 if (atomic_read(&d->activeconns)*2
645 < atomic_read(&d->weight)) {
646 return 1;
647 }
648 }
649 }
650 return 0;
651}
652
653
654
655
656
657static struct ip_vs_dest *
658ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
659{
660 struct ip_vs_lblcr_table *tbl = svc->sched_data;
661 struct ip_vs_iphdr iph;
662 struct ip_vs_dest *dest;
663 struct ip_vs_lblcr_entry *en;
664
665 ip_vs_fill_iph_addr_only(svc->af, skb, &iph);
666
667 IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);
668
669
670 en = ip_vs_lblcr_get(svc->af, tbl, &iph.daddr);
671 if (en) {
672 en->lastuse = jiffies;
673
674
675 dest = ip_vs_dest_set_min(&en->set);
676
677
678 if (atomic_read(&en->set.size) > 1 &&
679 time_after(jiffies, en->set.lastmod +
680 sysctl_lblcr_expiration(svc))) {
681 spin_lock_bh(&svc->sched_lock);
682 if (atomic_read(&en->set.size) > 1) {
683 struct ip_vs_dest *m;
684
685 m = ip_vs_dest_set_max(&en->set);
686 if (m)
687 ip_vs_dest_set_erase(&en->set, m);
688 }
689 spin_unlock_bh(&svc->sched_lock);
690 }
691
692
693 if (dest && !is_overloaded(dest, svc))
694 goto out;
695
696
697 dest = __ip_vs_lblcr_schedule(svc);
698 if (!dest) {
699 ip_vs_scheduler_err(svc, "no destination available");
700 return NULL;
701 }
702
703
704 spin_lock_bh(&svc->sched_lock);
705 if (!tbl->dead)
706 ip_vs_dest_set_insert(&en->set, dest, true);
707 spin_unlock_bh(&svc->sched_lock);
708 goto out;
709 }
710
711
712 dest = __ip_vs_lblcr_schedule(svc);
713 if (!dest) {
714 IP_VS_DBG(1, "no destination available\n");
715 return NULL;
716 }
717
718
719 spin_lock_bh(&svc->sched_lock);
720 if (!tbl->dead)
721 ip_vs_lblcr_new(tbl, &iph.daddr, dest);
722 spin_unlock_bh(&svc->sched_lock);
723
724out:
725 IP_VS_DBG_BUF(6, "LBLCR: destination IP address %s --> server %s:%d\n",
726 IP_VS_DBG_ADDR(svc->af, &iph.daddr),
727 IP_VS_DBG_ADDR(svc->af, &dest->addr), ntohs(dest->port));
728
729 return dest;
730}
731
732
733
734
735
736static struct ip_vs_scheduler ip_vs_lblcr_scheduler =
737{
738 .name = "lblcr",
739 .refcnt = ATOMIC_INIT(0),
740 .module = THIS_MODULE,
741 .n_list = LIST_HEAD_INIT(ip_vs_lblcr_scheduler.n_list),
742 .init_service = ip_vs_lblcr_init_svc,
743 .done_service = ip_vs_lblcr_done_svc,
744 .schedule = ip_vs_lblcr_schedule,
745};
746
747
748
749
750#ifdef CONFIG_SYSCTL
751static int __net_init __ip_vs_lblcr_init(struct net *net)
752{
753 struct netns_ipvs *ipvs = net_ipvs(net);
754
755 if (!ipvs)
756 return -ENOENT;
757
758 if (!net_eq(net, &init_net)) {
759 ipvs->lblcr_ctl_table = kmemdup(vs_vars_table,
760 sizeof(vs_vars_table),
761 GFP_KERNEL);
762 if (ipvs->lblcr_ctl_table == NULL)
763 return -ENOMEM;
764
765
766 if (net->user_ns != &init_user_ns)
767 ipvs->lblcr_ctl_table[0].procname = NULL;
768 } else
769 ipvs->lblcr_ctl_table = vs_vars_table;
770 ipvs->sysctl_lblcr_expiration = DEFAULT_EXPIRATION;
771 ipvs->lblcr_ctl_table[0].data = &ipvs->sysctl_lblcr_expiration;
772
773 ipvs->lblcr_ctl_header =
774 register_net_sysctl(net, "net/ipv4/vs", ipvs->lblcr_ctl_table);
775 if (!ipvs->lblcr_ctl_header) {
776 if (!net_eq(net, &init_net))
777 kfree(ipvs->lblcr_ctl_table);
778 return -ENOMEM;
779 }
780
781 return 0;
782}
783
784static void __net_exit __ip_vs_lblcr_exit(struct net *net)
785{
786 struct netns_ipvs *ipvs = net_ipvs(net);
787
788 unregister_net_sysctl_table(ipvs->lblcr_ctl_header);
789
790 if (!net_eq(net, &init_net))
791 kfree(ipvs->lblcr_ctl_table);
792}
793
794#else
795
796static int __net_init __ip_vs_lblcr_init(struct net *net) { return 0; }
797static void __net_exit __ip_vs_lblcr_exit(struct net *net) { }
798
799#endif
800
801static struct pernet_operations ip_vs_lblcr_ops = {
802 .init = __ip_vs_lblcr_init,
803 .exit = __ip_vs_lblcr_exit,
804};
805
806static int __init ip_vs_lblcr_init(void)
807{
808 int ret;
809
810 ret = register_pernet_subsys(&ip_vs_lblcr_ops);
811 if (ret)
812 return ret;
813
814 ret = register_ip_vs_scheduler(&ip_vs_lblcr_scheduler);
815 if (ret)
816 unregister_pernet_subsys(&ip_vs_lblcr_ops);
817 return ret;
818}
819
820static void __exit ip_vs_lblcr_cleanup(void)
821{
822 unregister_ip_vs_scheduler(&ip_vs_lblcr_scheduler);
823 unregister_pernet_subsys(&ip_vs_lblcr_ops);
824 synchronize_rcu();
825}
826
827
828module_init(ip_vs_lblcr_init);
829module_exit(ip_vs_lblcr_cleanup);
830MODULE_LICENSE("GPL");
831