1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#define KMSG_COMPONENT "IPVS"
41#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
42
43#include <linux/ip.h>
44#include <linux/module.h>
45#include <linux/kernel.h>
46#include <linux/skbuff.h>
47#include <linux/jiffies.h>
48#include <linux/list.h>
49#include <linux/slab.h>
50
51
52#include <linux/fs.h>
53#include <linux/sysctl.h>
54#include <net/net_namespace.h>
55
56#include <net/ip_vs.h>
57
58
59
60
61
62
63#define CHECK_EXPIRE_INTERVAL (60*HZ)
64#define ENTRY_TIMEOUT (6*60*HZ)
65
66#define DEFAULT_EXPIRATION (24*60*60*HZ)
67
68
69
70
71
72
73
74#define COUNT_FOR_FULL_EXPIRATION 30
75
76
77
78
79#ifndef CONFIG_IP_VS_LBLCR_TAB_BITS
80#define CONFIG_IP_VS_LBLCR_TAB_BITS 10
81#endif
82#define IP_VS_LBLCR_TAB_BITS CONFIG_IP_VS_LBLCR_TAB_BITS
83#define IP_VS_LBLCR_TAB_SIZE (1 << IP_VS_LBLCR_TAB_BITS)
84#define IP_VS_LBLCR_TAB_MASK (IP_VS_LBLCR_TAB_SIZE - 1)
85
86
87
88
89
90struct ip_vs_dest_set_elem {
91 struct list_head list;
92 struct ip_vs_dest *dest;
93 struct rcu_head rcu_head;
94};
95
96struct ip_vs_dest_set {
97 atomic_t size;
98 unsigned long lastmod;
99 struct list_head list;
100};
101
102
103static void ip_vs_dest_set_insert(struct ip_vs_dest_set *set,
104 struct ip_vs_dest *dest, bool check)
105{
106 struct ip_vs_dest_set_elem *e;
107
108 if (check) {
109 list_for_each_entry(e, &set->list, list) {
110 if (e->dest == dest)
111 return;
112 }
113 }
114
115 e = kmalloc(sizeof(*e), GFP_ATOMIC);
116 if (e == NULL)
117 return;
118
119 ip_vs_dest_hold(dest);
120 e->dest = dest;
121
122 list_add_rcu(&e->list, &set->list);
123 atomic_inc(&set->size);
124
125 set->lastmod = jiffies;
126}
127
128static void ip_vs_lblcr_elem_rcu_free(struct rcu_head *head)
129{
130 struct ip_vs_dest_set_elem *e;
131
132 e = container_of(head, struct ip_vs_dest_set_elem, rcu_head);
133 ip_vs_dest_put_and_free(e->dest);
134 kfree(e);
135}
136
137static void
138ip_vs_dest_set_erase(struct ip_vs_dest_set *set, struct ip_vs_dest *dest)
139{
140 struct ip_vs_dest_set_elem *e;
141
142 list_for_each_entry(e, &set->list, list) {
143 if (e->dest == dest) {
144
145 atomic_dec(&set->size);
146 set->lastmod = jiffies;
147 list_del_rcu(&e->list);
148 call_rcu(&e->rcu_head, ip_vs_lblcr_elem_rcu_free);
149 break;
150 }
151 }
152}
153
154static void ip_vs_dest_set_eraseall(struct ip_vs_dest_set *set)
155{
156 struct ip_vs_dest_set_elem *e, *ep;
157
158 list_for_each_entry_safe(e, ep, &set->list, list) {
159 list_del_rcu(&e->list);
160 call_rcu(&e->rcu_head, ip_vs_lblcr_elem_rcu_free);
161 }
162}
163
164
165static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set)
166{
167 register struct ip_vs_dest_set_elem *e;
168 struct ip_vs_dest *dest, *least;
169 int loh, doh;
170
171
172 list_for_each_entry_rcu(e, &set->list, list) {
173 least = e->dest;
174 if (least->flags & IP_VS_DEST_F_OVERLOAD)
175 continue;
176
177 if ((atomic_read(&least->weight) > 0)
178 && (least->flags & IP_VS_DEST_F_AVAILABLE)) {
179 loh = ip_vs_dest_conn_overhead(least);
180 goto nextstage;
181 }
182 }
183 return NULL;
184
185
186 nextstage:
187 list_for_each_entry_continue_rcu(e, &set->list, list) {
188 dest = e->dest;
189 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
190 continue;
191
192 doh = ip_vs_dest_conn_overhead(dest);
193 if (((__s64)loh * atomic_read(&dest->weight) >
194 (__s64)doh * atomic_read(&least->weight))
195 && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
196 least = dest;
197 loh = doh;
198 }
199 }
200
201 IP_VS_DBG_BUF(6, "%s(): server %s:%d "
202 "activeconns %d refcnt %d weight %d overhead %d\n",
203 __func__,
204 IP_VS_DBG_ADDR(least->af, &least->addr),
205 ntohs(least->port),
206 atomic_read(&least->activeconns),
207 atomic_read(&least->refcnt),
208 atomic_read(&least->weight), loh);
209 return least;
210}
211
212
213
214static inline struct ip_vs_dest *ip_vs_dest_set_max(struct ip_vs_dest_set *set)
215{
216 register struct ip_vs_dest_set_elem *e;
217 struct ip_vs_dest *dest, *most;
218 int moh, doh;
219
220 if (set == NULL)
221 return NULL;
222
223
224 list_for_each_entry(e, &set->list, list) {
225 most = e->dest;
226 if (atomic_read(&most->weight) > 0) {
227 moh = ip_vs_dest_conn_overhead(most);
228 goto nextstage;
229 }
230 }
231 return NULL;
232
233
234 nextstage:
235 list_for_each_entry_continue(e, &set->list, list) {
236 dest = e->dest;
237 doh = ip_vs_dest_conn_overhead(dest);
238
239 if (((__s64)moh * atomic_read(&dest->weight) <
240 (__s64)doh * atomic_read(&most->weight))
241 && (atomic_read(&dest->weight) > 0)) {
242 most = dest;
243 moh = doh;
244 }
245 }
246
247 IP_VS_DBG_BUF(6, "%s(): server %s:%d "
248 "activeconns %d refcnt %d weight %d overhead %d\n",
249 __func__,
250 IP_VS_DBG_ADDR(most->af, &most->addr), ntohs(most->port),
251 atomic_read(&most->activeconns),
252 atomic_read(&most->refcnt),
253 atomic_read(&most->weight), moh);
254 return most;
255}
256
257
258
259
260
261
262struct ip_vs_lblcr_entry {
263 struct hlist_node list;
264 int af;
265 union nf_inet_addr addr;
266 struct ip_vs_dest_set set;
267 unsigned long lastuse;
268 struct rcu_head rcu_head;
269};
270
271
272
273
274
275struct ip_vs_lblcr_table {
276 struct rcu_head rcu_head;
277 struct hlist_head bucket[IP_VS_LBLCR_TAB_SIZE];
278 atomic_t entries;
279 int max_size;
280 struct timer_list periodic_timer;
281 int rover;
282 int counter;
283 bool dead;
284};
285
286
287#ifdef CONFIG_SYSCTL
288
289
290
291
292static struct ctl_table vs_vars_table[] = {
293 {
294 .procname = "lblcr_expiration",
295 .data = NULL,
296 .maxlen = sizeof(int),
297 .mode = 0644,
298 .proc_handler = proc_dointvec_jiffies,
299 },
300 { }
301};
302#endif
303
304static inline void ip_vs_lblcr_free(struct ip_vs_lblcr_entry *en)
305{
306 hlist_del_rcu(&en->list);
307 ip_vs_dest_set_eraseall(&en->set);
308 kfree_rcu(en, rcu_head);
309}
310
311
312
313
314
315static inline unsigned int
316ip_vs_lblcr_hashkey(int af, const union nf_inet_addr *addr)
317{
318 __be32 addr_fold = addr->ip;
319
320#ifdef CONFIG_IP_VS_IPV6
321 if (af == AF_INET6)
322 addr_fold = addr->ip6[0]^addr->ip6[1]^
323 addr->ip6[2]^addr->ip6[3];
324#endif
325 return (ntohl(addr_fold)*2654435761UL) & IP_VS_LBLCR_TAB_MASK;
326}
327
328
329
330
331
332
333static void
334ip_vs_lblcr_hash(struct ip_vs_lblcr_table *tbl, struct ip_vs_lblcr_entry *en)
335{
336 unsigned int hash = ip_vs_lblcr_hashkey(en->af, &en->addr);
337
338 hlist_add_head_rcu(&en->list, &tbl->bucket[hash]);
339 atomic_inc(&tbl->entries);
340}
341
342
343
344static inline struct ip_vs_lblcr_entry *
345ip_vs_lblcr_get(int af, struct ip_vs_lblcr_table *tbl,
346 const union nf_inet_addr *addr)
347{
348 unsigned int hash = ip_vs_lblcr_hashkey(af, addr);
349 struct ip_vs_lblcr_entry *en;
350
351 hlist_for_each_entry_rcu(en, &tbl->bucket[hash], list)
352 if (ip_vs_addr_equal(af, &en->addr, addr))
353 return en;
354
355 return NULL;
356}
357
358
359
360
361
362
363static inline struct ip_vs_lblcr_entry *
364ip_vs_lblcr_new(struct ip_vs_lblcr_table *tbl, const union nf_inet_addr *daddr,
365 u16 af, struct ip_vs_dest *dest)
366{
367 struct ip_vs_lblcr_entry *en;
368
369 en = ip_vs_lblcr_get(af, tbl, daddr);
370 if (!en) {
371 en = kmalloc(sizeof(*en), GFP_ATOMIC);
372 if (!en)
373 return NULL;
374
375 en->af = af;
376 ip_vs_addr_copy(af, &en->addr, daddr);
377 en->lastuse = jiffies;
378
379
380 atomic_set(&(en->set.size), 0);
381 INIT_LIST_HEAD(&en->set.list);
382
383 ip_vs_dest_set_insert(&en->set, dest, false);
384
385 ip_vs_lblcr_hash(tbl, en);
386 return en;
387 }
388
389 ip_vs_dest_set_insert(&en->set, dest, true);
390
391 return en;
392}
393
394
395
396
397
398static void ip_vs_lblcr_flush(struct ip_vs_service *svc)
399{
400 struct ip_vs_lblcr_table *tbl = svc->sched_data;
401 int i;
402 struct ip_vs_lblcr_entry *en;
403 struct hlist_node *next;
404
405 spin_lock_bh(&svc->sched_lock);
406 tbl->dead = 1;
407 for (i = 0; i < IP_VS_LBLCR_TAB_SIZE; i++) {
408 hlist_for_each_entry_safe(en, next, &tbl->bucket[i], list) {
409 ip_vs_lblcr_free(en);
410 }
411 }
412 spin_unlock_bh(&svc->sched_lock);
413}
414
415static int sysctl_lblcr_expiration(struct ip_vs_service *svc)
416{
417#ifdef CONFIG_SYSCTL
418 return svc->ipvs->sysctl_lblcr_expiration;
419#else
420 return DEFAULT_EXPIRATION;
421#endif
422}
423
424static inline void ip_vs_lblcr_full_check(struct ip_vs_service *svc)
425{
426 struct ip_vs_lblcr_table *tbl = svc->sched_data;
427 unsigned long now = jiffies;
428 int i, j;
429 struct ip_vs_lblcr_entry *en;
430 struct hlist_node *next;
431
432 for (i = 0, j = tbl->rover; i < IP_VS_LBLCR_TAB_SIZE; i++) {
433 j = (j + 1) & IP_VS_LBLCR_TAB_MASK;
434
435 spin_lock(&svc->sched_lock);
436 hlist_for_each_entry_safe(en, next, &tbl->bucket[j], list) {
437 if (time_after(en->lastuse +
438 sysctl_lblcr_expiration(svc), now))
439 continue;
440
441 ip_vs_lblcr_free(en);
442 atomic_dec(&tbl->entries);
443 }
444 spin_unlock(&svc->sched_lock);
445 }
446 tbl->rover = j;
447}
448
449
450
451
452
453
454
455
456
457
458
459
460
461static void ip_vs_lblcr_check_expire(unsigned long data)
462{
463 struct ip_vs_service *svc = (struct ip_vs_service *) data;
464 struct ip_vs_lblcr_table *tbl = svc->sched_data;
465 unsigned long now = jiffies;
466 int goal;
467 int i, j;
468 struct ip_vs_lblcr_entry *en;
469 struct hlist_node *next;
470
471 if ((tbl->counter % COUNT_FOR_FULL_EXPIRATION) == 0) {
472
473 ip_vs_lblcr_full_check(svc);
474 tbl->counter = 1;
475 goto out;
476 }
477
478 if (atomic_read(&tbl->entries) <= tbl->max_size) {
479 tbl->counter++;
480 goto out;
481 }
482
483 goal = (atomic_read(&tbl->entries) - tbl->max_size)*4/3;
484 if (goal > tbl->max_size/2)
485 goal = tbl->max_size/2;
486
487 for (i = 0, j = tbl->rover; i < IP_VS_LBLCR_TAB_SIZE; i++) {
488 j = (j + 1) & IP_VS_LBLCR_TAB_MASK;
489
490 spin_lock(&svc->sched_lock);
491 hlist_for_each_entry_safe(en, next, &tbl->bucket[j], list) {
492 if (time_before(now, en->lastuse+ENTRY_TIMEOUT))
493 continue;
494
495 ip_vs_lblcr_free(en);
496 atomic_dec(&tbl->entries);
497 goal--;
498 }
499 spin_unlock(&svc->sched_lock);
500 if (goal <= 0)
501 break;
502 }
503 tbl->rover = j;
504
505 out:
506 mod_timer(&tbl->periodic_timer, jiffies+CHECK_EXPIRE_INTERVAL);
507}
508
509static int ip_vs_lblcr_init_svc(struct ip_vs_service *svc)
510{
511 int i;
512 struct ip_vs_lblcr_table *tbl;
513
514
515
516
517 tbl = kmalloc(sizeof(*tbl), GFP_KERNEL);
518 if (tbl == NULL)
519 return -ENOMEM;
520
521 svc->sched_data = tbl;
522 IP_VS_DBG(6, "LBLCR hash table (memory=%Zdbytes) allocated for "
523 "current service\n", sizeof(*tbl));
524
525
526
527
528 for (i = 0; i < IP_VS_LBLCR_TAB_SIZE; i++) {
529 INIT_HLIST_HEAD(&tbl->bucket[i]);
530 }
531 tbl->max_size = IP_VS_LBLCR_TAB_SIZE*16;
532 tbl->rover = 0;
533 tbl->counter = 1;
534 tbl->dead = 0;
535
536
537
538
539 setup_timer(&tbl->periodic_timer, ip_vs_lblcr_check_expire,
540 (unsigned long)svc);
541 mod_timer(&tbl->periodic_timer, jiffies + CHECK_EXPIRE_INTERVAL);
542
543 return 0;
544}
545
546
547static void ip_vs_lblcr_done_svc(struct ip_vs_service *svc)
548{
549 struct ip_vs_lblcr_table *tbl = svc->sched_data;
550
551
552 del_timer_sync(&tbl->periodic_timer);
553
554
555 ip_vs_lblcr_flush(svc);
556
557
558 kfree_rcu(tbl, rcu_head);
559 IP_VS_DBG(6, "LBLCR hash table (memory=%Zdbytes) released\n",
560 sizeof(*tbl));
561}
562
563
564static inline struct ip_vs_dest *
565__ip_vs_lblcr_schedule(struct ip_vs_service *svc)
566{
567 struct ip_vs_dest *dest, *least;
568 int loh, doh;
569
570
571
572
573
574
575
576
577
578
579
580
581
582 list_for_each_entry_rcu(dest, &svc->destinations, n_list) {
583 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
584 continue;
585
586 if (atomic_read(&dest->weight) > 0) {
587 least = dest;
588 loh = ip_vs_dest_conn_overhead(least);
589 goto nextstage;
590 }
591 }
592 return NULL;
593
594
595
596
597 nextstage:
598 list_for_each_entry_continue_rcu(dest, &svc->destinations, n_list) {
599 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
600 continue;
601
602 doh = ip_vs_dest_conn_overhead(dest);
603 if ((__s64)loh * atomic_read(&dest->weight) >
604 (__s64)doh * atomic_read(&least->weight)) {
605 least = dest;
606 loh = doh;
607 }
608 }
609
610 IP_VS_DBG_BUF(6, "LBLCR: server %s:%d "
611 "activeconns %d refcnt %d weight %d overhead %d\n",
612 IP_VS_DBG_ADDR(least->af, &least->addr),
613 ntohs(least->port),
614 atomic_read(&least->activeconns),
615 atomic_read(&least->refcnt),
616 atomic_read(&least->weight), loh);
617
618 return least;
619}
620
621
622
623
624
625
626static inline int
627is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc)
628{
629 if (atomic_read(&dest->activeconns) > atomic_read(&dest->weight)) {
630 struct ip_vs_dest *d;
631
632 list_for_each_entry_rcu(d, &svc->destinations, n_list) {
633 if (atomic_read(&d->activeconns)*2
634 < atomic_read(&d->weight)) {
635 return 1;
636 }
637 }
638 }
639 return 0;
640}
641
642
643
644
645
646static struct ip_vs_dest *
647ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb,
648 struct ip_vs_iphdr *iph)
649{
650 struct ip_vs_lblcr_table *tbl = svc->sched_data;
651 struct ip_vs_dest *dest;
652 struct ip_vs_lblcr_entry *en;
653
654 IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);
655
656
657 en = ip_vs_lblcr_get(svc->af, tbl, &iph->daddr);
658 if (en) {
659 en->lastuse = jiffies;
660
661
662 dest = ip_vs_dest_set_min(&en->set);
663
664
665 if (atomic_read(&en->set.size) > 1 &&
666 time_after(jiffies, en->set.lastmod +
667 sysctl_lblcr_expiration(svc))) {
668 spin_lock_bh(&svc->sched_lock);
669 if (atomic_read(&en->set.size) > 1) {
670 struct ip_vs_dest *m;
671
672 m = ip_vs_dest_set_max(&en->set);
673 if (m)
674 ip_vs_dest_set_erase(&en->set, m);
675 }
676 spin_unlock_bh(&svc->sched_lock);
677 }
678
679
680 if (dest && !is_overloaded(dest, svc))
681 goto out;
682
683
684 dest = __ip_vs_lblcr_schedule(svc);
685 if (!dest) {
686 ip_vs_scheduler_err(svc, "no destination available");
687 return NULL;
688 }
689
690
691 spin_lock_bh(&svc->sched_lock);
692 if (!tbl->dead)
693 ip_vs_dest_set_insert(&en->set, dest, true);
694 spin_unlock_bh(&svc->sched_lock);
695 goto out;
696 }
697
698
699 dest = __ip_vs_lblcr_schedule(svc);
700 if (!dest) {
701 IP_VS_DBG(1, "no destination available\n");
702 return NULL;
703 }
704
705
706 spin_lock_bh(&svc->sched_lock);
707 if (!tbl->dead)
708 ip_vs_lblcr_new(tbl, &iph->daddr, svc->af, dest);
709 spin_unlock_bh(&svc->sched_lock);
710
711out:
712 IP_VS_DBG_BUF(6, "LBLCR: destination IP address %s --> server %s:%d\n",
713 IP_VS_DBG_ADDR(svc->af, &iph->daddr),
714 IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port));
715
716 return dest;
717}
718
719
720
721
722
723static struct ip_vs_scheduler ip_vs_lblcr_scheduler =
724{
725 .name = "lblcr",
726 .refcnt = ATOMIC_INIT(0),
727 .module = THIS_MODULE,
728 .n_list = LIST_HEAD_INIT(ip_vs_lblcr_scheduler.n_list),
729 .init_service = ip_vs_lblcr_init_svc,
730 .done_service = ip_vs_lblcr_done_svc,
731 .schedule = ip_vs_lblcr_schedule,
732};
733
734
735
736
737#ifdef CONFIG_SYSCTL
738static int __net_init __ip_vs_lblcr_init(struct net *net)
739{
740 struct netns_ipvs *ipvs = net_ipvs(net);
741
742 if (!ipvs)
743 return -ENOENT;
744
745 if (!net_eq(net, &init_net)) {
746 ipvs->lblcr_ctl_table = kmemdup(vs_vars_table,
747 sizeof(vs_vars_table),
748 GFP_KERNEL);
749 if (ipvs->lblcr_ctl_table == NULL)
750 return -ENOMEM;
751
752
753 if (net->user_ns != &init_user_ns)
754 ipvs->lblcr_ctl_table[0].procname = NULL;
755 } else
756 ipvs->lblcr_ctl_table = vs_vars_table;
757 ipvs->sysctl_lblcr_expiration = DEFAULT_EXPIRATION;
758 ipvs->lblcr_ctl_table[0].data = &ipvs->sysctl_lblcr_expiration;
759
760 ipvs->lblcr_ctl_header =
761 register_net_sysctl(net, "net/ipv4/vs", ipvs->lblcr_ctl_table);
762 if (!ipvs->lblcr_ctl_header) {
763 if (!net_eq(net, &init_net))
764 kfree(ipvs->lblcr_ctl_table);
765 return -ENOMEM;
766 }
767
768 return 0;
769}
770
771static void __net_exit __ip_vs_lblcr_exit(struct net *net)
772{
773 struct netns_ipvs *ipvs = net_ipvs(net);
774
775 unregister_net_sysctl_table(ipvs->lblcr_ctl_header);
776
777 if (!net_eq(net, &init_net))
778 kfree(ipvs->lblcr_ctl_table);
779}
780
781#else
782
783static int __net_init __ip_vs_lblcr_init(struct net *net) { return 0; }
784static void __net_exit __ip_vs_lblcr_exit(struct net *net) { }
785
786#endif
787
788static struct pernet_operations ip_vs_lblcr_ops = {
789 .init = __ip_vs_lblcr_init,
790 .exit = __ip_vs_lblcr_exit,
791};
792
793static int __init ip_vs_lblcr_init(void)
794{
795 int ret;
796
797 ret = register_pernet_subsys(&ip_vs_lblcr_ops);
798 if (ret)
799 return ret;
800
801 ret = register_ip_vs_scheduler(&ip_vs_lblcr_scheduler);
802 if (ret)
803 unregister_pernet_subsys(&ip_vs_lblcr_ops);
804 return ret;
805}
806
807static void __exit ip_vs_lblcr_cleanup(void)
808{
809 unregister_ip_vs_scheduler(&ip_vs_lblcr_scheduler);
810 unregister_pernet_subsys(&ip_vs_lblcr_ops);
811 rcu_barrier();
812}
813
814
815module_init(ip_vs_lblcr_init);
816module_exit(ip_vs_lblcr_cleanup);
817MODULE_LICENSE("GPL");
818