1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
35
36#include <linux/types.h>
37#include <linux/kernel.h>
38#include <linux/mm.h>
39#include <linux/sched.h>
40#include <linux/module.h>
41#include <linux/workqueue.h>
42#include <linux/debugfs.h>
43#include <linux/seq_file.h>
44#include <asm/hypervisor.h>
45
46MODULE_AUTHOR("VMware, Inc.");
47MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");
48MODULE_VERSION("1.2.1.2-k");
49MODULE_ALIAS("dmi:*:svnVMware*:*");
50MODULE_ALIAS("vmware_vmmemctl");
51MODULE_LICENSE("GPL");
52
53
54
55
56
57
58
59
60
61
62#define VMW_BALLOON_NOSLEEP_ALLOC_MAX 16384U
63
64
65
66
67
68#define VMW_BALLOON_RATE_ALLOC_MIN 512U
69#define VMW_BALLOON_RATE_ALLOC_MAX 2048U
70#define VMW_BALLOON_RATE_ALLOC_INC 16U
71
72
73
74
75#define VMW_BALLOON_RATE_FREE_MIN 512U
76#define VMW_BALLOON_RATE_FREE_MAX 16384U
77#define VMW_BALLOON_RATE_FREE_INC 16U
78
79
80
81
82
83#define VMW_BALLOON_SLOW_CYCLES 4
84
85
86
87
88
89
90#define VMW_PAGE_ALLOC_NOSLEEP (__GFP_HIGHMEM|__GFP_NOWARN)
91
92
93
94
95
96
97
98
99#define VMW_PAGE_ALLOC_CANSLEEP (GFP_HIGHUSER)
100
101
102#define VMW_BALLOON_YIELD_THRESHOLD 1024
103
104
105#define VMW_BALLOON_MAX_REFUSED 16
106
107
108
109
110#define VMW_BALLOON_HV_PORT 0x5670
111#define VMW_BALLOON_HV_MAGIC 0x456c6d6f
112#define VMW_BALLOON_PROTOCOL_VERSION 2
113#define VMW_BALLOON_GUEST_ID 1
114
115#define VMW_BALLOON_CMD_START 0
116#define VMW_BALLOON_CMD_GET_TARGET 1
117#define VMW_BALLOON_CMD_LOCK 2
118#define VMW_BALLOON_CMD_UNLOCK 3
119#define VMW_BALLOON_CMD_GUEST_ID 4
120
121
122#define VMW_BALLOON_SUCCESS 0
123#define VMW_BALLOON_FAILURE -1
124#define VMW_BALLOON_ERROR_CMD_INVALID 1
125#define VMW_BALLOON_ERROR_PPN_INVALID 2
126#define VMW_BALLOON_ERROR_PPN_LOCKED 3
127#define VMW_BALLOON_ERROR_PPN_UNLOCKED 4
128#define VMW_BALLOON_ERROR_PPN_PINNED 5
129#define VMW_BALLOON_ERROR_PPN_NOTNEEDED 6
130#define VMW_BALLOON_ERROR_RESET 7
131#define VMW_BALLOON_ERROR_BUSY 8
132
133#define VMWARE_BALLOON_CMD(cmd, data, result) \
134({ \
135 unsigned long __stat, __dummy1, __dummy2; \
136 __asm__ __volatile__ ("inl (%%dx)" : \
137 "=a"(__stat), \
138 "=c"(__dummy1), \
139 "=d"(__dummy2), \
140 "=b"(result) : \
141 "0"(VMW_BALLOON_HV_MAGIC), \
142 "1"(VMW_BALLOON_CMD_##cmd), \
143 "2"(VMW_BALLOON_HV_PORT), \
144 "3"(data) : \
145 "memory"); \
146 result &= -1UL; \
147 __stat & -1UL; \
148})
149
150#ifdef CONFIG_DEBUG_FS
151struct vmballoon_stats {
152 unsigned int timer;
153
154
155 unsigned int alloc;
156 unsigned int alloc_fail;
157 unsigned int sleep_alloc;
158 unsigned int sleep_alloc_fail;
159 unsigned int refused_alloc;
160 unsigned int refused_free;
161 unsigned int free;
162
163
164 unsigned int lock;
165 unsigned int lock_fail;
166 unsigned int unlock;
167 unsigned int unlock_fail;
168 unsigned int target;
169 unsigned int target_fail;
170 unsigned int start;
171 unsigned int start_fail;
172 unsigned int guest_type;
173 unsigned int guest_type_fail;
174};
175
176#define STATS_INC(stat) (stat)++
177#else
178#define STATS_INC(stat)
179#endif
180
181struct vmballoon {
182
183
184 struct list_head pages;
185
186
187 struct list_head refused_pages;
188 unsigned int n_refused_pages;
189
190
191 unsigned int size;
192 unsigned int target;
193
194
195 bool reset_required;
196
197
198 unsigned int rate_alloc;
199 unsigned int rate_free;
200
201
202 unsigned int slow_allocation_cycles;
203
204#ifdef CONFIG_DEBUG_FS
205
206 struct vmballoon_stats stats;
207
208
209 struct dentry *dbg_entry;
210#endif
211
212 struct sysinfo sysinfo;
213
214 struct delayed_work dwork;
215};
216
217static struct vmballoon balloon;
218static struct workqueue_struct *vmballoon_wq;
219
220
221
222
223
224static bool vmballoon_send_start(struct vmballoon *b)
225{
226 unsigned long status, dummy;
227
228 STATS_INC(b->stats.start);
229
230 status = VMWARE_BALLOON_CMD(START, VMW_BALLOON_PROTOCOL_VERSION, dummy);
231 if (status == VMW_BALLOON_SUCCESS)
232 return true;
233
234 pr_debug("%s - failed, hv returns %ld\n", __func__, status);
235 STATS_INC(b->stats.start_fail);
236 return false;
237}
238
239static bool vmballoon_check_status(struct vmballoon *b, unsigned long status)
240{
241 switch (status) {
242 case VMW_BALLOON_SUCCESS:
243 return true;
244
245 case VMW_BALLOON_ERROR_RESET:
246 b->reset_required = true;
247
248
249 default:
250 return false;
251 }
252}
253
254
255
256
257
258
259
260static bool vmballoon_send_guest_id(struct vmballoon *b)
261{
262 unsigned long status, dummy;
263
264 status = VMWARE_BALLOON_CMD(GUEST_ID, VMW_BALLOON_GUEST_ID, dummy);
265
266 STATS_INC(b->stats.guest_type);
267
268 if (vmballoon_check_status(b, status))
269 return true;
270
271 pr_debug("%s - failed, hv returns %ld\n", __func__, status);
272 STATS_INC(b->stats.guest_type_fail);
273 return false;
274}
275
276
277
278
279static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target)
280{
281 unsigned long status;
282 unsigned long target;
283 unsigned long limit;
284 u32 limit32;
285
286
287
288
289
290
291 si_meminfo(&b->sysinfo);
292 limit = b->sysinfo.totalram;
293
294
295 limit32 = (u32)limit;
296 if (limit != limit32)
297 return false;
298
299
300 STATS_INC(b->stats.target);
301
302 status = VMWARE_BALLOON_CMD(GET_TARGET, limit, target);
303 if (vmballoon_check_status(b, status)) {
304 *new_target = target;
305 return true;
306 }
307
308 pr_debug("%s - failed, hv returns %ld\n", __func__, status);
309 STATS_INC(b->stats.target_fail);
310 return false;
311}
312
313
314
315
316
317
318static bool vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn,
319 unsigned int *hv_status)
320{
321 unsigned long status, dummy;
322 u32 pfn32;
323
324 pfn32 = (u32)pfn;
325 if (pfn32 != pfn)
326 return false;
327
328 STATS_INC(b->stats.lock);
329
330 *hv_status = status = VMWARE_BALLOON_CMD(LOCK, pfn, dummy);
331 if (vmballoon_check_status(b, status))
332 return true;
333
334 pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status);
335 STATS_INC(b->stats.lock_fail);
336 return false;
337}
338
339
340
341
342
343static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn)
344{
345 unsigned long status, dummy;
346 u32 pfn32;
347
348 pfn32 = (u32)pfn;
349 if (pfn32 != pfn)
350 return false;
351
352 STATS_INC(b->stats.unlock);
353
354 status = VMWARE_BALLOON_CMD(UNLOCK, pfn, dummy);
355 if (vmballoon_check_status(b, status))
356 return true;
357
358 pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status);
359 STATS_INC(b->stats.unlock_fail);
360 return false;
361}
362
363
364
365
366
367
368
369static void vmballoon_pop(struct vmballoon *b)
370{
371 struct page *page, *next;
372 unsigned int count = 0;
373
374 list_for_each_entry_safe(page, next, &b->pages, lru) {
375 list_del(&page->lru);
376 __free_page(page);
377 STATS_INC(b->stats.free);
378 b->size--;
379
380 if (++count >= b->rate_free) {
381 count = 0;
382 cond_resched();
383 }
384 }
385}
386
387
388
389
390
391
392static void vmballoon_reset(struct vmballoon *b)
393{
394
395 vmballoon_pop(b);
396
397 if (vmballoon_send_start(b)) {
398 b->reset_required = false;
399 if (!vmballoon_send_guest_id(b))
400 pr_err("failed to send guest ID to the host\n");
401 }
402}
403
404
405
406
407
408
409
410static int vmballoon_reserve_page(struct vmballoon *b, bool can_sleep)
411{
412 struct page *page;
413 gfp_t flags;
414 unsigned int hv_status;
415 bool locked = false;
416
417 do {
418 if (!can_sleep)
419 STATS_INC(b->stats.alloc);
420 else
421 STATS_INC(b->stats.sleep_alloc);
422
423 flags = can_sleep ? VMW_PAGE_ALLOC_CANSLEEP : VMW_PAGE_ALLOC_NOSLEEP;
424 page = alloc_page(flags);
425 if (!page) {
426 if (!can_sleep)
427 STATS_INC(b->stats.alloc_fail);
428 else
429 STATS_INC(b->stats.sleep_alloc_fail);
430 return -ENOMEM;
431 }
432
433
434 locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status);
435 if (!locked) {
436 STATS_INC(b->stats.refused_alloc);
437
438 if (hv_status == VMW_BALLOON_ERROR_RESET ||
439 hv_status == VMW_BALLOON_ERROR_PPN_NOTNEEDED) {
440 __free_page(page);
441 return -EIO;
442 }
443
444
445
446
447
448
449 list_add(&page->lru, &b->refused_pages);
450 if (++b->n_refused_pages >= VMW_BALLOON_MAX_REFUSED)
451 return -EIO;
452 }
453 } while (!locked);
454
455
456 list_add(&page->lru, &b->pages);
457
458
459 b->size++;
460
461 return 0;
462}
463
464
465
466
467
468
469static int vmballoon_release_page(struct vmballoon *b, struct page *page)
470{
471 if (!vmballoon_send_unlock_page(b, page_to_pfn(page)))
472 return -EIO;
473
474 list_del(&page->lru);
475
476
477 __free_page(page);
478 STATS_INC(b->stats.free);
479
480
481 b->size--;
482
483 return 0;
484}
485
486
487
488
489
490static void vmballoon_release_refused_pages(struct vmballoon *b)
491{
492 struct page *page, *next;
493
494 list_for_each_entry_safe(page, next, &b->refused_pages, lru) {
495 list_del(&page->lru);
496 __free_page(page);
497 STATS_INC(b->stats.refused_free);
498 }
499
500 b->n_refused_pages = 0;
501}
502
503
504
505
506
507
508static void vmballoon_inflate(struct vmballoon *b)
509{
510 unsigned int goal;
511 unsigned int rate;
512 unsigned int i;
513 unsigned int allocations = 0;
514 int error = 0;
515 bool alloc_can_sleep = false;
516
517 pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534 goal = b->target - b->size;
535
536
537
538
539 rate = b->slow_allocation_cycles ?
540 b->rate_alloc : VMW_BALLOON_NOSLEEP_ALLOC_MAX;
541
542 pr_debug("%s - goal: %d, no-sleep rate: %d, sleep rate: %d\n",
543 __func__, goal, rate, b->rate_alloc);
544
545 for (i = 0; i < goal; i++) {
546
547 error = vmballoon_reserve_page(b, alloc_can_sleep);
548 if (error) {
549 if (error != -ENOMEM) {
550
551
552
553
554
555 break;
556 }
557
558 if (alloc_can_sleep) {
559
560
561
562
563
564 b->rate_alloc = max(b->rate_alloc / 2,
565 VMW_BALLOON_RATE_ALLOC_MIN);
566 break;
567 }
568
569
570
571
572
573
574
575
576
577 b->slow_allocation_cycles = VMW_BALLOON_SLOW_CYCLES;
578
579 if (i >= b->rate_alloc)
580 break;
581
582 alloc_can_sleep = true;
583
584 rate = b->rate_alloc;
585 }
586
587 if (++allocations > VMW_BALLOON_YIELD_THRESHOLD) {
588 cond_resched();
589 allocations = 0;
590 }
591
592 if (i >= rate) {
593
594 break;
595 }
596 }
597
598
599
600
601
602 if (error == 0 && i >= b->rate_alloc) {
603 unsigned int mult = i / b->rate_alloc;
604
605 b->rate_alloc =
606 min(b->rate_alloc + mult * VMW_BALLOON_RATE_ALLOC_INC,
607 VMW_BALLOON_RATE_ALLOC_MAX);
608 }
609
610 vmballoon_release_refused_pages(b);
611}
612
613
614
615
616static void vmballoon_deflate(struct vmballoon *b)
617{
618 struct page *page, *next;
619 unsigned int i = 0;
620 unsigned int goal;
621 int error;
622
623 pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target);
624
625
626 goal = min(b->size - b->target, b->rate_free);
627
628 pr_debug("%s - goal: %d, rate: %d\n", __func__, goal, b->rate_free);
629
630
631 list_for_each_entry_safe(page, next, &b->pages, lru) {
632 error = vmballoon_release_page(b, page);
633 if (error) {
634
635 b->rate_free = max(b->rate_free / 2,
636 VMW_BALLOON_RATE_FREE_MIN);
637 return;
638 }
639
640 if (++i >= goal)
641 break;
642 }
643
644
645 b->rate_free = min(b->rate_free + VMW_BALLOON_RATE_FREE_INC,
646 VMW_BALLOON_RATE_FREE_MAX);
647}
648
649
650
651
652
653static void vmballoon_work(struct work_struct *work)
654{
655 struct delayed_work *dwork = to_delayed_work(work);
656 struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
657 unsigned int target;
658
659 STATS_INC(b->stats.timer);
660
661 if (b->reset_required)
662 vmballoon_reset(b);
663
664 if (b->slow_allocation_cycles > 0)
665 b->slow_allocation_cycles--;
666
667 if (vmballoon_send_get_target(b, &target)) {
668
669 b->target = target;
670
671 if (b->size < target)
672 vmballoon_inflate(b);
673 else if (b->size > target)
674 vmballoon_deflate(b);
675 }
676
677 queue_delayed_work(vmballoon_wq, dwork, round_jiffies_relative(HZ));
678}
679
680
681
682
683#ifdef CONFIG_DEBUG_FS
684
685static int vmballoon_debug_show(struct seq_file *f, void *offset)
686{
687 struct vmballoon *b = f->private;
688 struct vmballoon_stats *stats = &b->stats;
689
690
691 seq_printf(f,
692 "target: %8d pages\n"
693 "current: %8d pages\n",
694 b->target, b->size);
695
696
697 seq_printf(f,
698 "rateNoSleepAlloc: %8d pages/sec\n"
699 "rateSleepAlloc: %8d pages/sec\n"
700 "rateFree: %8d pages/sec\n",
701 VMW_BALLOON_NOSLEEP_ALLOC_MAX,
702 b->rate_alloc, b->rate_free);
703
704 seq_printf(f,
705 "\n"
706 "timer: %8u\n"
707 "start: %8u (%4u failed)\n"
708 "guestType: %8u (%4u failed)\n"
709 "lock: %8u (%4u failed)\n"
710 "unlock: %8u (%4u failed)\n"
711 "target: %8u (%4u failed)\n"
712 "primNoSleepAlloc: %8u (%4u failed)\n"
713 "primCanSleepAlloc: %8u (%4u failed)\n"
714 "primFree: %8u\n"
715 "errAlloc: %8u\n"
716 "errFree: %8u\n",
717 stats->timer,
718 stats->start, stats->start_fail,
719 stats->guest_type, stats->guest_type_fail,
720 stats->lock, stats->lock_fail,
721 stats->unlock, stats->unlock_fail,
722 stats->target, stats->target_fail,
723 stats->alloc, stats->alloc_fail,
724 stats->sleep_alloc, stats->sleep_alloc_fail,
725 stats->free,
726 stats->refused_alloc, stats->refused_free);
727
728 return 0;
729}
730
731static int vmballoon_debug_open(struct inode *inode, struct file *file)
732{
733 return single_open(file, vmballoon_debug_show, inode->i_private);
734}
735
736static const struct file_operations vmballoon_debug_fops = {
737 .owner = THIS_MODULE,
738 .open = vmballoon_debug_open,
739 .read = seq_read,
740 .llseek = seq_lseek,
741 .release = single_release,
742};
743
744static int __init vmballoon_debugfs_init(struct vmballoon *b)
745{
746 int error;
747
748 b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,
749 &vmballoon_debug_fops);
750 if (IS_ERR(b->dbg_entry)) {
751 error = PTR_ERR(b->dbg_entry);
752 pr_err("failed to create debugfs entry, error: %d\n", error);
753 return error;
754 }
755
756 return 0;
757}
758
759static void __exit vmballoon_debugfs_exit(struct vmballoon *b)
760{
761 debugfs_remove(b->dbg_entry);
762}
763
764#else
765
766static inline int vmballoon_debugfs_init(struct vmballoon *b)
767{
768 return 0;
769}
770
771static inline void vmballoon_debugfs_exit(struct vmballoon *b)
772{
773}
774
775#endif
776
777static int __init vmballoon_init(void)
778{
779 int error;
780
781
782
783
784
785 if (x86_hyper != &x86_hyper_vmware)
786 return -ENODEV;
787
788 vmballoon_wq = create_freezable_workqueue("vmmemctl");
789 if (!vmballoon_wq) {
790 pr_err("failed to create workqueue\n");
791 return -ENOMEM;
792 }
793
794 INIT_LIST_HEAD(&balloon.pages);
795 INIT_LIST_HEAD(&balloon.refused_pages);
796
797
798 balloon.rate_alloc = VMW_BALLOON_RATE_ALLOC_MAX;
799 balloon.rate_free = VMW_BALLOON_RATE_FREE_MAX;
800
801 INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
802
803
804
805
806 if (!vmballoon_send_start(&balloon)) {
807 pr_err("failed to send start command to the host\n");
808 error = -EIO;
809 goto fail;
810 }
811
812 if (!vmballoon_send_guest_id(&balloon)) {
813 pr_err("failed to send guest ID to the host\n");
814 error = -EIO;
815 goto fail;
816 }
817
818 error = vmballoon_debugfs_init(&balloon);
819 if (error)
820 goto fail;
821
822 queue_delayed_work(vmballoon_wq, &balloon.dwork, 0);
823
824 return 0;
825
826fail:
827 destroy_workqueue(vmballoon_wq);
828 return error;
829}
830module_init(vmballoon_init);
831
832static void __exit vmballoon_exit(void)
833{
834 cancel_delayed_work_sync(&balloon.dwork);
835 destroy_workqueue(vmballoon_wq);
836
837 vmballoon_debugfs_exit(&balloon);
838
839
840
841
842
843
844 vmballoon_send_start(&balloon);
845 vmballoon_pop(&balloon);
846}
847module_exit(vmballoon_exit);
848