1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
40
41#include <linux/cpu.h>
42#include <linux/kernel.h>
43#include <linux/sched.h>
44#include <linux/errno.h>
45#include <linux/module.h>
46#include <linux/mm.h>
47#include <linux/bootmem.h>
48#include <linux/pagemap.h>
49#include <linux/highmem.h>
50#include <linux/mutex.h>
51#include <linux/list.h>
52#include <linux/gfp.h>
53#include <linux/notifier.h>
54#include <linux/memory.h>
55#include <linux/memory_hotplug.h>
56#include <linux/percpu-defs.h>
57
58#include <asm/page.h>
59#include <asm/pgalloc.h>
60#include <asm/pgtable.h>
61#include <asm/tlb.h>
62
63#include <asm/xen/hypervisor.h>
64#include <asm/xen/hypercall.h>
65
66#include <xen/xen.h>
67#include <xen/interface/xen.h>
68#include <xen/interface/memory.h>
69#include <xen/balloon.h>
70#include <xen/features.h>
71#include <xen/page.h>
72
73
74
75
76
77
78
79
80
81enum bp_state {
82 BP_DONE,
83 BP_EAGAIN,
84 BP_ECANCELED
85};
86
87
88static DEFINE_MUTEX(balloon_mutex);
89
90struct balloon_stats balloon_stats;
91EXPORT_SYMBOL_GPL(balloon_stats);
92
93
94static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)];
95static DEFINE_PER_CPU(struct page *, balloon_scratch_page);
96
97
98
99static LIST_HEAD(ballooned_pages);
100
101
102static void balloon_process(struct work_struct *work);
103static DECLARE_DELAYED_WORK(balloon_worker, balloon_process);
104
105
106
107#define GFP_BALLOON \
108 (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
109
110static void scrub_page(struct page *page)
111{
112#ifdef CONFIG_XEN_SCRUB_PAGES
113 clear_highpage(page);
114#endif
115}
116
117
118static void __balloon_append(struct page *page)
119{
120
121 if (PageHighMem(page)) {
122 list_add_tail(&page->lru, &ballooned_pages);
123 balloon_stats.balloon_high++;
124 } else {
125 list_add(&page->lru, &ballooned_pages);
126 balloon_stats.balloon_low++;
127 }
128}
129
130static void balloon_append(struct page *page)
131{
132 __balloon_append(page);
133 adjust_managed_page_count(page, -1);
134}
135
136
137static struct page *balloon_retrieve(bool prefer_highmem)
138{
139 struct page *page;
140
141 if (list_empty(&ballooned_pages))
142 return NULL;
143
144 if (prefer_highmem)
145 page = list_entry(ballooned_pages.prev, struct page, lru);
146 else
147 page = list_entry(ballooned_pages.next, struct page, lru);
148 list_del(&page->lru);
149
150 if (PageHighMem(page))
151 balloon_stats.balloon_high--;
152 else
153 balloon_stats.balloon_low--;
154
155 adjust_managed_page_count(page, 1);
156
157 return page;
158}
159
160static struct page *balloon_next_page(struct page *page)
161{
162 struct list_head *next = page->lru.next;
163 if (next == &ballooned_pages)
164 return NULL;
165 return list_entry(next, struct page, lru);
166}
167
168static enum bp_state update_schedule(enum bp_state state)
169{
170 if (state == BP_ECANCELED)
171 return BP_ECANCELED;
172
173 if (state == BP_DONE) {
174 balloon_stats.schedule_delay = 1;
175 balloon_stats.retry_count = 1;
176 return BP_DONE;
177 }
178
179 ++balloon_stats.retry_count;
180
181 if (balloon_stats.max_retry_count != RETRY_UNLIMITED &&
182 balloon_stats.retry_count > balloon_stats.max_retry_count) {
183 balloon_stats.schedule_delay = 1;
184 balloon_stats.retry_count = 1;
185 return BP_ECANCELED;
186 }
187
188 balloon_stats.schedule_delay <<= 1;
189
190 if (balloon_stats.schedule_delay > balloon_stats.max_schedule_delay)
191 balloon_stats.schedule_delay = balloon_stats.max_schedule_delay;
192
193 return BP_EAGAIN;
194}
195
196#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
197static long current_credit(void)
198{
199 return balloon_stats.target_pages - balloon_stats.current_pages -
200 balloon_stats.hotplug_pages;
201}
202
203static bool balloon_is_inflated(void)
204{
205 if (balloon_stats.balloon_low || balloon_stats.balloon_high ||
206 balloon_stats.balloon_hotplug)
207 return true;
208 else
209 return false;
210}
211
212
213
214
215
216
217
218
219
220
221
222
223static enum bp_state reserve_additional_memory(long credit)
224{
225 int nid, rc;
226 u64 hotplug_start_paddr;
227 unsigned long balloon_hotplug = credit;
228
229 hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
230 balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
231 nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
232
233 rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
234
235 if (rc) {
236 pr_warn("Cannot add additional memory (%i)\n", rc);
237 return BP_ECANCELED;
238 }
239
240 balloon_hotplug -= credit;
241
242 balloon_stats.hotplug_pages += credit;
243 balloon_stats.balloon_hotplug = balloon_hotplug;
244
245 return BP_DONE;
246}
247
248static void xen_online_page(struct page *page)
249{
250 __online_page_set_limits(page);
251
252 mutex_lock(&balloon_mutex);
253
254 __balloon_append(page);
255
256 if (balloon_stats.hotplug_pages)
257 --balloon_stats.hotplug_pages;
258 else
259 --balloon_stats.balloon_hotplug;
260
261 mutex_unlock(&balloon_mutex);
262}
263
264static int xen_memory_notifier(struct notifier_block *nb, unsigned long val, void *v)
265{
266 if (val == MEM_ONLINE)
267 schedule_delayed_work(&balloon_worker, 0);
268
269 return NOTIFY_OK;
270}
271
272static struct notifier_block xen_memory_nb = {
273 .notifier_call = xen_memory_notifier,
274 .priority = 0
275};
276#else
277static long current_credit(void)
278{
279 unsigned long target = balloon_stats.target_pages;
280
281 target = min(target,
282 balloon_stats.current_pages +
283 balloon_stats.balloon_low +
284 balloon_stats.balloon_high);
285
286 return target - balloon_stats.current_pages;
287}
288
289static bool balloon_is_inflated(void)
290{
291 if (balloon_stats.balloon_low || balloon_stats.balloon_high)
292 return true;
293 else
294 return false;
295}
296
297static enum bp_state reserve_additional_memory(long credit)
298{
299 balloon_stats.target_pages = balloon_stats.current_pages;
300 return BP_DONE;
301}
302#endif
303
304static enum bp_state increase_reservation(unsigned long nr_pages)
305{
306 int rc;
307 unsigned long pfn, i;
308 struct page *page;
309 struct xen_memory_reservation reservation = {
310 .address_bits = 0,
311 .extent_order = 0,
312 .domid = DOMID_SELF
313 };
314
315#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
316 if (!balloon_stats.balloon_low && !balloon_stats.balloon_high) {
317 nr_pages = min(nr_pages, balloon_stats.balloon_hotplug);
318 balloon_stats.hotplug_pages += nr_pages;
319 balloon_stats.balloon_hotplug -= nr_pages;
320 return BP_DONE;
321 }
322#endif
323
324 if (nr_pages > ARRAY_SIZE(frame_list))
325 nr_pages = ARRAY_SIZE(frame_list);
326
327 page = list_first_entry_or_null(&ballooned_pages, struct page, lru);
328 for (i = 0; i < nr_pages; i++) {
329 if (!page) {
330 nr_pages = i;
331 break;
332 }
333 frame_list[i] = page_to_pfn(page);
334 page = balloon_next_page(page);
335 }
336
337 set_xen_guest_handle(reservation.extent_start, frame_list);
338 reservation.nr_extents = nr_pages;
339 rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
340 if (rc <= 0)
341 return BP_EAGAIN;
342
343 for (i = 0; i < rc; i++) {
344 page = balloon_retrieve(false);
345 BUG_ON(page == NULL);
346
347 pfn = page_to_pfn(page);
348
349#ifdef CONFIG_XEN_HAVE_PVMMU
350 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
351 set_phys_to_machine(pfn, frame_list[i]);
352
353
354 if (!PageHighMem(page)) {
355 int ret;
356 ret = HYPERVISOR_update_va_mapping(
357 (unsigned long)__va(pfn << PAGE_SHIFT),
358 mfn_pte(frame_list[i], PAGE_KERNEL),
359 0);
360 BUG_ON(ret);
361 }
362 }
363#endif
364
365
366 __free_reserved_page(page);
367 }
368
369 balloon_stats.current_pages += rc;
370
371 return BP_DONE;
372}
373
374static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
375{
376 enum bp_state state = BP_DONE;
377 unsigned long pfn, i;
378 struct page *page;
379 int ret;
380 struct xen_memory_reservation reservation = {
381 .address_bits = 0,
382 .extent_order = 0,
383 .domid = DOMID_SELF
384 };
385
386#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
387 if (balloon_stats.hotplug_pages) {
388 nr_pages = min(nr_pages, balloon_stats.hotplug_pages);
389 balloon_stats.hotplug_pages -= nr_pages;
390 balloon_stats.balloon_hotplug += nr_pages;
391 return BP_DONE;
392 }
393#endif
394
395 if (nr_pages > ARRAY_SIZE(frame_list))
396 nr_pages = ARRAY_SIZE(frame_list);
397
398 for (i = 0; i < nr_pages; i++) {
399 page = alloc_page(gfp);
400 if (page == NULL) {
401 nr_pages = i;
402 state = BP_EAGAIN;
403 break;
404 }
405 scrub_page(page);
406
407 frame_list[i] = page_to_pfn(page);
408 }
409
410
411
412
413
414
415
416
417 kmap_flush_unused();
418
419
420 for (i = 0; i < nr_pages; i++) {
421 pfn = frame_list[i];
422 frame_list[i] = pfn_to_mfn(pfn);
423 page = pfn_to_page(pfn);
424
425#ifdef CONFIG_XEN_HAVE_PVMMU
426
427
428
429
430
431 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
432 if (!PageHighMem(page)) {
433 struct page *scratch_page = get_balloon_scratch_page();
434
435 ret = HYPERVISOR_update_va_mapping(
436 (unsigned long)__va(pfn << PAGE_SHIFT),
437 pfn_pte(page_to_pfn(scratch_page),
438 PAGE_KERNEL_RO), 0);
439 BUG_ON(ret);
440
441 put_balloon_scratch_page();
442 }
443 __set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
444 }
445#endif
446
447 balloon_append(page);
448 }
449
450 flush_tlb_all();
451
452 set_xen_guest_handle(reservation.extent_start, frame_list);
453 reservation.nr_extents = nr_pages;
454 ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
455 BUG_ON(ret != nr_pages);
456
457 balloon_stats.current_pages -= nr_pages;
458
459 return state;
460}
461
462
463
464
465
466
467
468static void balloon_process(struct work_struct *work)
469{
470 enum bp_state state = BP_DONE;
471 long credit;
472
473 mutex_lock(&balloon_mutex);
474
475 do {
476 credit = current_credit();
477
478 if (credit > 0) {
479 if (balloon_is_inflated())
480 state = increase_reservation(credit);
481 else
482 state = reserve_additional_memory(credit);
483 }
484
485 if (credit < 0)
486 state = decrease_reservation(-credit, GFP_BALLOON);
487
488 state = update_schedule(state);
489
490#ifndef CONFIG_PREEMPT
491 if (need_resched())
492 schedule();
493#endif
494 } while (credit && state == BP_DONE);
495
496
497 if (state == BP_EAGAIN)
498 schedule_delayed_work(&balloon_worker, balloon_stats.schedule_delay * HZ);
499
500 mutex_unlock(&balloon_mutex);
501}
502
503struct page *get_balloon_scratch_page(void)
504{
505 struct page *ret = get_cpu_var(balloon_scratch_page);
506 BUG_ON(ret == NULL);
507 return ret;
508}
509
510void put_balloon_scratch_page(void)
511{
512 put_cpu_var(balloon_scratch_page);
513}
514
515
516void balloon_set_new_target(unsigned long target)
517{
518
519 balloon_stats.target_pages = target;
520 schedule_delayed_work(&balloon_worker, 0);
521}
522EXPORT_SYMBOL_GPL(balloon_set_new_target);
523
524
525
526
527
528
529
530
531int alloc_xenballooned_pages(int nr_pages, struct page **pages, bool highmem)
532{
533 int pgno = 0;
534 struct page *page;
535 mutex_lock(&balloon_mutex);
536 while (pgno < nr_pages) {
537 page = balloon_retrieve(highmem);
538 if (page && (highmem || !PageHighMem(page))) {
539 pages[pgno++] = page;
540 } else {
541 enum bp_state st;
542 if (page)
543 balloon_append(page);
544 st = decrease_reservation(nr_pages - pgno,
545 highmem ? GFP_HIGHUSER : GFP_USER);
546 if (st != BP_DONE)
547 goto out_undo;
548 }
549 }
550 mutex_unlock(&balloon_mutex);
551 return 0;
552 out_undo:
553 while (pgno)
554 balloon_append(pages[--pgno]);
555
556 schedule_delayed_work(&balloon_worker, 0);
557 mutex_unlock(&balloon_mutex);
558 return -ENOMEM;
559}
560EXPORT_SYMBOL(alloc_xenballooned_pages);
561
562
563
564
565
566
567void free_xenballooned_pages(int nr_pages, struct page **pages)
568{
569 int i;
570
571 mutex_lock(&balloon_mutex);
572
573 for (i = 0; i < nr_pages; i++) {
574 if (pages[i])
575 balloon_append(pages[i]);
576 }
577
578
579 if (current_credit())
580 schedule_delayed_work(&balloon_worker, 0);
581
582 mutex_unlock(&balloon_mutex);
583}
584EXPORT_SYMBOL(free_xenballooned_pages);
585
586static void __init balloon_add_region(unsigned long start_pfn,
587 unsigned long pages)
588{
589 unsigned long pfn, extra_pfn_end;
590 struct page *page;
591
592
593
594
595
596
597 extra_pfn_end = min(max_pfn, start_pfn + pages);
598
599 for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) {
600 page = pfn_to_page(pfn);
601
602
603
604 __balloon_append(page);
605 }
606}
607
608static int alloc_balloon_scratch_page(int cpu)
609{
610 if (per_cpu(balloon_scratch_page, cpu) != NULL)
611 return 0;
612
613 per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL);
614 if (per_cpu(balloon_scratch_page, cpu) == NULL) {
615 pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu);
616 return -ENOMEM;
617 }
618
619 return 0;
620}
621
622
623static int balloon_cpu_notify(struct notifier_block *self,
624 unsigned long action, void *hcpu)
625{
626 int cpu = (long)hcpu;
627 switch (action) {
628 case CPU_UP_PREPARE:
629 if (alloc_balloon_scratch_page(cpu))
630 return NOTIFY_BAD;
631 break;
632 default:
633 break;
634 }
635 return NOTIFY_OK;
636}
637
638static struct notifier_block balloon_cpu_notifier = {
639 .notifier_call = balloon_cpu_notify,
640};
641
642static int __init balloon_init(void)
643{
644 int i, cpu;
645
646 if (!xen_domain())
647 return -ENODEV;
648
649 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
650 register_cpu_notifier(&balloon_cpu_notifier);
651
652 get_online_cpus();
653 for_each_online_cpu(cpu) {
654 if (alloc_balloon_scratch_page(cpu)) {
655 put_online_cpus();
656 unregister_cpu_notifier(&balloon_cpu_notifier);
657 return -ENOMEM;
658 }
659 }
660 put_online_cpus();
661 }
662
663 pr_info("Initialising balloon driver\n");
664
665 balloon_stats.current_pages = xen_pv_domain()
666 ? min(xen_start_info->nr_pages - xen_released_pages, max_pfn)
667 : get_num_physpages();
668 balloon_stats.target_pages = balloon_stats.current_pages;
669 balloon_stats.balloon_low = 0;
670 balloon_stats.balloon_high = 0;
671
672 balloon_stats.schedule_delay = 1;
673 balloon_stats.max_schedule_delay = 32;
674 balloon_stats.retry_count = 1;
675 balloon_stats.max_retry_count = RETRY_UNLIMITED;
676
677#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
678 balloon_stats.hotplug_pages = 0;
679 balloon_stats.balloon_hotplug = 0;
680
681 set_online_page_callback(&xen_online_page);
682 register_memory_notifier(&xen_memory_nb);
683#endif
684
685
686
687
688
689 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++)
690 if (xen_extra_mem[i].size)
691 balloon_add_region(PFN_UP(xen_extra_mem[i].start),
692 PFN_DOWN(xen_extra_mem[i].size));
693
694 return 0;
695}
696
697subsys_initcall(balloon_init);
698
699static int __init balloon_clear(void)
700{
701 int cpu;
702
703 for_each_possible_cpu(cpu)
704 per_cpu(balloon_scratch_page, cpu) = NULL;
705
706 return 0;
707}
708early_initcall(balloon_clear);
709
710MODULE_LICENSE("GPL");
711