1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <linux/virtio.h>
23#include <linux/virtio_balloon.h>
24#include <linux/swap.h>
25#include <linux/kthread.h>
26#include <linux/freezer.h>
27#include <linux/delay.h>
28#include <linux/slab.h>
29#include <linux/module.h>
30#include <linux/balloon_compaction.h>
31
32
33
34
35
36
37#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
38#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
39
40struct virtio_balloon
41{
42 struct virtio_device *vdev;
43 struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
44
45
46 wait_queue_head_t config_change;
47
48
49 struct task_struct *thread;
50
51
52 wait_queue_head_t acked;
53
54
55 unsigned int num_pages;
56
57
58
59
60
61
62 struct balloon_dev_info *vb_dev_info;
63
64
65 struct mutex balloon_lock;
66
67
68 unsigned int num_pfns;
69 u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
70
71
72 int need_stats_update;
73 struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
74};
75
76static struct virtio_device_id id_table[] = {
77 { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
78 { 0 },
79};
80
81static u32 page_to_balloon_pfn(struct page *page)
82{
83 unsigned long pfn = page_to_pfn(page);
84
85 BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
86
87 return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
88}
89
90static struct page *balloon_pfn_to_page(u32 pfn)
91{
92 BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
93 return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
94}
95
96static void balloon_ack(struct virtqueue *vq)
97{
98 struct virtio_balloon *vb = vq->vdev->priv;
99
100 wake_up(&vb->acked);
101}
102
103static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
104{
105 struct scatterlist sg;
106 unsigned int len;
107
108 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
109
110
111 if (virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL) < 0)
112 BUG();
113 virtqueue_kick(vq);
114
115
116 wait_event(vb->acked, virtqueue_get_buf(vq, &len));
117}
118
119static void set_page_pfns(u32 pfns[], struct page *page)
120{
121 unsigned int i;
122
123
124
125 for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
126 pfns[i] = page_to_balloon_pfn(page) + i;
127}
128
129static void fill_balloon(struct virtio_balloon *vb, size_t num)
130{
131 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
132
133
134 num = min(num, ARRAY_SIZE(vb->pfns));
135
136 mutex_lock(&vb->balloon_lock);
137 for (vb->num_pfns = 0; vb->num_pfns < num;
138 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
139 struct page *page = balloon_page_enqueue(vb_dev_info);
140
141 if (!page) {
142 dev_info_ratelimited(&vb->vdev->dev,
143 "Out of puff! Can't get %u pages\n",
144 VIRTIO_BALLOON_PAGES_PER_PAGE);
145
146 msleep(200);
147 break;
148 }
149 set_page_pfns(vb->pfns + vb->num_pfns, page);
150 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
151 adjust_managed_page_count(page, -1);
152 }
153
154
155 if (vb->num_pfns != 0)
156 tell_host(vb, vb->inflate_vq);
157 mutex_unlock(&vb->balloon_lock);
158}
159
160static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
161{
162 unsigned int i;
163
164
165 for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
166 struct page *page = balloon_pfn_to_page(pfns[i]);
167 balloon_page_free(page);
168 adjust_managed_page_count(page, 1);
169 }
170}
171
172static void leak_balloon(struct virtio_balloon *vb, size_t num)
173{
174 struct page *page;
175 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
176
177
178 num = min(num, ARRAY_SIZE(vb->pfns));
179
180 mutex_lock(&vb->balloon_lock);
181 for (vb->num_pfns = 0; vb->num_pfns < num;
182 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
183 page = balloon_page_dequeue(vb_dev_info);
184 if (!page)
185 break;
186 set_page_pfns(vb->pfns + vb->num_pfns, page);
187 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
188 }
189
190
191
192
193
194
195 if (vb->num_pfns != 0)
196 tell_host(vb, vb->deflate_vq);
197 mutex_unlock(&vb->balloon_lock);
198 release_pages_by_pfn(vb->pfns, vb->num_pfns);
199}
200
201static inline void update_stat(struct virtio_balloon *vb, int idx,
202 u16 tag, u64 val)
203{
204 BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
205 vb->stats[idx].tag = tag;
206 vb->stats[idx].val = val;
207}
208
209#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)
210
211static void update_balloon_stats(struct virtio_balloon *vb)
212{
213 unsigned long events[NR_VM_EVENT_ITEMS];
214 struct sysinfo i;
215 int idx = 0;
216
217 all_vm_events(events);
218 si_meminfo(&i);
219
220 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
221 pages_to_bytes(events[PSWPIN]));
222 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
223 pages_to_bytes(events[PSWPOUT]));
224 update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
225 update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
226 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
227 pages_to_bytes(i.freeram));
228 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
229 pages_to_bytes(i.totalram));
230}
231
232
233
234
235
236
237
238
239
240static void stats_request(struct virtqueue *vq)
241{
242 struct virtio_balloon *vb = vq->vdev->priv;
243
244 vb->need_stats_update = 1;
245 wake_up(&vb->config_change);
246}
247
248static void stats_handle_request(struct virtio_balloon *vb)
249{
250 struct virtqueue *vq;
251 struct scatterlist sg;
252 unsigned int len;
253
254 vb->need_stats_update = 0;
255 update_balloon_stats(vb);
256
257 vq = vb->stats_vq;
258 if (!virtqueue_get_buf(vq, &len))
259 return;
260 sg_init_one(&sg, vb->stats, sizeof(vb->stats));
261 if (virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL) < 0)
262 BUG();
263 virtqueue_kick(vq);
264}
265
266static void virtballoon_changed(struct virtio_device *vdev)
267{
268 struct virtio_balloon *vb = vdev->priv;
269
270 wake_up(&vb->config_change);
271}
272
273static inline s64 towards_target(struct virtio_balloon *vb)
274{
275 __le32 v;
276 s64 target;
277
278 vb->vdev->config->get(vb->vdev,
279 offsetof(struct virtio_balloon_config, num_pages),
280 &v, sizeof(v));
281 target = le32_to_cpu(v);
282 return target - vb->num_pages;
283}
284
285static void update_balloon_size(struct virtio_balloon *vb)
286{
287 __le32 actual = cpu_to_le32(vb->num_pages);
288
289 vb->vdev->config->set(vb->vdev,
290 offsetof(struct virtio_balloon_config, actual),
291 &actual, sizeof(actual));
292}
293
294static int balloon(void *_vballoon)
295{
296 struct virtio_balloon *vb = _vballoon;
297
298 set_freezable();
299 while (!kthread_should_stop()) {
300 s64 diff;
301
302 try_to_freeze();
303 wait_event_interruptible(vb->config_change,
304 (diff = towards_target(vb)) != 0
305 || vb->need_stats_update
306 || kthread_should_stop()
307 || freezing(current));
308 if (vb->need_stats_update)
309 stats_handle_request(vb);
310 if (diff > 0)
311 fill_balloon(vb, diff);
312 else if (diff < 0)
313 leak_balloon(vb, -diff);
314 update_balloon_size(vb);
315 }
316 return 0;
317}
318
319static int init_vqs(struct virtio_balloon *vb)
320{
321 struct virtqueue *vqs[3];
322 vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
323 const char *names[] = { "inflate", "deflate", "stats" };
324 int err, nvqs;
325
326
327
328
329
330 nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
331 err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
332 if (err)
333 return err;
334
335 vb->inflate_vq = vqs[0];
336 vb->deflate_vq = vqs[1];
337 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
338 struct scatterlist sg;
339 vb->stats_vq = vqs[2];
340
341
342
343
344
345 sg_init_one(&sg, vb->stats, sizeof vb->stats);
346 if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
347 < 0)
348 BUG();
349 virtqueue_kick(vb->stats_vq);
350 }
351 return 0;
352}
353
354static const struct address_space_operations virtio_balloon_aops;
355#ifdef CONFIG_BALLOON_COMPACTION
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374int virtballoon_migratepage(struct address_space *mapping,
375 struct page *newpage, struct page *page, enum migrate_mode mode)
376{
377 struct balloon_dev_info *vb_dev_info = balloon_page_device(page);
378 struct virtio_balloon *vb;
379 unsigned long flags;
380
381 BUG_ON(!vb_dev_info);
382
383 vb = vb_dev_info->balloon_device;
384
385
386
387
388
389
390
391
392
393 if (!mutex_trylock(&vb->balloon_lock))
394 return -EAGAIN;
395
396
397 spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
398 balloon_page_insert(newpage, mapping, &vb_dev_info->pages);
399 vb_dev_info->isolated_pages--;
400 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
401 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
402 set_page_pfns(vb->pfns, newpage);
403 tell_host(vb, vb->inflate_vq);
404
405
406
407
408
409
410
411 balloon_page_delete(page);
412 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
413 set_page_pfns(vb->pfns, page);
414 tell_host(vb, vb->deflate_vq);
415
416 mutex_unlock(&vb->balloon_lock);
417
418 return MIGRATEPAGE_BALLOON_SUCCESS;
419}
420
421
422static const struct address_space_operations virtio_balloon_aops = {
423 .migratepage = virtballoon_migratepage,
424};
425#endif
426
427static int virtballoon_probe(struct virtio_device *vdev)
428{
429 struct virtio_balloon *vb;
430 struct address_space *vb_mapping;
431 struct balloon_dev_info *vb_devinfo;
432 int err;
433
434 vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
435 if (!vb) {
436 err = -ENOMEM;
437 goto out;
438 }
439
440 vb->num_pages = 0;
441 mutex_init(&vb->balloon_lock);
442 init_waitqueue_head(&vb->config_change);
443 init_waitqueue_head(&vb->acked);
444 vb->vdev = vdev;
445 vb->need_stats_update = 0;
446
447 vb_devinfo = balloon_devinfo_alloc(vb);
448 if (IS_ERR(vb_devinfo)) {
449 err = PTR_ERR(vb_devinfo);
450 goto out_free_vb;
451 }
452
453 vb_mapping = balloon_mapping_alloc(vb_devinfo,
454 (balloon_compaction_check()) ?
455 &virtio_balloon_aops : NULL);
456 if (IS_ERR(vb_mapping)) {
457
458
459
460
461 err = PTR_ERR(vb_mapping);
462 if (err != -EOPNOTSUPP)
463 goto out_free_vb_devinfo;
464 }
465
466 vb->vb_dev_info = vb_devinfo;
467
468 err = init_vqs(vb);
469 if (err)
470 goto out_free_vb_mapping;
471
472 vb->thread = kthread_run(balloon, vb, "vballoon");
473 if (IS_ERR(vb->thread)) {
474 err = PTR_ERR(vb->thread);
475 goto out_del_vqs;
476 }
477
478 return 0;
479
480out_del_vqs:
481 vdev->config->del_vqs(vdev);
482out_free_vb_mapping:
483 balloon_mapping_free(vb_mapping);
484out_free_vb_devinfo:
485 balloon_devinfo_free(vb_devinfo);
486out_free_vb:
487 kfree(vb);
488out:
489 return err;
490}
491
492static void remove_common(struct virtio_balloon *vb)
493{
494
495 while (vb->num_pages)
496 leak_balloon(vb, vb->num_pages);
497 update_balloon_size(vb);
498
499
500 vb->vdev->config->reset(vb->vdev);
501
502 vb->vdev->config->del_vqs(vb->vdev);
503}
504
505static void virtballoon_remove(struct virtio_device *vdev)
506{
507 struct virtio_balloon *vb = vdev->priv;
508
509 kthread_stop(vb->thread);
510 remove_common(vb);
511 balloon_mapping_free(vb->vb_dev_info->mapping);
512 balloon_devinfo_free(vb->vb_dev_info);
513 kfree(vb);
514}
515
516#ifdef CONFIG_PM
517static int virtballoon_freeze(struct virtio_device *vdev)
518{
519 struct virtio_balloon *vb = vdev->priv;
520
521
522
523
524
525
526 remove_common(vb);
527 return 0;
528}
529
530static int virtballoon_restore(struct virtio_device *vdev)
531{
532 struct virtio_balloon *vb = vdev->priv;
533 int ret;
534
535 ret = init_vqs(vdev->priv);
536 if (ret)
537 return ret;
538
539 fill_balloon(vb, towards_target(vb));
540 update_balloon_size(vb);
541 return 0;
542}
543#endif
544
545static unsigned int features[] = {
546 VIRTIO_BALLOON_F_MUST_TELL_HOST,
547 VIRTIO_BALLOON_F_STATS_VQ,
548};
549
550static struct virtio_driver virtio_balloon_driver = {
551 .feature_table = features,
552 .feature_table_size = ARRAY_SIZE(features),
553 .driver.name = KBUILD_MODNAME,
554 .driver.owner = THIS_MODULE,
555 .id_table = id_table,
556 .probe = virtballoon_probe,
557 .remove = virtballoon_remove,
558 .config_changed = virtballoon_changed,
559#ifdef CONFIG_PM
560 .freeze = virtballoon_freeze,
561 .restore = virtballoon_restore,
562#endif
563};
564
565module_virtio_driver(virtio_balloon_driver);
566MODULE_DEVICE_TABLE(virtio, id_table);
567MODULE_DESCRIPTION("Virtio balloon driver");
568MODULE_LICENSE("GPL");
569