1
2
3
4
5
6
7
8
9#include <linux/ctype.h>
10#include <linux/delay.h>
11#include <linux/errno.h>
12#include <linux/fs.h>
13#include <linux/gfp.h>
14#include <linux/kthread.h>
15#include <linux/module.h>
16#include <linux/oom.h>
17#include <linux/reboot.h>
18#include <linux/sched.h>
19#include <linux/stringify.h>
20#include <linux/swap.h>
21#include <linux/device.h>
22#include <linux/mount.h>
23#include <linux/pseudo_fs.h>
24#include <linux/magic.h>
25#include <linux/balloon_compaction.h>
26#include <asm/firmware.h>
27#include <asm/hvcall.h>
28#include <asm/mmu.h>
29#include <linux/uaccess.h>
30#include <linux/memory.h>
31#include <asm/plpar_wrappers.h>
32
33#include "pseries.h"
34
35#define CMM_DRIVER_VERSION "1.0.0"
36#define CMM_DEFAULT_DELAY 1
37#define CMM_HOTPLUG_DELAY 5
38#define CMM_DEBUG 0
39#define CMM_DISABLE 0
40#define CMM_OOM_KB 1024
41#define CMM_MIN_MEM_MB 256
42#define KB2PAGES(_p) ((_p)>>(PAGE_SHIFT-10))
43#define PAGES2KB(_p) ((_p)<<(PAGE_SHIFT-10))
44
45#define CMM_MEM_HOTPLUG_PRI 1
46
47static unsigned int delay = CMM_DEFAULT_DELAY;
48static unsigned int hotplug_delay = CMM_HOTPLUG_DELAY;
49static unsigned int oom_kb = CMM_OOM_KB;
50static unsigned int cmm_debug = CMM_DEBUG;
51static unsigned int cmm_disabled = CMM_DISABLE;
52static unsigned long min_mem_mb = CMM_MIN_MEM_MB;
53static bool __read_mostly simulate;
54static unsigned long simulate_loan_target_kb;
55static struct device cmm_dev;
56
57MODULE_AUTHOR("Brian King <brking@linux.vnet.ibm.com>");
58MODULE_DESCRIPTION("IBM System p Collaborative Memory Manager");
59MODULE_LICENSE("GPL");
60MODULE_VERSION(CMM_DRIVER_VERSION);
61
62module_param_named(delay, delay, uint, 0644);
63MODULE_PARM_DESC(delay, "Delay (in seconds) between polls to query hypervisor paging requests. "
64 "[Default=" __stringify(CMM_DEFAULT_DELAY) "]");
65module_param_named(hotplug_delay, hotplug_delay, uint, 0644);
66MODULE_PARM_DESC(hotplug_delay, "Delay (in seconds) after memory hotplug remove "
67 "before loaning resumes. "
68 "[Default=" __stringify(CMM_HOTPLUG_DELAY) "]");
69module_param_named(oom_kb, oom_kb, uint, 0644);
70MODULE_PARM_DESC(oom_kb, "Amount of memory in kb to free on OOM. "
71 "[Default=" __stringify(CMM_OOM_KB) "]");
72module_param_named(min_mem_mb, min_mem_mb, ulong, 0644);
73MODULE_PARM_DESC(min_mem_mb, "Minimum amount of memory (in MB) to not balloon. "
74 "[Default=" __stringify(CMM_MIN_MEM_MB) "]");
75module_param_named(debug, cmm_debug, uint, 0644);
76MODULE_PARM_DESC(debug, "Enable module debugging logging. Set to 1 to enable. "
77 "[Default=" __stringify(CMM_DEBUG) "]");
78module_param_named(simulate, simulate, bool, 0444);
79MODULE_PARM_DESC(simulate, "Enable simulation mode (no communication with hw).");
80
81#define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); }
82
83static atomic_long_t loaned_pages;
84static unsigned long loaned_pages_target;
85static unsigned long oom_freed_pages;
86
87static DEFINE_MUTEX(hotplug_mutex);
88static int hotplug_occurred;
89
90static struct task_struct *cmm_thread_ptr;
91static struct balloon_dev_info b_dev_info;
92
93static long plpar_page_set_loaned(struct page *page)
94{
95 const unsigned long vpa = page_to_phys(page);
96 unsigned long cmo_page_sz = cmo_get_page_size();
97 long rc = 0;
98 int i;
99
100 if (unlikely(simulate))
101 return 0;
102
103 for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
104 rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0);
105
106 for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
107 plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE,
108 vpa + i - cmo_page_sz, 0);
109
110 return rc;
111}
112
113static long plpar_page_set_active(struct page *page)
114{
115 const unsigned long vpa = page_to_phys(page);
116 unsigned long cmo_page_sz = cmo_get_page_size();
117 long rc = 0;
118 int i;
119
120 if (unlikely(simulate))
121 return 0;
122
123 for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
124 rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0);
125
126 for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
127 plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED,
128 vpa + i - cmo_page_sz, 0);
129
130 return rc;
131}
132
133
134
135
136
137
138
139
140static long cmm_alloc_pages(long nr)
141{
142 struct page *page;
143 long rc;
144
145 cmm_dbg("Begin request for %ld pages\n", nr);
146
147 while (nr) {
148
149 if (mutex_trylock(&hotplug_mutex)) {
150 if (hotplug_occurred) {
151 mutex_unlock(&hotplug_mutex);
152 break;
153 }
154 mutex_unlock(&hotplug_mutex);
155 } else {
156 break;
157 }
158
159 page = balloon_page_alloc();
160 if (!page)
161 break;
162 rc = plpar_page_set_loaned(page);
163 if (rc) {
164 pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc);
165 __free_page(page);
166 break;
167 }
168
169 balloon_page_enqueue(&b_dev_info, page);
170 atomic_long_inc(&loaned_pages);
171 adjust_managed_page_count(page, -1);
172 nr--;
173 }
174
175 cmm_dbg("End request with %ld pages unfulfilled\n", nr);
176 return nr;
177}
178
179
180
181
182
183
184
185
186static long cmm_free_pages(long nr)
187{
188 struct page *page;
189
190 cmm_dbg("Begin free of %ld pages.\n", nr);
191 while (nr) {
192 page = balloon_page_dequeue(&b_dev_info);
193 if (!page)
194 break;
195 plpar_page_set_active(page);
196 adjust_managed_page_count(page, 1);
197 __free_page(page);
198 atomic_long_dec(&loaned_pages);
199 nr--;
200 }
201 cmm_dbg("End request with %ld pages unfulfilled\n", nr);
202 return nr;
203}
204
205
206
207
208
209
210
211
212
213
214static int cmm_oom_notify(struct notifier_block *self,
215 unsigned long dummy, void *parm)
216{
217 unsigned long *freed = parm;
218 long nr = KB2PAGES(oom_kb);
219
220 cmm_dbg("OOM processing started\n");
221 nr = cmm_free_pages(nr);
222 loaned_pages_target = atomic_long_read(&loaned_pages);
223 *freed += KB2PAGES(oom_kb) - nr;
224 oom_freed_pages += KB2PAGES(oom_kb) - nr;
225 cmm_dbg("OOM processing complete\n");
226 return NOTIFY_OK;
227}
228
229
230
231
232
233
234
235
236
237static void cmm_get_mpp(void)
238{
239 const long __loaned_pages = atomic_long_read(&loaned_pages);
240 const long total_pages = totalram_pages() + __loaned_pages;
241 int rc;
242 struct hvcall_mpp_data mpp_data;
243 signed long active_pages_target, page_loan_request, target;
244 signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE;
245
246 if (likely(!simulate)) {
247 rc = h_get_mpp(&mpp_data);
248 if (rc != H_SUCCESS)
249 return;
250 page_loan_request = div_s64((s64)mpp_data.loan_request,
251 PAGE_SIZE);
252 target = page_loan_request + __loaned_pages;
253 } else {
254 target = KB2PAGES(simulate_loan_target_kb);
255 page_loan_request = target - __loaned_pages;
256 }
257
258 if (target < 0 || total_pages < min_mem_pages)
259 target = 0;
260
261 if (target > oom_freed_pages)
262 target -= oom_freed_pages;
263 else
264 target = 0;
265
266 active_pages_target = total_pages - target;
267
268 if (min_mem_pages > active_pages_target)
269 target = total_pages - min_mem_pages;
270
271 if (target < 0)
272 target = 0;
273
274 loaned_pages_target = target;
275
276 cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n",
277 page_loan_request, __loaned_pages, loaned_pages_target,
278 oom_freed_pages, totalram_pages());
279}
280
281static struct notifier_block cmm_oom_nb = {
282 .notifier_call = cmm_oom_notify
283};
284
285
286
287
288
289
290
291
292static int cmm_thread(void *dummy)
293{
294 unsigned long timeleft;
295 long __loaned_pages;
296
297 while (1) {
298 timeleft = msleep_interruptible(delay * 1000);
299
300 if (kthread_should_stop() || timeleft)
301 break;
302
303 if (mutex_trylock(&hotplug_mutex)) {
304 if (hotplug_occurred) {
305 hotplug_occurred = 0;
306 mutex_unlock(&hotplug_mutex);
307 cmm_dbg("Hotplug operation has occurred, "
308 "loaning activity suspended "
309 "for %d seconds.\n",
310 hotplug_delay);
311 timeleft = msleep_interruptible(hotplug_delay *
312 1000);
313 if (kthread_should_stop() || timeleft)
314 break;
315 continue;
316 }
317 mutex_unlock(&hotplug_mutex);
318 } else {
319 cmm_dbg("Hotplug operation in progress, activity "
320 "suspended\n");
321 continue;
322 }
323
324 cmm_get_mpp();
325
326 __loaned_pages = atomic_long_read(&loaned_pages);
327 if (loaned_pages_target > __loaned_pages) {
328 if (cmm_alloc_pages(loaned_pages_target - __loaned_pages))
329 loaned_pages_target = __loaned_pages;
330 } else if (loaned_pages_target < __loaned_pages)
331 cmm_free_pages(__loaned_pages - loaned_pages_target);
332 }
333 return 0;
334}
335
336#define CMM_SHOW(name, format, args...) \
337 static ssize_t show_##name(struct device *dev, \
338 struct device_attribute *attr, \
339 char *buf) \
340 { \
341 return sprintf(buf, format, ##args); \
342 } \
343 static DEVICE_ATTR(name, 0444, show_##name, NULL)
344
345CMM_SHOW(loaned_kb, "%lu\n", PAGES2KB(atomic_long_read(&loaned_pages)));
346CMM_SHOW(loaned_target_kb, "%lu\n", PAGES2KB(loaned_pages_target));
347
348static ssize_t show_oom_pages(struct device *dev,
349 struct device_attribute *attr, char *buf)
350{
351 return sprintf(buf, "%lu\n", PAGES2KB(oom_freed_pages));
352}
353
354static ssize_t store_oom_pages(struct device *dev,
355 struct device_attribute *attr,
356 const char *buf, size_t count)
357{
358 unsigned long val = simple_strtoul (buf, NULL, 10);
359
360 if (!capable(CAP_SYS_ADMIN))
361 return -EPERM;
362 if (val != 0)
363 return -EBADMSG;
364
365 oom_freed_pages = 0;
366 return count;
367}
368
369static DEVICE_ATTR(oom_freed_kb, 0644,
370 show_oom_pages, store_oom_pages);
371
372static struct device_attribute *cmm_attrs[] = {
373 &dev_attr_loaned_kb,
374 &dev_attr_loaned_target_kb,
375 &dev_attr_oom_freed_kb,
376};
377
378static DEVICE_ULONG_ATTR(simulate_loan_target_kb, 0644,
379 simulate_loan_target_kb);
380
381static struct bus_type cmm_subsys = {
382 .name = "cmm",
383 .dev_name = "cmm",
384};
385
386static void cmm_release_device(struct device *dev)
387{
388}
389
390
391
392
393
394
395
396static int cmm_sysfs_register(struct device *dev)
397{
398 int i, rc;
399
400 if ((rc = subsys_system_register(&cmm_subsys, NULL)))
401 return rc;
402
403 dev->id = 0;
404 dev->bus = &cmm_subsys;
405 dev->release = cmm_release_device;
406
407 if ((rc = device_register(dev)))
408 goto subsys_unregister;
409
410 for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) {
411 if ((rc = device_create_file(dev, cmm_attrs[i])))
412 goto fail;
413 }
414
415 if (!simulate)
416 return 0;
417 rc = device_create_file(dev, &dev_attr_simulate_loan_target_kb.attr);
418 if (rc)
419 goto fail;
420 return 0;
421
422fail:
423 while (--i >= 0)
424 device_remove_file(dev, cmm_attrs[i]);
425 device_unregister(dev);
426subsys_unregister:
427 bus_unregister(&cmm_subsys);
428 return rc;
429}
430
431
432
433
434
435static void cmm_unregister_sysfs(struct device *dev)
436{
437 int i;
438
439 for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++)
440 device_remove_file(dev, cmm_attrs[i]);
441 device_unregister(dev);
442 bus_unregister(&cmm_subsys);
443}
444
445
446
447
448
449static int cmm_reboot_notifier(struct notifier_block *nb,
450 unsigned long action, void *unused)
451{
452 if (action == SYS_RESTART) {
453 if (cmm_thread_ptr)
454 kthread_stop(cmm_thread_ptr);
455 cmm_thread_ptr = NULL;
456 cmm_free_pages(atomic_long_read(&loaned_pages));
457 }
458 return NOTIFY_DONE;
459}
460
461static struct notifier_block cmm_reboot_nb = {
462 .notifier_call = cmm_reboot_notifier,
463};
464
465
466
467
468
469
470
471
472
473
474
475static int cmm_memory_cb(struct notifier_block *self,
476 unsigned long action, void *arg)
477{
478 int ret = 0;
479
480 switch (action) {
481 case MEM_GOING_OFFLINE:
482 mutex_lock(&hotplug_mutex);
483 hotplug_occurred = 1;
484 break;
485 case MEM_OFFLINE:
486 case MEM_CANCEL_OFFLINE:
487 mutex_unlock(&hotplug_mutex);
488 cmm_dbg("Memory offline operation complete.\n");
489 break;
490 case MEM_GOING_ONLINE:
491 case MEM_ONLINE:
492 case MEM_CANCEL_ONLINE:
493 break;
494 }
495
496 return notifier_from_errno(ret);
497}
498
499static struct notifier_block cmm_mem_nb = {
500 .notifier_call = cmm_memory_cb,
501 .priority = CMM_MEM_HOTPLUG_PRI
502};
503
504#ifdef CONFIG_BALLOON_COMPACTION
505static struct vfsmount *balloon_mnt;
506
507static int cmm_init_fs_context(struct fs_context *fc)
508{
509 return init_pseudo(fc, PPC_CMM_MAGIC) ? 0 : -ENOMEM;
510}
511
512static struct file_system_type balloon_fs = {
513 .name = "ppc-cmm",
514 .init_fs_context = cmm_init_fs_context,
515 .kill_sb = kill_anon_super,
516};
517
518static int cmm_migratepage(struct balloon_dev_info *b_dev_info,
519 struct page *newpage, struct page *page,
520 enum migrate_mode mode)
521{
522 unsigned long flags;
523
524
525
526
527
528
529
530
531
532 if (plpar_page_set_loaned(newpage)) {
533
534 pr_err_ratelimited("%s: Cannot set page to loaned.", __func__);
535 return -EBUSY;
536 }
537
538
539 get_page(newpage);
540
541
542
543
544
545
546 if (page_zone(page) != page_zone(newpage)) {
547 adjust_managed_page_count(page, 1);
548 adjust_managed_page_count(newpage, -1);
549 }
550
551 spin_lock_irqsave(&b_dev_info->pages_lock, flags);
552 balloon_page_insert(b_dev_info, newpage);
553 balloon_page_delete(page);
554 b_dev_info->isolated_pages--;
555 spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
556
557
558
559
560
561 plpar_page_set_active(page);
562
563
564 put_page(page);
565
566 return MIGRATEPAGE_SUCCESS;
567}
568
569static int cmm_balloon_compaction_init(void)
570{
571 int rc;
572
573 balloon_devinfo_init(&b_dev_info);
574 b_dev_info.migratepage = cmm_migratepage;
575
576 balloon_mnt = kern_mount(&balloon_fs);
577 if (IS_ERR(balloon_mnt)) {
578 rc = PTR_ERR(balloon_mnt);
579 balloon_mnt = NULL;
580 return rc;
581 }
582
583 b_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
584 if (IS_ERR(b_dev_info.inode)) {
585 rc = PTR_ERR(b_dev_info.inode);
586 b_dev_info.inode = NULL;
587 kern_unmount(balloon_mnt);
588 balloon_mnt = NULL;
589 return rc;
590 }
591
592 b_dev_info.inode->i_mapping->a_ops = &balloon_aops;
593 return 0;
594}
595static void cmm_balloon_compaction_deinit(void)
596{
597 if (b_dev_info.inode)
598 iput(b_dev_info.inode);
599 b_dev_info.inode = NULL;
600 kern_unmount(balloon_mnt);
601 balloon_mnt = NULL;
602}
603#else
604static int cmm_balloon_compaction_init(void)
605{
606 return 0;
607}
608
609static void cmm_balloon_compaction_deinit(void)
610{
611}
612#endif
613
614
615
616
617
618
619
620static int cmm_init(void)
621{
622 int rc;
623
624 if (!firmware_has_feature(FW_FEATURE_CMO) && !simulate)
625 return -EOPNOTSUPP;
626
627 rc = cmm_balloon_compaction_init();
628 if (rc)
629 return rc;
630
631 rc = register_oom_notifier(&cmm_oom_nb);
632 if (rc < 0)
633 goto out_balloon_compaction;
634
635 if ((rc = register_reboot_notifier(&cmm_reboot_nb)))
636 goto out_oom_notifier;
637
638 if ((rc = cmm_sysfs_register(&cmm_dev)))
639 goto out_reboot_notifier;
640
641 rc = register_memory_notifier(&cmm_mem_nb);
642 if (rc)
643 goto out_unregister_notifier;
644
645 if (cmm_disabled)
646 return 0;
647
648 cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread");
649 if (IS_ERR(cmm_thread_ptr)) {
650 rc = PTR_ERR(cmm_thread_ptr);
651 goto out_unregister_notifier;
652 }
653
654 return 0;
655out_unregister_notifier:
656 unregister_memory_notifier(&cmm_mem_nb);
657 cmm_unregister_sysfs(&cmm_dev);
658out_reboot_notifier:
659 unregister_reboot_notifier(&cmm_reboot_nb);
660out_oom_notifier:
661 unregister_oom_notifier(&cmm_oom_nb);
662out_balloon_compaction:
663 cmm_balloon_compaction_deinit();
664 return rc;
665}
666
667
668
669
670
671
672
673static void cmm_exit(void)
674{
675 if (cmm_thread_ptr)
676 kthread_stop(cmm_thread_ptr);
677 unregister_oom_notifier(&cmm_oom_nb);
678 unregister_reboot_notifier(&cmm_reboot_nb);
679 unregister_memory_notifier(&cmm_mem_nb);
680 cmm_free_pages(atomic_long_read(&loaned_pages));
681 cmm_unregister_sysfs(&cmm_dev);
682 cmm_balloon_compaction_deinit();
683}
684
685
686
687
688
689
690
691static int cmm_set_disable(const char *val, const struct kernel_param *kp)
692{
693 int disable = simple_strtoul(val, NULL, 10);
694
695 if (disable != 0 && disable != 1)
696 return -EINVAL;
697
698 if (disable && !cmm_disabled) {
699 if (cmm_thread_ptr)
700 kthread_stop(cmm_thread_ptr);
701 cmm_thread_ptr = NULL;
702 cmm_free_pages(atomic_long_read(&loaned_pages));
703 } else if (!disable && cmm_disabled) {
704 cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread");
705 if (IS_ERR(cmm_thread_ptr))
706 return PTR_ERR(cmm_thread_ptr);
707 }
708
709 cmm_disabled = disable;
710 return 0;
711}
712
713module_param_call(disable, cmm_set_disable, param_get_uint,
714 &cmm_disabled, 0644);
715MODULE_PARM_DESC(disable, "Disable CMM. Set to 1 to disable. "
716 "[Default=" __stringify(CMM_DISABLE) "]");
717
718module_init(cmm_init);
719module_exit(cmm_exit);
720