1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/module.h>
15#include <linux/init.h>
16#include <linux/topology.h>
17#include <linux/capability.h>
18#include <linux/device.h>
19#include <linux/memory.h>
20#include <linux/memory_hotplug.h>
21#include <linux/mm.h>
22#include <linux/stat.h>
23#include <linux/slab.h>
24
25#include <linux/atomic.h>
26#include <linux/uaccess.h>
27
28#define MEMORY_CLASS_NAME "memory"
29
30static const char *const online_type_to_str[] = {
31 [MMOP_OFFLINE] = "offline",
32 [MMOP_ONLINE] = "online",
33 [MMOP_ONLINE_KERNEL] = "online_kernel",
34 [MMOP_ONLINE_MOVABLE] = "online_movable",
35};
36
37int memhp_online_type_from_str(const char *str)
38{
39 int i;
40
41 for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) {
42 if (sysfs_streq(str, online_type_to_str[i]))
43 return i;
44 }
45 return -EINVAL;
46}
47
48#define to_memory_block(dev) container_of(dev, struct memory_block, dev)
49
50static int sections_per_block;
51
52static inline unsigned long base_memory_block_id(unsigned long section_nr)
53{
54 return section_nr / sections_per_block;
55}
56
57static inline unsigned long pfn_to_block_id(unsigned long pfn)
58{
59 return base_memory_block_id(pfn_to_section_nr(pfn));
60}
61
62static inline unsigned long phys_to_block_id(unsigned long phys)
63{
64 return pfn_to_block_id(PFN_DOWN(phys));
65}
66
67static int memory_subsys_online(struct device *dev);
68static int memory_subsys_offline(struct device *dev);
69
70static struct bus_type memory_subsys = {
71 .name = MEMORY_CLASS_NAME,
72 .dev_name = MEMORY_CLASS_NAME,
73 .online = memory_subsys_online,
74 .offline = memory_subsys_offline,
75};
76
77static BLOCKING_NOTIFIER_HEAD(memory_chain);
78
79int register_memory_notifier(struct notifier_block *nb)
80{
81 return blocking_notifier_chain_register(&memory_chain, nb);
82}
83EXPORT_SYMBOL(register_memory_notifier);
84
85void unregister_memory_notifier(struct notifier_block *nb)
86{
87 blocking_notifier_chain_unregister(&memory_chain, nb);
88}
89EXPORT_SYMBOL(unregister_memory_notifier);
90
91static void memory_block_release(struct device *dev)
92{
93 struct memory_block *mem = to_memory_block(dev);
94
95 kfree(mem);
96}
97
98unsigned long __weak memory_block_size_bytes(void)
99{
100 return MIN_MEMORY_BLOCK_SIZE;
101}
102EXPORT_SYMBOL_GPL(memory_block_size_bytes);
103
104
105
106
107static ssize_t phys_index_show(struct device *dev,
108 struct device_attribute *attr, char *buf)
109{
110 struct memory_block *mem = to_memory_block(dev);
111 unsigned long phys_index;
112
113 phys_index = mem->start_section_nr / sections_per_block;
114 return sprintf(buf, "%08lx\n", phys_index);
115}
116
117
118
119
120
121static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
122 char *buf)
123{
124 return sprintf(buf, "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE));
125}
126
127
128
129
130static ssize_t state_show(struct device *dev, struct device_attribute *attr,
131 char *buf)
132{
133 struct memory_block *mem = to_memory_block(dev);
134 ssize_t len = 0;
135
136
137
138
139
140 switch (mem->state) {
141 case MEM_ONLINE:
142 len = sprintf(buf, "online\n");
143 break;
144 case MEM_OFFLINE:
145 len = sprintf(buf, "offline\n");
146 break;
147 case MEM_GOING_OFFLINE:
148 len = sprintf(buf, "going-offline\n");
149 break;
150 default:
151 len = sprintf(buf, "ERROR-UNKNOWN-%ld\n",
152 mem->state);
153 WARN_ON(1);
154 break;
155 }
156
157 return len;
158}
159
160int memory_notify(unsigned long val, void *v)
161{
162 return blocking_notifier_call_chain(&memory_chain, val, v);
163}
164
165
166
167
168
169static int
170memory_block_action(unsigned long start_section_nr, unsigned long action,
171 int online_type, int nid)
172{
173 unsigned long start_pfn;
174 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
175 int ret;
176
177 start_pfn = section_nr_to_pfn(start_section_nr);
178
179 switch (action) {
180 case MEM_ONLINE:
181 ret = online_pages(start_pfn, nr_pages, online_type, nid);
182 break;
183 case MEM_OFFLINE:
184 ret = offline_pages(start_pfn, nr_pages);
185 break;
186 default:
187 WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
188 "%ld\n", __func__, start_section_nr, action, action);
189 ret = -EINVAL;
190 }
191
192 return ret;
193}
194
195static int memory_block_change_state(struct memory_block *mem,
196 unsigned long to_state, unsigned long from_state_req)
197{
198 int ret = 0;
199
200 if (mem->state != from_state_req)
201 return -EINVAL;
202
203 if (to_state == MEM_OFFLINE)
204 mem->state = MEM_GOING_OFFLINE;
205
206 ret = memory_block_action(mem->start_section_nr, to_state,
207 mem->online_type, mem->nid);
208
209 mem->state = ret ? from_state_req : to_state;
210
211 return ret;
212}
213
214
215static int memory_subsys_online(struct device *dev)
216{
217 struct memory_block *mem = to_memory_block(dev);
218 int ret;
219
220 if (mem->state == MEM_ONLINE)
221 return 0;
222
223
224
225
226
227 if (mem->online_type == MMOP_OFFLINE)
228 mem->online_type = MMOP_ONLINE;
229
230 ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);
231 mem->online_type = MMOP_OFFLINE;
232
233 return ret;
234}
235
236static int memory_subsys_offline(struct device *dev)
237{
238 struct memory_block *mem = to_memory_block(dev);
239
240 if (mem->state == MEM_OFFLINE)
241 return 0;
242
243 return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);
244}
245
246static ssize_t state_store(struct device *dev, struct device_attribute *attr,
247 const char *buf, size_t count)
248{
249 const int online_type = memhp_online_type_from_str(buf);
250 struct memory_block *mem = to_memory_block(dev);
251 int ret;
252
253 if (online_type < 0)
254 return -EINVAL;
255
256 ret = lock_device_hotplug_sysfs();
257 if (ret)
258 return ret;
259
260 switch (online_type) {
261 case MMOP_ONLINE_KERNEL:
262 case MMOP_ONLINE_MOVABLE:
263 case MMOP_ONLINE:
264
265 mem->online_type = online_type;
266 ret = device_online(&mem->dev);
267 break;
268 case MMOP_OFFLINE:
269 ret = device_offline(&mem->dev);
270 break;
271 default:
272 ret = -EINVAL;
273 }
274
275 unlock_device_hotplug();
276
277 if (ret < 0)
278 return ret;
279 if (ret)
280 return -EINVAL;
281
282 return count;
283}
284
285
286
287
288
289
290
291
292
293
294static ssize_t phys_device_show(struct device *dev,
295 struct device_attribute *attr, char *buf)
296{
297 struct memory_block *mem = to_memory_block(dev);
298 return sprintf(buf, "%d\n", mem->phys_device);
299}
300
301#ifdef CONFIG_MEMORY_HOTREMOVE
302static void print_allowed_zone(char *buf, int nid, unsigned long start_pfn,
303 unsigned long nr_pages, int online_type,
304 struct zone *default_zone)
305{
306 struct zone *zone;
307
308 zone = zone_for_pfn_range(online_type, nid, start_pfn, nr_pages);
309 if (zone != default_zone) {
310 strcat(buf, " ");
311 strcat(buf, zone->name);
312 }
313}
314
315static ssize_t valid_zones_show(struct device *dev,
316 struct device_attribute *attr, char *buf)
317{
318 struct memory_block *mem = to_memory_block(dev);
319 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
320 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
321 struct zone *default_zone;
322 int nid;
323
324
325
326
327
328 if (mem->state == MEM_ONLINE) {
329
330
331
332
333 default_zone = test_pages_in_a_zone(start_pfn,
334 start_pfn + nr_pages);
335 if (!default_zone)
336 return sprintf(buf, "none\n");
337 strcat(buf, default_zone->name);
338 goto out;
339 }
340
341 nid = mem->nid;
342 default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, start_pfn,
343 nr_pages);
344 strcat(buf, default_zone->name);
345
346 print_allowed_zone(buf, nid, start_pfn, nr_pages, MMOP_ONLINE_KERNEL,
347 default_zone);
348 print_allowed_zone(buf, nid, start_pfn, nr_pages, MMOP_ONLINE_MOVABLE,
349 default_zone);
350out:
351 strcat(buf, "\n");
352
353 return strlen(buf);
354}
355static DEVICE_ATTR_RO(valid_zones);
356#endif
357
358static DEVICE_ATTR_RO(phys_index);
359static DEVICE_ATTR_RW(state);
360static DEVICE_ATTR_RO(phys_device);
361static DEVICE_ATTR_RO(removable);
362
363
364
365
366static ssize_t block_size_bytes_show(struct device *dev,
367 struct device_attribute *attr, char *buf)
368{
369 return sprintf(buf, "%lx\n", memory_block_size_bytes());
370}
371
372static DEVICE_ATTR_RO(block_size_bytes);
373
374
375
376
377
378static ssize_t auto_online_blocks_show(struct device *dev,
379 struct device_attribute *attr, char *buf)
380{
381 return sprintf(buf, "%s\n",
382 online_type_to_str[memhp_default_online_type]);
383}
384
385static ssize_t auto_online_blocks_store(struct device *dev,
386 struct device_attribute *attr,
387 const char *buf, size_t count)
388{
389 const int online_type = memhp_online_type_from_str(buf);
390
391 if (online_type < 0)
392 return -EINVAL;
393
394 memhp_default_online_type = online_type;
395 return count;
396}
397
398static DEVICE_ATTR_RW(auto_online_blocks);
399
400
401
402
403
404
405
406#ifdef CONFIG_ARCH_MEMORY_PROBE
407static ssize_t probe_store(struct device *dev, struct device_attribute *attr,
408 const char *buf, size_t count)
409{
410 u64 phys_addr;
411 int nid, ret;
412 unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
413
414 ret = kstrtoull(buf, 0, &phys_addr);
415 if (ret)
416 return ret;
417
418 if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
419 return -EINVAL;
420
421 ret = lock_device_hotplug_sysfs();
422 if (ret)
423 return ret;
424
425 nid = memory_add_physaddr_to_nid(phys_addr);
426 ret = __add_memory(nid, phys_addr,
427 MIN_MEMORY_BLOCK_SIZE * sections_per_block);
428
429 if (ret)
430 goto out;
431
432 ret = count;
433out:
434 unlock_device_hotplug();
435 return ret;
436}
437
438static DEVICE_ATTR_WO(probe);
439#endif
440
441#ifdef CONFIG_MEMORY_FAILURE
442
443
444
445
446
447static ssize_t soft_offline_page_store(struct device *dev,
448 struct device_attribute *attr,
449 const char *buf, size_t count)
450{
451 int ret;
452 u64 pfn;
453 if (!capable(CAP_SYS_ADMIN))
454 return -EPERM;
455 if (kstrtoull(buf, 0, &pfn) < 0)
456 return -EINVAL;
457 pfn >>= PAGE_SHIFT;
458 ret = soft_offline_page(pfn, 0);
459 return ret == 0 ? count : ret;
460}
461
462
463static ssize_t hard_offline_page_store(struct device *dev,
464 struct device_attribute *attr,
465 const char *buf, size_t count)
466{
467 int ret;
468 u64 pfn;
469 if (!capable(CAP_SYS_ADMIN))
470 return -EPERM;
471 if (kstrtoull(buf, 0, &pfn) < 0)
472 return -EINVAL;
473 pfn >>= PAGE_SHIFT;
474 ret = memory_failure(pfn, 0);
475 return ret ? ret : count;
476}
477
478static DEVICE_ATTR_WO(soft_offline_page);
479static DEVICE_ATTR_WO(hard_offline_page);
480#endif
481
482
483
484
485
486
487int __weak arch_get_memory_phys_device(unsigned long start_pfn)
488{
489 return 0;
490}
491
492
493static struct memory_block *find_memory_block_by_id(unsigned long block_id)
494{
495 struct device *dev;
496
497 dev = subsys_find_device_by_id(&memory_subsys, block_id, NULL);
498 return dev ? to_memory_block(dev) : NULL;
499}
500
501
502
503
504
505
506
507
508
509struct memory_block *find_memory_block(struct mem_section *section)
510{
511 unsigned long block_id = base_memory_block_id(__section_nr(section));
512
513 return find_memory_block_by_id(block_id);
514}
515
516static struct attribute *memory_memblk_attrs[] = {
517 &dev_attr_phys_index.attr,
518 &dev_attr_state.attr,
519 &dev_attr_phys_device.attr,
520 &dev_attr_removable.attr,
521#ifdef CONFIG_MEMORY_HOTREMOVE
522 &dev_attr_valid_zones.attr,
523#endif
524 NULL
525};
526
527static struct attribute_group memory_memblk_attr_group = {
528 .attrs = memory_memblk_attrs,
529};
530
531static const struct attribute_group *memory_memblk_attr_groups[] = {
532 &memory_memblk_attr_group,
533 NULL,
534};
535
536
537
538
539static
540int register_memory(struct memory_block *memory)
541{
542 int ret;
543
544 memory->dev.bus = &memory_subsys;
545 memory->dev.id = memory->start_section_nr / sections_per_block;
546 memory->dev.release = memory_block_release;
547 memory->dev.groups = memory_memblk_attr_groups;
548 memory->dev.offline = memory->state == MEM_OFFLINE;
549
550 ret = device_register(&memory->dev);
551 if (ret)
552 put_device(&memory->dev);
553
554 return ret;
555}
556
557static int init_memory_block(struct memory_block **memory,
558 unsigned long block_id, unsigned long state)
559{
560 struct memory_block *mem;
561 unsigned long start_pfn;
562 int ret = 0;
563
564 mem = find_memory_block_by_id(block_id);
565 if (mem) {
566 put_device(&mem->dev);
567 return -EEXIST;
568 }
569 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
570 if (!mem)
571 return -ENOMEM;
572
573 mem->start_section_nr = block_id * sections_per_block;
574 mem->state = state;
575 start_pfn = section_nr_to_pfn(mem->start_section_nr);
576 mem->phys_device = arch_get_memory_phys_device(start_pfn);
577 mem->nid = NUMA_NO_NODE;
578
579 ret = register_memory(mem);
580
581 *memory = mem;
582 return ret;
583}
584
585static int add_memory_block(unsigned long base_section_nr)
586{
587 int section_count = 0;
588 struct memory_block *mem;
589 unsigned long nr;
590
591 for (nr = base_section_nr; nr < base_section_nr + sections_per_block;
592 nr++)
593 if (present_section_nr(nr))
594 section_count++;
595
596 if (section_count == 0)
597 return 0;
598 return init_memory_block(&mem, base_memory_block_id(base_section_nr),
599 MEM_ONLINE);
600}
601
602static void unregister_memory(struct memory_block *memory)
603{
604 if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys))
605 return;
606
607
608 put_device(&memory->dev);
609 device_unregister(&memory->dev);
610}
611
612
613
614
615
616
617
618
619int create_memory_block_devices(unsigned long start, unsigned long size)
620{
621 const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
622 unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
623 struct memory_block *mem;
624 unsigned long block_id;
625 int ret = 0;
626
627 if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
628 !IS_ALIGNED(size, memory_block_size_bytes())))
629 return -EINVAL;
630
631 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
632 ret = init_memory_block(&mem, block_id, MEM_OFFLINE);
633 if (ret)
634 break;
635 }
636 if (ret) {
637 end_block_id = block_id;
638 for (block_id = start_block_id; block_id != end_block_id;
639 block_id++) {
640 mem = find_memory_block_by_id(block_id);
641 if (WARN_ON_ONCE(!mem))
642 continue;
643 unregister_memory(mem);
644 }
645 }
646 return ret;
647}
648
649
650
651
652
653
654
655
656void remove_memory_block_devices(unsigned long start, unsigned long size)
657{
658 const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
659 const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
660 struct memory_block *mem;
661 unsigned long block_id;
662
663 if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
664 !IS_ALIGNED(size, memory_block_size_bytes())))
665 return;
666
667 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
668 mem = find_memory_block_by_id(block_id);
669 if (WARN_ON_ONCE(!mem))
670 continue;
671 unregister_memory_block_under_nodes(mem);
672 unregister_memory(mem);
673 }
674}
675
676
677bool is_memblock_offlined(struct memory_block *mem)
678{
679 return mem->state == MEM_OFFLINE;
680}
681
682static struct attribute *memory_root_attrs[] = {
683#ifdef CONFIG_ARCH_MEMORY_PROBE
684 &dev_attr_probe.attr,
685#endif
686
687#ifdef CONFIG_MEMORY_FAILURE
688 &dev_attr_soft_offline_page.attr,
689 &dev_attr_hard_offline_page.attr,
690#endif
691
692 &dev_attr_block_size_bytes.attr,
693 &dev_attr_auto_online_blocks.attr,
694 NULL
695};
696
697static struct attribute_group memory_root_attr_group = {
698 .attrs = memory_root_attrs,
699};
700
701static const struct attribute_group *memory_root_attr_groups[] = {
702 &memory_root_attr_group,
703 NULL,
704};
705
706
707
708
709
710
711void __init memory_dev_init(void)
712{
713 int ret;
714 unsigned long block_sz, nr;
715
716
717 block_sz = memory_block_size_bytes();
718 if (!is_power_of_2(block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE)
719 panic("Memory block size not suitable: 0x%lx\n", block_sz);
720 sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
721
722 ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
723 if (ret)
724 panic("%s() failed to register subsystem: %d\n", __func__, ret);
725
726
727
728
729
730 for (nr = 0; nr <= __highest_present_section_nr;
731 nr += sections_per_block) {
732 ret = add_memory_block(nr);
733 if (ret)
734 panic("%s() failed to add memory block: %d\n", __func__,
735 ret);
736 }
737}
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754int walk_memory_blocks(unsigned long start, unsigned long size,
755 void *arg, walk_memory_blocks_func_t func)
756{
757 const unsigned long start_block_id = phys_to_block_id(start);
758 const unsigned long end_block_id = phys_to_block_id(start + size - 1);
759 struct memory_block *mem;
760 unsigned long block_id;
761 int ret = 0;
762
763 if (!size)
764 return 0;
765
766 for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
767 mem = find_memory_block_by_id(block_id);
768 if (!mem)
769 continue;
770
771 ret = func(mem, arg);
772 put_device(&mem->dev);
773 if (ret)
774 break;
775 }
776 return ret;
777}
778
779struct for_each_memory_block_cb_data {
780 walk_memory_blocks_func_t func;
781 void *arg;
782};
783
784static int for_each_memory_block_cb(struct device *dev, void *data)
785{
786 struct memory_block *mem = to_memory_block(dev);
787 struct for_each_memory_block_cb_data *cb_data = data;
788
789 return cb_data->func(mem, cb_data->arg);
790}
791
792
793
794
795
796
797
798
799
800
801
802
803
804int for_each_memory_block(void *arg, walk_memory_blocks_func_t func)
805{
806 struct for_each_memory_block_cb_data cb_data = {
807 .func = func,
808 .arg = arg,
809 };
810
811 return bus_for_each_dev(&memory_subsys, NULL, &cb_data,
812 for_each_memory_block_cb);
813}
814