1
2
3
4
5
6
7
8#include <linux/stddef.h>
9#include <linux/mm.h>
10#include <linux/sched/signal.h>
11#include <linux/swap.h>
12#include <linux/interrupt.h>
13#include <linux/pagemap.h>
14#include <linux/compiler.h>
15#include <linux/export.h>
16#include <linux/pagevec.h>
17#include <linux/writeback.h>
18#include <linux/slab.h>
19#include <linux/sysctl.h>
20#include <linux/cpu.h>
21#include <linux/memory.h>
22#include <linux/memremap.h>
23#include <linux/memory_hotplug.h>
24#include <linux/highmem.h>
25#include <linux/vmalloc.h>
26#include <linux/ioport.h>
27#include <linux/delay.h>
28#include <linux/migrate.h>
29#include <linux/page-isolation.h>
30#include <linux/pfn.h>
31#include <linux/suspend.h>
32#include <linux/mm_inline.h>
33#include <linux/firmware-map.h>
34#include <linux/stop_machine.h>
35#include <linux/hugetlb.h>
36#include <linux/memblock.h>
37#include <linux/compaction.h>
38#include <linux/rmap.h>
39
40#include <asm/tlbflush.h>
41
42#include "internal.h"
43#include "shuffle.h"
44
45
46
47
48
49
50
51
52static void generic_online_page(struct page *page, unsigned int order);
53
54static online_page_callback_t online_page_callback = generic_online_page;
55static DEFINE_MUTEX(online_page_callback_lock);
56
57DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock);
58
59void get_online_mems(void)
60{
61 percpu_down_read(&mem_hotplug_lock);
62}
63
64void put_online_mems(void)
65{
66 percpu_up_read(&mem_hotplug_lock);
67}
68
69bool movable_node_enabled = false;
70
71#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE
72bool memhp_auto_online;
73#else
74bool memhp_auto_online = true;
75#endif
76EXPORT_SYMBOL_GPL(memhp_auto_online);
77
78static int __init setup_memhp_default_state(char *str)
79{
80 if (!strcmp(str, "online"))
81 memhp_auto_online = true;
82 else if (!strcmp(str, "offline"))
83 memhp_auto_online = false;
84
85 return 1;
86}
87__setup("memhp_default_state=", setup_memhp_default_state);
88
89void mem_hotplug_begin(void)
90{
91 cpus_read_lock();
92 percpu_down_write(&mem_hotplug_lock);
93}
94
95void mem_hotplug_done(void)
96{
97 percpu_up_write(&mem_hotplug_lock);
98 cpus_read_unlock();
99}
100
101u64 max_mem_size = U64_MAX;
102
103
104static struct resource *register_memory_resource(u64 start, u64 size)
105{
106 struct resource *res;
107 unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
108 char *resource_name = "System RAM";
109
110 if (start + size > max_mem_size)
111 return ERR_PTR(-E2BIG);
112
113
114
115
116
117
118 res = __request_region(&iomem_resource, start, size,
119 resource_name, flags);
120
121 if (!res) {
122 pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",
123 start, start + size);
124 return ERR_PTR(-EEXIST);
125 }
126 return res;
127}
128
129static void release_memory_resource(struct resource *res)
130{
131 if (!res)
132 return;
133 release_resource(res);
134 kfree(res);
135 return;
136}
137
138#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
139void get_page_bootmem(unsigned long info, struct page *page,
140 unsigned long type)
141{
142 page->freelist = (void *)type;
143 SetPagePrivate(page);
144 set_page_private(page, info);
145 page_ref_inc(page);
146}
147
148void put_page_bootmem(struct page *page)
149{
150 unsigned long type;
151
152 type = (unsigned long) page->freelist;
153 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
154 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
155
156 if (page_ref_dec_return(page) == 1) {
157 page->freelist = NULL;
158 ClearPagePrivate(page);
159 set_page_private(page, 0);
160 INIT_LIST_HEAD(&page->lru);
161 free_reserved_page(page);
162 }
163}
164
165#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
166#ifndef CONFIG_SPARSEMEM_VMEMMAP
167static void register_page_bootmem_info_section(unsigned long start_pfn)
168{
169 unsigned long *usemap, mapsize, section_nr, i;
170 struct mem_section *ms;
171 struct page *page, *memmap;
172
173 section_nr = pfn_to_section_nr(start_pfn);
174 ms = __nr_to_section(section_nr);
175
176
177 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
178
179
180
181
182
183 page = virt_to_page(memmap);
184 mapsize = sizeof(struct page) * PAGES_PER_SECTION;
185 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
186
187
188 for (i = 0; i < mapsize; i++, page++)
189 get_page_bootmem(section_nr, page, SECTION_INFO);
190
191 usemap = ms->pageblock_flags;
192 page = virt_to_page(usemap);
193
194 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
195
196 for (i = 0; i < mapsize; i++, page++)
197 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
198
199}
200#else
201static void register_page_bootmem_info_section(unsigned long start_pfn)
202{
203 unsigned long *usemap, mapsize, section_nr, i;
204 struct mem_section *ms;
205 struct page *page, *memmap;
206
207 section_nr = pfn_to_section_nr(start_pfn);
208 ms = __nr_to_section(section_nr);
209
210 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
211
212 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
213
214 usemap = ms->pageblock_flags;
215 page = virt_to_page(usemap);
216
217 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
218
219 for (i = 0; i < mapsize; i++, page++)
220 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
221}
222#endif
223
224void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
225{
226 unsigned long i, pfn, end_pfn, nr_pages;
227 int node = pgdat->node_id;
228 struct page *page;
229
230 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
231 page = virt_to_page(pgdat);
232
233 for (i = 0; i < nr_pages; i++, page++)
234 get_page_bootmem(node, page, NODE_INFO);
235
236 pfn = pgdat->node_start_pfn;
237 end_pfn = pgdat_end_pfn(pgdat);
238
239
240 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
241
242
243
244
245
246
247 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
248 register_page_bootmem_info_section(pfn);
249 }
250}
251#endif
252
253static int __meminit __add_section(int nid, unsigned long phys_start_pfn,
254 struct vmem_altmap *altmap, bool want_memblock)
255{
256 int ret;
257
258 if (pfn_valid(phys_start_pfn))
259 return -EEXIST;
260
261 ret = sparse_add_one_section(nid, phys_start_pfn, altmap);
262 if (ret < 0)
263 return ret;
264
265 if (!want_memblock)
266 return 0;
267
268 return hotplug_memory_register(nid, __pfn_to_section(phys_start_pfn));
269}
270
271
272
273
274
275
276
277int __ref __add_pages(int nid, unsigned long phys_start_pfn,
278 unsigned long nr_pages, struct mhp_restrictions *restrictions)
279{
280 unsigned long i;
281 int err = 0;
282 int start_sec, end_sec;
283 struct vmem_altmap *altmap = restrictions->altmap;
284
285
286 start_sec = pfn_to_section_nr(phys_start_pfn);
287 end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);
288
289 if (altmap) {
290
291
292
293 if (altmap->base_pfn != phys_start_pfn
294 || vmem_altmap_offset(altmap) > nr_pages) {
295 pr_warn_once("memory add fail, invalid altmap\n");
296 err = -EINVAL;
297 goto out;
298 }
299 altmap->alloc = 0;
300 }
301
302 for (i = start_sec; i <= end_sec; i++) {
303 err = __add_section(nid, section_nr_to_pfn(i), altmap,
304 restrictions->flags & MHP_MEMBLOCK_API);
305
306
307
308
309
310
311 if (err && (err != -EEXIST))
312 break;
313 err = 0;
314 cond_resched();
315 }
316 vmemmap_populate_print_last();
317out:
318 return err;
319}
320
321#ifdef CONFIG_MEMORY_HOTREMOVE
322
323static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,
324 unsigned long start_pfn,
325 unsigned long end_pfn)
326{
327 struct mem_section *ms;
328
329 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
330 ms = __pfn_to_section(start_pfn);
331
332 if (unlikely(!valid_section(ms)))
333 continue;
334
335 if (unlikely(pfn_to_nid(start_pfn) != nid))
336 continue;
337
338 if (zone && zone != page_zone(pfn_to_page(start_pfn)))
339 continue;
340
341 return start_pfn;
342 }
343
344 return 0;
345}
346
347
348static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,
349 unsigned long start_pfn,
350 unsigned long end_pfn)
351{
352 struct mem_section *ms;
353 unsigned long pfn;
354
355
356 pfn = end_pfn - 1;
357 for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
358 ms = __pfn_to_section(pfn);
359
360 if (unlikely(!valid_section(ms)))
361 continue;
362
363 if (unlikely(pfn_to_nid(pfn) != nid))
364 continue;
365
366 if (zone && zone != page_zone(pfn_to_page(pfn)))
367 continue;
368
369 return pfn;
370 }
371
372 return 0;
373}
374
375static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
376 unsigned long end_pfn)
377{
378 unsigned long zone_start_pfn = zone->zone_start_pfn;
379 unsigned long z = zone_end_pfn(zone);
380 unsigned long zone_end_pfn = z;
381 unsigned long pfn;
382 struct mem_section *ms;
383 int nid = zone_to_nid(zone);
384
385 zone_span_writelock(zone);
386 if (zone_start_pfn == start_pfn) {
387
388
389
390
391
392
393 pfn = find_smallest_section_pfn(nid, zone, end_pfn,
394 zone_end_pfn);
395 if (pfn) {
396 zone->zone_start_pfn = pfn;
397 zone->spanned_pages = zone_end_pfn - pfn;
398 }
399 } else if (zone_end_pfn == end_pfn) {
400
401
402
403
404
405
406 pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
407 start_pfn);
408 if (pfn)
409 zone->spanned_pages = pfn - zone_start_pfn + 1;
410 }
411
412
413
414
415
416
417
418 pfn = zone_start_pfn;
419 for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
420 ms = __pfn_to_section(pfn);
421
422 if (unlikely(!valid_section(ms)))
423 continue;
424
425 if (page_zone(pfn_to_page(pfn)) != zone)
426 continue;
427
428
429 if (start_pfn == pfn)
430 continue;
431
432
433 zone_span_writeunlock(zone);
434 return;
435 }
436
437
438 zone->zone_start_pfn = 0;
439 zone->spanned_pages = 0;
440 zone_span_writeunlock(zone);
441}
442
443static void shrink_pgdat_span(struct pglist_data *pgdat,
444 unsigned long start_pfn, unsigned long end_pfn)
445{
446 unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
447 unsigned long p = pgdat_end_pfn(pgdat);
448 unsigned long pgdat_end_pfn = p;
449 unsigned long pfn;
450 struct mem_section *ms;
451 int nid = pgdat->node_id;
452
453 if (pgdat_start_pfn == start_pfn) {
454
455
456
457
458
459
460 pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
461 pgdat_end_pfn);
462 if (pfn) {
463 pgdat->node_start_pfn = pfn;
464 pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
465 }
466 } else if (pgdat_end_pfn == end_pfn) {
467
468
469
470
471
472
473 pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
474 start_pfn);
475 if (pfn)
476 pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
477 }
478
479
480
481
482
483
484
485
486 pfn = pgdat_start_pfn;
487 for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
488 ms = __pfn_to_section(pfn);
489
490 if (unlikely(!valid_section(ms)))
491 continue;
492
493 if (pfn_to_nid(pfn) != nid)
494 continue;
495
496
497 if (start_pfn == pfn)
498 continue;
499
500
501 return;
502 }
503
504
505 pgdat->node_start_pfn = 0;
506 pgdat->node_spanned_pages = 0;
507}
508
509static void __remove_zone(struct zone *zone, unsigned long start_pfn)
510{
511 struct pglist_data *pgdat = zone->zone_pgdat;
512 int nr_pages = PAGES_PER_SECTION;
513 unsigned long flags;
514
515 pgdat_resize_lock(zone->zone_pgdat, &flags);
516 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
517 shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
518 pgdat_resize_unlock(zone->zone_pgdat, &flags);
519}
520
521static void __remove_section(struct zone *zone, struct mem_section *ms,
522 unsigned long map_offset,
523 struct vmem_altmap *altmap)
524{
525 unsigned long start_pfn;
526 int scn_nr;
527
528 if (WARN_ON_ONCE(!valid_section(ms)))
529 return;
530
531 unregister_memory_section(ms);
532
533 scn_nr = __section_nr(ms);
534 start_pfn = section_nr_to_pfn((unsigned long)scn_nr);
535 __remove_zone(zone, start_pfn);
536
537 sparse_remove_one_section(zone, ms, map_offset, altmap);
538}
539
540
541
542
543
544
545
546
547
548
549
550
551
552void __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
553 unsigned long nr_pages, struct vmem_altmap *altmap)
554{
555 unsigned long i;
556 unsigned long map_offset = 0;
557 int sections_to_remove;
558
559
560 if (is_dev_zone(zone)) {
561 if (altmap)
562 map_offset = vmem_altmap_offset(altmap);
563 }
564
565 clear_zone_contiguous(zone);
566
567
568
569
570 BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
571 BUG_ON(nr_pages % PAGES_PER_SECTION);
572
573 sections_to_remove = nr_pages / PAGES_PER_SECTION;
574 for (i = 0; i < sections_to_remove; i++) {
575 unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
576
577 cond_resched();
578 __remove_section(zone, __pfn_to_section(pfn), map_offset,
579 altmap);
580 map_offset = 0;
581 }
582
583 set_zone_contiguous(zone);
584}
585#endif
586
587int set_online_page_callback(online_page_callback_t callback)
588{
589 int rc = -EINVAL;
590
591 get_online_mems();
592 mutex_lock(&online_page_callback_lock);
593
594 if (online_page_callback == generic_online_page) {
595 online_page_callback = callback;
596 rc = 0;
597 }
598
599 mutex_unlock(&online_page_callback_lock);
600 put_online_mems();
601
602 return rc;
603}
604EXPORT_SYMBOL_GPL(set_online_page_callback);
605
606int restore_online_page_callback(online_page_callback_t callback)
607{
608 int rc = -EINVAL;
609
610 get_online_mems();
611 mutex_lock(&online_page_callback_lock);
612
613 if (online_page_callback == callback) {
614 online_page_callback = generic_online_page;
615 rc = 0;
616 }
617
618 mutex_unlock(&online_page_callback_lock);
619 put_online_mems();
620
621 return rc;
622}
623EXPORT_SYMBOL_GPL(restore_online_page_callback);
624
625void __online_page_set_limits(struct page *page)
626{
627}
628EXPORT_SYMBOL_GPL(__online_page_set_limits);
629
630void __online_page_increment_counters(struct page *page)
631{
632 adjust_managed_page_count(page, 1);
633}
634EXPORT_SYMBOL_GPL(__online_page_increment_counters);
635
636void __online_page_free(struct page *page)
637{
638 __free_reserved_page(page);
639}
640EXPORT_SYMBOL_GPL(__online_page_free);
641
642static void generic_online_page(struct page *page, unsigned int order)
643{
644 kernel_map_pages(page, 1 << order, 1);
645 __free_pages_core(page, order);
646 totalram_pages_add(1UL << order);
647#ifdef CONFIG_HIGHMEM
648 if (PageHighMem(page))
649 totalhigh_pages_add(1UL << order);
650#endif
651}
652
653static int online_pages_blocks(unsigned long start, unsigned long nr_pages)
654{
655 unsigned long end = start + nr_pages;
656 int order, onlined_pages = 0;
657
658 while (start < end) {
659 order = min(MAX_ORDER - 1,
660 get_order(PFN_PHYS(end) - PFN_PHYS(start)));
661 (*online_page_callback)(pfn_to_page(start), order);
662
663 onlined_pages += (1UL << order);
664 start += (1UL << order);
665 }
666 return onlined_pages;
667}
668
669static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
670 void *arg)
671{
672 unsigned long onlined_pages = *(unsigned long *)arg;
673
674 if (PageReserved(pfn_to_page(start_pfn)))
675 onlined_pages += online_pages_blocks(start_pfn, nr_pages);
676
677 online_mem_sections(start_pfn, start_pfn + nr_pages);
678
679 *(unsigned long *)arg = onlined_pages;
680 return 0;
681}
682
683
684static void node_states_check_changes_online(unsigned long nr_pages,
685 struct zone *zone, struct memory_notify *arg)
686{
687 int nid = zone_to_nid(zone);
688
689 arg->status_change_nid = NUMA_NO_NODE;
690 arg->status_change_nid_normal = NUMA_NO_NODE;
691 arg->status_change_nid_high = NUMA_NO_NODE;
692
693 if (!node_state(nid, N_MEMORY))
694 arg->status_change_nid = nid;
695 if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))
696 arg->status_change_nid_normal = nid;
697#ifdef CONFIG_HIGHMEM
698 if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY))
699 arg->status_change_nid_high = nid;
700#endif
701}
702
703static void node_states_set_node(int node, struct memory_notify *arg)
704{
705 if (arg->status_change_nid_normal >= 0)
706 node_set_state(node, N_NORMAL_MEMORY);
707
708 if (arg->status_change_nid_high >= 0)
709 node_set_state(node, N_HIGH_MEMORY);
710
711 if (arg->status_change_nid >= 0)
712 node_set_state(node, N_MEMORY);
713}
714
715static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,
716 unsigned long nr_pages)
717{
718 unsigned long old_end_pfn = zone_end_pfn(zone);
719
720 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
721 zone->zone_start_pfn = start_pfn;
722
723 zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;
724}
725
726static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn,
727 unsigned long nr_pages)
728{
729 unsigned long old_end_pfn = pgdat_end_pfn(pgdat);
730
731 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
732 pgdat->node_start_pfn = start_pfn;
733
734 pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn;
735}
736
737void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
738 unsigned long nr_pages, struct vmem_altmap *altmap)
739{
740 struct pglist_data *pgdat = zone->zone_pgdat;
741 int nid = pgdat->node_id;
742 unsigned long flags;
743
744 clear_zone_contiguous(zone);
745
746
747 pgdat_resize_lock(pgdat, &flags);
748 zone_span_writelock(zone);
749 if (zone_is_empty(zone))
750 init_currently_empty_zone(zone, start_pfn, nr_pages);
751 resize_zone_range(zone, start_pfn, nr_pages);
752 zone_span_writeunlock(zone);
753 resize_pgdat_range(pgdat, start_pfn, nr_pages);
754 pgdat_resize_unlock(pgdat, &flags);
755
756
757
758
759
760
761
762 memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn,
763 MEMMAP_HOTPLUG, altmap);
764
765 set_zone_contiguous(zone);
766}
767
768
769
770
771
772
773static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,
774 unsigned long nr_pages)
775{
776 struct pglist_data *pgdat = NODE_DATA(nid);
777 int zid;
778
779 for (zid = 0; zid <= ZONE_NORMAL; zid++) {
780 struct zone *zone = &pgdat->node_zones[zid];
781
782 if (zone_intersects(zone, start_pfn, nr_pages))
783 return zone;
784 }
785
786 return &pgdat->node_zones[ZONE_NORMAL];
787}
788
789static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,
790 unsigned long nr_pages)
791{
792 struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,
793 nr_pages);
794 struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
795 bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages);
796 bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages);
797
798
799
800
801
802 if (in_kernel ^ in_movable)
803 return (in_kernel) ? kernel_zone : movable_zone;
804
805
806
807
808
809
810 return movable_node_enabled ? movable_zone : kernel_zone;
811}
812
813struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn,
814 unsigned long nr_pages)
815{
816 if (online_type == MMOP_ONLINE_KERNEL)
817 return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages);
818
819 if (online_type == MMOP_ONLINE_MOVABLE)
820 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
821
822 return default_zone_for_pfn(nid, start_pfn, nr_pages);
823}
824
825
826
827
828
829static struct zone * __meminit move_pfn_range(int online_type, int nid,
830 unsigned long start_pfn, unsigned long nr_pages)
831{
832 struct zone *zone;
833
834 zone = zone_for_pfn_range(online_type, nid, start_pfn, nr_pages);
835 move_pfn_range_to_zone(zone, start_pfn, nr_pages, NULL);
836 return zone;
837}
838
839int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
840{
841 unsigned long flags;
842 unsigned long onlined_pages = 0;
843 struct zone *zone;
844 int need_zonelists_rebuild = 0;
845 int nid;
846 int ret;
847 struct memory_notify arg;
848 struct memory_block *mem;
849
850 mem_hotplug_begin();
851
852
853
854
855
856 mem = find_memory_block(__pfn_to_section(pfn));
857 nid = mem->nid;
858 put_device(&mem->dev);
859
860
861 zone = move_pfn_range(online_type, nid, pfn, nr_pages);
862
863 arg.start_pfn = pfn;
864 arg.nr_pages = nr_pages;
865 node_states_check_changes_online(nr_pages, zone, &arg);
866
867 ret = memory_notify(MEM_GOING_ONLINE, &arg);
868 ret = notifier_to_errno(ret);
869 if (ret)
870 goto failed_addition;
871
872
873
874
875
876
877 if (!populated_zone(zone)) {
878 need_zonelists_rebuild = 1;
879 setup_zone_pageset(zone);
880 }
881
882 ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
883 online_pages_range);
884 if (ret) {
885 if (need_zonelists_rebuild)
886 zone_pcp_reset(zone);
887 goto failed_addition;
888 }
889
890 zone->present_pages += onlined_pages;
891
892 pgdat_resize_lock(zone->zone_pgdat, &flags);
893 zone->zone_pgdat->node_present_pages += onlined_pages;
894 pgdat_resize_unlock(zone->zone_pgdat, &flags);
895
896 shuffle_zone(zone);
897
898 if (onlined_pages) {
899 node_states_set_node(nid, &arg);
900 if (need_zonelists_rebuild)
901 build_all_zonelists(NULL);
902 else
903 zone_pcp_update(zone);
904 }
905
906 init_per_zone_wmark_min();
907
908 if (onlined_pages) {
909 kswapd_run(nid);
910 kcompactd_run(nid);
911 }
912
913 vm_total_pages = nr_free_pagecache_pages();
914
915 writeback_set_ratelimit();
916
917 if (onlined_pages)
918 memory_notify(MEM_ONLINE, &arg);
919 mem_hotplug_done();
920 return 0;
921
922failed_addition:
923 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
924 (unsigned long long) pfn << PAGE_SHIFT,
925 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
926 memory_notify(MEM_CANCEL_ONLINE, &arg);
927 mem_hotplug_done();
928 return ret;
929}
930#endif
931
932static void reset_node_present_pages(pg_data_t *pgdat)
933{
934 struct zone *z;
935
936 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
937 z->present_pages = 0;
938
939 pgdat->node_present_pages = 0;
940}
941
942
943static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
944{
945 struct pglist_data *pgdat;
946 unsigned long start_pfn = PFN_DOWN(start);
947
948 pgdat = NODE_DATA(nid);
949 if (!pgdat) {
950 pgdat = arch_alloc_nodedata(nid);
951 if (!pgdat)
952 return NULL;
953
954 arch_refresh_nodedata(nid, pgdat);
955 } else {
956
957
958
959
960
961 pgdat->nr_zones = 0;
962 pgdat->kswapd_order = 0;
963 pgdat->kswapd_classzone_idx = 0;
964 }
965
966
967
968 pgdat->node_id = nid;
969 pgdat->node_start_pfn = start_pfn;
970
971
972 free_area_init_core_hotplug(nid);
973 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);
974
975
976
977
978
979 build_all_zonelists(pgdat);
980
981
982
983
984
985
986 reset_node_managed_pages(pgdat);
987 reset_node_present_pages(pgdat);
988
989 return pgdat;
990}
991
992static void rollback_node_hotadd(int nid)
993{
994 pg_data_t *pgdat = NODE_DATA(nid);
995
996 arch_refresh_nodedata(nid, NULL);
997 free_percpu(pgdat->per_cpu_nodestats);
998 arch_free_nodedata(pgdat);
999 return;
1000}
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015static int __try_online_node(int nid, u64 start, bool set_node_online)
1016{
1017 pg_data_t *pgdat;
1018 int ret = 1;
1019
1020 if (node_online(nid))
1021 return 0;
1022
1023 pgdat = hotadd_new_pgdat(nid, start);
1024 if (!pgdat) {
1025 pr_err("Cannot online node %d due to NULL pgdat\n", nid);
1026 ret = -ENOMEM;
1027 goto out;
1028 }
1029
1030 if (set_node_online) {
1031 node_set_online(nid);
1032 ret = register_one_node(nid);
1033 BUG_ON(ret);
1034 }
1035out:
1036 return ret;
1037}
1038
1039
1040
1041
1042int try_online_node(int nid)
1043{
1044 int ret;
1045
1046 mem_hotplug_begin();
1047 ret = __try_online_node(nid, 0, true);
1048 mem_hotplug_done();
1049 return ret;
1050}
1051
1052static int check_hotplug_memory_range(u64 start, u64 size)
1053{
1054 unsigned long block_sz = memory_block_size_bytes();
1055 u64 block_nr_pages = block_sz >> PAGE_SHIFT;
1056 u64 nr_pages = size >> PAGE_SHIFT;
1057 u64 start_pfn = PFN_DOWN(start);
1058
1059
1060 if (!nr_pages || !IS_ALIGNED(start_pfn, block_nr_pages) ||
1061 !IS_ALIGNED(nr_pages, block_nr_pages)) {
1062 pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",
1063 block_sz, start, size);
1064 return -EINVAL;
1065 }
1066
1067 return 0;
1068}
1069
1070static int online_memory_block(struct memory_block *mem, void *arg)
1071{
1072 return device_online(&mem->dev);
1073}
1074
1075
1076
1077
1078
1079
1080
1081int __ref add_memory_resource(int nid, struct resource *res)
1082{
1083 struct mhp_restrictions restrictions = {
1084 .flags = MHP_MEMBLOCK_API,
1085 };
1086 u64 start, size;
1087 bool new_node = false;
1088 int ret;
1089
1090 start = res->start;
1091 size = resource_size(res);
1092
1093 ret = check_hotplug_memory_range(start, size);
1094 if (ret)
1095 return ret;
1096
1097 mem_hotplug_begin();
1098
1099
1100
1101
1102
1103
1104
1105 memblock_add_node(start, size, nid);
1106
1107 ret = __try_online_node(nid, start, false);
1108 if (ret < 0)
1109 goto error;
1110 new_node = ret;
1111
1112
1113 ret = arch_add_memory(nid, start, size, &restrictions);
1114 if (ret < 0)
1115 goto error;
1116
1117 if (new_node) {
1118
1119
1120
1121
1122
1123 node_set_online(nid);
1124 ret = __register_one_node(nid);
1125 BUG_ON(ret);
1126 }
1127
1128
1129 ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1));
1130 BUG_ON(ret);
1131
1132
1133 firmware_map_add_hotplug(start, start + size, "System RAM");
1134
1135
1136 mem_hotplug_done();
1137
1138
1139 if (memhp_auto_online)
1140 walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1),
1141 NULL, online_memory_block);
1142
1143 return ret;
1144error:
1145
1146 if (new_node)
1147 rollback_node_hotadd(nid);
1148 memblock_remove(start, size);
1149 mem_hotplug_done();
1150 return ret;
1151}
1152
1153
1154int __ref __add_memory(int nid, u64 start, u64 size)
1155{
1156 struct resource *res;
1157 int ret;
1158
1159 res = register_memory_resource(start, size);
1160 if (IS_ERR(res))
1161 return PTR_ERR(res);
1162
1163 ret = add_memory_resource(nid, res);
1164 if (ret < 0)
1165 release_memory_resource(res);
1166 return ret;
1167}
1168
1169int add_memory(int nid, u64 start, u64 size)
1170{
1171 int rc;
1172
1173 lock_device_hotplug();
1174 rc = __add_memory(nid, start, size);
1175 unlock_device_hotplug();
1176
1177 return rc;
1178}
1179EXPORT_SYMBOL_GPL(add_memory);
1180
1181#ifdef CONFIG_MEMORY_HOTREMOVE
1182
1183
1184
1185
1186
1187
1188
1189static inline int pageblock_free(struct page *page)
1190{
1191 return PageBuddy(page) && page_order(page) >= pageblock_order;
1192}
1193
1194
1195static unsigned long next_active_pageblock(unsigned long pfn)
1196{
1197 struct page *page = pfn_to_page(pfn);
1198
1199
1200 BUG_ON(pfn & (pageblock_nr_pages - 1));
1201
1202
1203 if (pageblock_free(page)) {
1204 int order;
1205
1206 order = page_order(page);
1207 if ((order < MAX_ORDER) && (order >= pageblock_order))
1208 return pfn + (1 << order);
1209 }
1210
1211 return pfn + pageblock_nr_pages;
1212}
1213
1214static bool is_pageblock_removable_nolock(unsigned long pfn)
1215{
1216 struct page *page = pfn_to_page(pfn);
1217 struct zone *zone;
1218
1219
1220
1221
1222
1223
1224
1225
1226 if (!node_online(page_to_nid(page)))
1227 return false;
1228
1229 zone = page_zone(page);
1230 pfn = page_to_pfn(page);
1231 if (!zone_spans_pfn(zone, pfn))
1232 return false;
1233
1234 return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, SKIP_HWPOISON);
1235}
1236
1237
1238bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1239{
1240 unsigned long end_pfn, pfn;
1241
1242 end_pfn = min(start_pfn + nr_pages,
1243 zone_end_pfn(page_zone(pfn_to_page(start_pfn))));
1244
1245
1246 for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) {
1247 if (!is_pageblock_removable_nolock(pfn))
1248 return false;
1249 cond_resched();
1250 }
1251
1252
1253 return true;
1254}
1255
1256
1257
1258
1259
1260int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
1261 unsigned long *valid_start, unsigned long *valid_end)
1262{
1263 unsigned long pfn, sec_end_pfn;
1264 unsigned long start, end;
1265 struct zone *zone = NULL;
1266 struct page *page;
1267 int i;
1268 for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1);
1269 pfn < end_pfn;
1270 pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) {
1271
1272 if (!present_section_nr(pfn_to_section_nr(pfn)))
1273 continue;
1274 for (; pfn < sec_end_pfn && pfn < end_pfn;
1275 pfn += MAX_ORDER_NR_PAGES) {
1276 i = 0;
1277
1278 while ((i < MAX_ORDER_NR_PAGES) &&
1279 !pfn_valid_within(pfn + i))
1280 i++;
1281 if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn)
1282 continue;
1283
1284 if (zone && !zone_spans_pfn(zone, pfn + i))
1285 return 0;
1286 page = pfn_to_page(pfn + i);
1287 if (zone && page_zone(page) != zone)
1288 return 0;
1289 if (!zone)
1290 start = pfn + i;
1291 zone = page_zone(page);
1292 end = pfn + MAX_ORDER_NR_PAGES;
1293 }
1294 }
1295
1296 if (zone) {
1297 *valid_start = start;
1298 *valid_end = min(end, end_pfn);
1299 return 1;
1300 } else {
1301 return 0;
1302 }
1303}
1304
1305
1306
1307
1308
1309
1310
1311static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
1312{
1313 unsigned long pfn;
1314
1315 for (pfn = start; pfn < end; pfn++) {
1316 struct page *page, *head;
1317 unsigned long skip;
1318
1319 if (!pfn_valid(pfn))
1320 continue;
1321 page = pfn_to_page(pfn);
1322 if (PageLRU(page))
1323 return pfn;
1324 if (__PageMovable(page))
1325 return pfn;
1326
1327 if (!PageHuge(page))
1328 continue;
1329 head = compound_head(page);
1330 if (page_huge_active(head))
1331 return pfn;
1332 skip = (1 << compound_order(head)) - (page - head);
1333 pfn += skip - 1;
1334 }
1335 return 0;
1336}
1337
1338static struct page *new_node_page(struct page *page, unsigned long private)
1339{
1340 int nid = page_to_nid(page);
1341 nodemask_t nmask = node_states[N_MEMORY];
1342
1343
1344
1345
1346
1347
1348 node_clear(nid, nmask);
1349 if (nodes_empty(nmask))
1350 node_set(nid, nmask);
1351
1352 return new_page_nodemask(page, nid, &nmask);
1353}
1354
1355static int
1356do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
1357{
1358 unsigned long pfn;
1359 struct page *page;
1360 int ret = 0;
1361 LIST_HEAD(source);
1362
1363 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1364 if (!pfn_valid(pfn))
1365 continue;
1366 page = pfn_to_page(pfn);
1367
1368 if (PageHuge(page)) {
1369 struct page *head = compound_head(page);
1370 pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
1371 isolate_huge_page(head, &source);
1372 continue;
1373 } else if (PageTransHuge(page))
1374 pfn = page_to_pfn(compound_head(page))
1375 + hpage_nr_pages(page) - 1;
1376
1377
1378
1379
1380
1381
1382
1383
1384 if (PageHWPoison(page)) {
1385 if (WARN_ON(PageLRU(page)))
1386 isolate_lru_page(page);
1387 if (page_mapped(page))
1388 try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS);
1389 continue;
1390 }
1391
1392 if (!get_page_unless_zero(page))
1393 continue;
1394
1395
1396
1397
1398 if (PageLRU(page))
1399 ret = isolate_lru_page(page);
1400 else
1401 ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE);
1402 if (!ret) {
1403 list_add_tail(&page->lru, &source);
1404 if (!__PageMovable(page))
1405 inc_node_page_state(page, NR_ISOLATED_ANON +
1406 page_is_file_cache(page));
1407
1408 } else {
1409 pr_warn("failed to isolate pfn %lx\n", pfn);
1410 dump_page(page, "isolation failed");
1411 }
1412 put_page(page);
1413 }
1414 if (!list_empty(&source)) {
1415
1416 ret = migrate_pages(&source, new_node_page, NULL, 0,
1417 MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1418 if (ret) {
1419 list_for_each_entry(page, &source, lru) {
1420 pr_warn("migrating pfn %lx failed ret:%d ",
1421 page_to_pfn(page), ret);
1422 dump_page(page, "migration failure");
1423 }
1424 putback_movable_pages(&source);
1425 }
1426 }
1427
1428 return ret;
1429}
1430
1431
1432
1433
1434static int
1435offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
1436 void *data)
1437{
1438 unsigned long *offlined_pages = (unsigned long *)data;
1439
1440 *offlined_pages += __offline_isolated_pages(start, start + nr_pages);
1441 return 0;
1442}
1443
1444
1445
1446
1447static int
1448check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
1449 void *data)
1450{
1451 return test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
1452}
1453
1454static int __init cmdline_parse_movable_node(char *p)
1455{
1456#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1457 movable_node_enabled = true;
1458#else
1459 pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n");
1460#endif
1461 return 0;
1462}
1463early_param("movable_node", cmdline_parse_movable_node);
1464
1465
1466static void node_states_check_changes_offline(unsigned long nr_pages,
1467 struct zone *zone, struct memory_notify *arg)
1468{
1469 struct pglist_data *pgdat = zone->zone_pgdat;
1470 unsigned long present_pages = 0;
1471 enum zone_type zt;
1472
1473 arg->status_change_nid = NUMA_NO_NODE;
1474 arg->status_change_nid_normal = NUMA_NO_NODE;
1475 arg->status_change_nid_high = NUMA_NO_NODE;
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485 for (zt = 0; zt <= ZONE_NORMAL; zt++)
1486 present_pages += pgdat->node_zones[zt].present_pages;
1487 if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)
1488 arg->status_change_nid_normal = zone_to_nid(zone);
1489
1490#ifdef CONFIG_HIGHMEM
1491
1492
1493
1494
1495
1496
1497
1498
1499 present_pages += pgdat->node_zones[ZONE_HIGHMEM].present_pages;
1500 if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages)
1501 arg->status_change_nid_high = zone_to_nid(zone);
1502#endif
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514 present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages;
1515
1516 if (nr_pages >= present_pages)
1517 arg->status_change_nid = zone_to_nid(zone);
1518}
1519
1520static void node_states_clear_node(int node, struct memory_notify *arg)
1521{
1522 if (arg->status_change_nid_normal >= 0)
1523 node_clear_state(node, N_NORMAL_MEMORY);
1524
1525 if (arg->status_change_nid_high >= 0)
1526 node_clear_state(node, N_HIGH_MEMORY);
1527
1528 if (arg->status_change_nid >= 0)
1529 node_clear_state(node, N_MEMORY);
1530}
1531
1532static int __ref __offline_pages(unsigned long start_pfn,
1533 unsigned long end_pfn)
1534{
1535 unsigned long pfn, nr_pages;
1536 unsigned long offlined_pages = 0;
1537 int ret, node, nr_isolate_pageblock;
1538 unsigned long flags;
1539 unsigned long valid_start, valid_end;
1540 struct zone *zone;
1541 struct memory_notify arg;
1542 char *reason;
1543
1544 mem_hotplug_begin();
1545
1546
1547
1548 if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start,
1549 &valid_end)) {
1550 ret = -EINVAL;
1551 reason = "multizone range";
1552 goto failed_removal;
1553 }
1554
1555 zone = page_zone(pfn_to_page(valid_start));
1556 node = zone_to_nid(zone);
1557 nr_pages = end_pfn - start_pfn;
1558
1559
1560 ret = start_isolate_page_range(start_pfn, end_pfn,
1561 MIGRATE_MOVABLE,
1562 SKIP_HWPOISON | REPORT_FAILURE);
1563 if (ret < 0) {
1564 reason = "failure to isolate range";
1565 goto failed_removal;
1566 }
1567 nr_isolate_pageblock = ret;
1568
1569 arg.start_pfn = start_pfn;
1570 arg.nr_pages = nr_pages;
1571 node_states_check_changes_offline(nr_pages, zone, &arg);
1572
1573 ret = memory_notify(MEM_GOING_OFFLINE, &arg);
1574 ret = notifier_to_errno(ret);
1575 if (ret) {
1576 reason = "notifier failure";
1577 goto failed_removal_isolated;
1578 }
1579
1580 do {
1581 for (pfn = start_pfn; pfn;) {
1582 if (signal_pending(current)) {
1583 ret = -EINTR;
1584 reason = "signal backoff";
1585 goto failed_removal_isolated;
1586 }
1587
1588 cond_resched();
1589 lru_add_drain_all();
1590
1591 pfn = scan_movable_pages(pfn, end_pfn);
1592 if (pfn) {
1593
1594
1595
1596
1597 do_migrate_range(pfn, end_pfn);
1598 }
1599 }
1600
1601
1602
1603
1604
1605
1606 ret = dissolve_free_huge_pages(start_pfn, end_pfn);
1607 if (ret) {
1608 reason = "failure to dissolve huge pages";
1609 goto failed_removal_isolated;
1610 }
1611
1612 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
1613 NULL, check_pages_isolated_cb);
1614 } while (ret);
1615
1616
1617
1618 walk_system_ram_range(start_pfn, end_pfn - start_pfn,
1619 &offlined_pages, offline_isolated_pages_cb);
1620 pr_info("Offlined Pages %ld\n", offlined_pages);
1621
1622
1623
1624
1625
1626 spin_lock_irqsave(&zone->lock, flags);
1627 zone->nr_isolate_pageblock -= nr_isolate_pageblock;
1628 spin_unlock_irqrestore(&zone->lock, flags);
1629
1630
1631 adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
1632 zone->present_pages -= offlined_pages;
1633
1634 pgdat_resize_lock(zone->zone_pgdat, &flags);
1635 zone->zone_pgdat->node_present_pages -= offlined_pages;
1636 pgdat_resize_unlock(zone->zone_pgdat, &flags);
1637
1638 init_per_zone_wmark_min();
1639
1640 if (!populated_zone(zone)) {
1641 zone_pcp_reset(zone);
1642 build_all_zonelists(NULL);
1643 } else
1644 zone_pcp_update(zone);
1645
1646 node_states_clear_node(node, &arg);
1647 if (arg.status_change_nid >= 0) {
1648 kswapd_stop(node);
1649 kcompactd_stop(node);
1650 }
1651
1652 vm_total_pages = nr_free_pagecache_pages();
1653 writeback_set_ratelimit();
1654
1655 memory_notify(MEM_OFFLINE, &arg);
1656 mem_hotplug_done();
1657 return 0;
1658
1659failed_removal_isolated:
1660 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1661 memory_notify(MEM_CANCEL_OFFLINE, &arg);
1662failed_removal:
1663 pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n",
1664 (unsigned long long) start_pfn << PAGE_SHIFT,
1665 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1,
1666 reason);
1667
1668 mem_hotplug_done();
1669 return ret;
1670}
1671
1672int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
1673{
1674 return __offline_pages(start_pfn, start_pfn + nr_pages);
1675}
1676#endif
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
1691 void *arg, int (*func)(struct memory_block *, void *))
1692{
1693 struct memory_block *mem = NULL;
1694 struct mem_section *section;
1695 unsigned long pfn, section_nr;
1696 int ret;
1697
1698 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1699 section_nr = pfn_to_section_nr(pfn);
1700 if (!present_section_nr(section_nr))
1701 continue;
1702
1703 section = __nr_to_section(section_nr);
1704
1705 if (mem)
1706 if ((section_nr >= mem->start_section_nr) &&
1707 (section_nr <= mem->end_section_nr))
1708 continue;
1709
1710 mem = find_memory_block_hinted(section, mem);
1711 if (!mem)
1712 continue;
1713
1714 ret = func(mem, arg);
1715 if (ret) {
1716 kobject_put(&mem->dev.kobj);
1717 return ret;
1718 }
1719 }
1720
1721 if (mem)
1722 kobject_put(&mem->dev.kobj);
1723
1724 return 0;
1725}
1726
1727#ifdef CONFIG_MEMORY_HOTREMOVE
1728static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
1729{
1730 int ret = !is_memblock_offlined(mem);
1731
1732 if (unlikely(ret)) {
1733 phys_addr_t beginpa, endpa;
1734
1735 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
1736 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
1737 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
1738 &beginpa, &endpa);
1739 }
1740
1741 return ret;
1742}
1743
1744static int check_cpu_on_node(pg_data_t *pgdat)
1745{
1746 int cpu;
1747
1748 for_each_present_cpu(cpu) {
1749 if (cpu_to_node(cpu) == pgdat->node_id)
1750
1751
1752
1753
1754 return -EBUSY;
1755 }
1756
1757 return 0;
1758}
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769void try_offline_node(int nid)
1770{
1771 pg_data_t *pgdat = NODE_DATA(nid);
1772 unsigned long start_pfn = pgdat->node_start_pfn;
1773 unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
1774 unsigned long pfn;
1775
1776 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1777 unsigned long section_nr = pfn_to_section_nr(pfn);
1778
1779 if (!present_section_nr(section_nr))
1780 continue;
1781
1782 if (pfn_to_nid(pfn) != nid)
1783 continue;
1784
1785
1786
1787
1788
1789 return;
1790 }
1791
1792 if (check_cpu_on_node(pgdat))
1793 return;
1794
1795
1796
1797
1798
1799 node_set_offline(nid);
1800 unregister_one_node(nid);
1801}
1802EXPORT_SYMBOL(try_offline_node);
1803
1804static void __release_memory_resource(resource_size_t start,
1805 resource_size_t size)
1806{
1807 int ret;
1808
1809
1810
1811
1812
1813
1814
1815 ret = release_mem_region_adjustable(&iomem_resource, start, size);
1816 if (ret) {
1817 resource_size_t endres = start + size - 1;
1818
1819 pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
1820 &start, &endres, ret);
1821 }
1822}
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834void __ref __remove_memory(int nid, u64 start, u64 size)
1835{
1836 int ret;
1837
1838 BUG_ON(check_hotplug_memory_range(start, size));
1839
1840 mem_hotplug_begin();
1841
1842
1843
1844
1845
1846
1847 ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
1848 check_memblock_offlined_cb);
1849 if (ret)
1850 BUG();
1851
1852
1853 firmware_map_remove(start, start + size, "System RAM");
1854 memblock_free(start, size);
1855 memblock_remove(start, size);
1856
1857 arch_remove_memory(nid, start, size, NULL);
1858 __release_memory_resource(start, size);
1859
1860 try_offline_node(nid);
1861
1862 mem_hotplug_done();
1863}
1864
1865void remove_memory(int nid, u64 start, u64 size)
1866{
1867 lock_device_hotplug();
1868 __remove_memory(nid, start, size);
1869 unlock_device_hotplug();
1870}
1871EXPORT_SYMBOL_GPL(remove_memory);
1872#endif
1873