1
2
3
4
5
6
7#include <linux/stddef.h>
8#include <linux/mm.h>
9#include <linux/swap.h>
10#include <linux/interrupt.h>
11#include <linux/pagemap.h>
12#include <linux/bootmem.h>
13#include <linux/compiler.h>
14#include <linux/export.h>
15#include <linux/pagevec.h>
16#include <linux/writeback.h>
17#include <linux/slab.h>
18#include <linux/sysctl.h>
19#include <linux/cpu.h>
20#include <linux/memory.h>
21#include <linux/memory_hotplug.h>
22#include <linux/highmem.h>
23#include <linux/vmalloc.h>
24#include <linux/ioport.h>
25#include <linux/delay.h>
26#include <linux/migrate.h>
27#include <linux/page-isolation.h>
28#include <linux/pfn.h>
29#include <linux/suspend.h>
30#include <linux/mm_inline.h>
31#include <linux/firmware-map.h>
32#include <linux/stop_machine.h>
33#include <linux/hugetlb.h>
34
35#include <asm/tlbflush.h>
36
37#include "internal.h"
38
39
40
41
42
43
44
45
46static void generic_online_page(struct page *page);
47
48static online_page_callback_t online_page_callback = generic_online_page;
49
50DEFINE_MUTEX(mem_hotplug_mutex);
51
52void lock_memory_hotplug(void)
53{
54 mutex_lock(&mem_hotplug_mutex);
55}
56
57void unlock_memory_hotplug(void)
58{
59 mutex_unlock(&mem_hotplug_mutex);
60}
61
62
63
64static struct resource *register_memory_resource(u64 start, u64 size)
65{
66 struct resource *res;
67 res = kzalloc(sizeof(struct resource), GFP_KERNEL);
68 BUG_ON(!res);
69
70 res->name = "System RAM";
71 res->start = start;
72 res->end = start + size - 1;
73 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
74 if (request_resource(&iomem_resource, res) < 0) {
75 pr_debug("System RAM resource %pR cannot be added\n", res);
76 kfree(res);
77 res = NULL;
78 }
79 return res;
80}
81
82static void release_memory_resource(struct resource *res)
83{
84 if (!res)
85 return;
86 release_resource(res);
87 kfree(res);
88 return;
89}
90
91#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
92void get_page_bootmem(unsigned long info, struct page *page,
93 unsigned long type)
94{
95 page->lru.next = (struct list_head *) type;
96 SetPagePrivate(page);
97 set_page_private(page, info);
98 atomic_inc(&page->_count);
99}
100
101void put_page_bootmem(struct page *page)
102{
103 unsigned long type;
104
105 type = (unsigned long) page->lru.next;
106 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
107 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
108
109 if (atomic_dec_return(&page->_count) == 1) {
110 ClearPagePrivate(page);
111 set_page_private(page, 0);
112 INIT_LIST_HEAD(&page->lru);
113 free_reserved_page(page);
114 }
115}
116
117#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
118#ifndef CONFIG_SPARSEMEM_VMEMMAP
119static void register_page_bootmem_info_section(unsigned long start_pfn)
120{
121 unsigned long *usemap, mapsize, section_nr, i;
122 struct mem_section *ms;
123 struct page *page, *memmap;
124
125 section_nr = pfn_to_section_nr(start_pfn);
126 ms = __nr_to_section(section_nr);
127
128
129 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
130
131
132
133
134
135 page = virt_to_page(memmap);
136 mapsize = sizeof(struct page) * PAGES_PER_SECTION;
137 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
138
139
140 for (i = 0; i < mapsize; i++, page++)
141 get_page_bootmem(section_nr, page, SECTION_INFO);
142
143 usemap = __nr_to_section(section_nr)->pageblock_flags;
144 page = virt_to_page(usemap);
145
146 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
147
148 for (i = 0; i < mapsize; i++, page++)
149 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
150
151}
152#else
153static void register_page_bootmem_info_section(unsigned long start_pfn)
154{
155 unsigned long *usemap, mapsize, section_nr, i;
156 struct mem_section *ms;
157 struct page *page, *memmap;
158
159 if (!pfn_valid(start_pfn))
160 return;
161
162 section_nr = pfn_to_section_nr(start_pfn);
163 ms = __nr_to_section(section_nr);
164
165 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
166
167 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
168
169 usemap = __nr_to_section(section_nr)->pageblock_flags;
170 page = virt_to_page(usemap);
171
172 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
173
174 for (i = 0; i < mapsize; i++, page++)
175 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
176}
177#endif
178
179void register_page_bootmem_info_node(struct pglist_data *pgdat)
180{
181 unsigned long i, pfn, end_pfn, nr_pages;
182 int node = pgdat->node_id;
183 struct page *page;
184 struct zone *zone;
185
186 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
187 page = virt_to_page(pgdat);
188
189 for (i = 0; i < nr_pages; i++, page++)
190 get_page_bootmem(node, page, NODE_INFO);
191
192 zone = &pgdat->node_zones[0];
193 for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
194 if (zone_is_initialized(zone)) {
195 nr_pages = zone->wait_table_hash_nr_entries
196 * sizeof(wait_queue_head_t);
197 nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
198 page = virt_to_page(zone->wait_table);
199
200 for (i = 0; i < nr_pages; i++, page++)
201 get_page_bootmem(node, page, NODE_INFO);
202 }
203 }
204
205 pfn = pgdat->node_start_pfn;
206 end_pfn = pgdat_end_pfn(pgdat);
207
208
209 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
210
211
212
213
214
215
216 if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node))
217 register_page_bootmem_info_section(pfn);
218 }
219}
220#endif
221
222static void grow_zone_span(struct zone *zone, unsigned long start_pfn,
223 unsigned long end_pfn)
224{
225 unsigned long old_zone_end_pfn;
226
227 zone_span_writelock(zone);
228
229 old_zone_end_pfn = zone_end_pfn(zone);
230 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
231 zone->zone_start_pfn = start_pfn;
232
233 zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
234 zone->zone_start_pfn;
235
236 zone_span_writeunlock(zone);
237}
238
239static void resize_zone(struct zone *zone, unsigned long start_pfn,
240 unsigned long end_pfn)
241{
242 zone_span_writelock(zone);
243
244 if (end_pfn - start_pfn) {
245 zone->zone_start_pfn = start_pfn;
246 zone->spanned_pages = end_pfn - start_pfn;
247 } else {
248
249
250
251
252 zone->zone_start_pfn = 0;
253 zone->spanned_pages = 0;
254 }
255
256 zone_span_writeunlock(zone);
257}
258
259static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
260 unsigned long end_pfn)
261{
262 enum zone_type zid = zone_idx(zone);
263 int nid = zone->zone_pgdat->node_id;
264 unsigned long pfn;
265
266 for (pfn = start_pfn; pfn < end_pfn; pfn++)
267 set_page_links(pfn_to_page(pfn), zid, nid, pfn);
268}
269
270
271
272static int __ref ensure_zone_is_initialized(struct zone *zone,
273 unsigned long start_pfn, unsigned long num_pages)
274{
275 if (!zone_is_initialized(zone))
276 return init_currently_empty_zone(zone, start_pfn, num_pages,
277 MEMMAP_HOTPLUG);
278 return 0;
279}
280
281static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
282 unsigned long start_pfn, unsigned long end_pfn)
283{
284 int ret;
285 unsigned long flags;
286 unsigned long z1_start_pfn;
287
288 ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
289 if (ret)
290 return ret;
291
292 pgdat_resize_lock(z1->zone_pgdat, &flags);
293
294
295 if (end_pfn > zone_end_pfn(z2))
296 goto out_fail;
297
298 if (start_pfn > z2->zone_start_pfn)
299 goto out_fail;
300
301 if (end_pfn <= z2->zone_start_pfn)
302 goto out_fail;
303
304
305 if (!zone_is_empty(z1))
306 z1_start_pfn = z1->zone_start_pfn;
307 else
308 z1_start_pfn = start_pfn;
309
310 resize_zone(z1, z1_start_pfn, end_pfn);
311 resize_zone(z2, end_pfn, zone_end_pfn(z2));
312
313 pgdat_resize_unlock(z1->zone_pgdat, &flags);
314
315 fix_zone_id(z1, start_pfn, end_pfn);
316
317 return 0;
318out_fail:
319 pgdat_resize_unlock(z1->zone_pgdat, &flags);
320 return -1;
321}
322
323static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
324 unsigned long start_pfn, unsigned long end_pfn)
325{
326 int ret;
327 unsigned long flags;
328 unsigned long z2_end_pfn;
329
330 ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
331 if (ret)
332 return ret;
333
334 pgdat_resize_lock(z1->zone_pgdat, &flags);
335
336
337 if (z1->zone_start_pfn > start_pfn)
338 goto out_fail;
339
340 if (zone_end_pfn(z1) > end_pfn)
341 goto out_fail;
342
343 if (start_pfn >= zone_end_pfn(z1))
344 goto out_fail;
345
346
347 if (!zone_is_empty(z2))
348 z2_end_pfn = zone_end_pfn(z2);
349 else
350 z2_end_pfn = end_pfn;
351
352 resize_zone(z1, z1->zone_start_pfn, start_pfn);
353 resize_zone(z2, start_pfn, z2_end_pfn);
354
355 pgdat_resize_unlock(z1->zone_pgdat, &flags);
356
357 fix_zone_id(z2, start_pfn, end_pfn);
358
359 return 0;
360out_fail:
361 pgdat_resize_unlock(z1->zone_pgdat, &flags);
362 return -1;
363}
364
365static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
366 unsigned long end_pfn)
367{
368 unsigned long old_pgdat_end_pfn =
369 pgdat->node_start_pfn + pgdat->node_spanned_pages;
370
371 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
372 pgdat->node_start_pfn = start_pfn;
373
374 pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
375 pgdat->node_start_pfn;
376}
377
378static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
379{
380 struct pglist_data *pgdat = zone->zone_pgdat;
381 int nr_pages = PAGES_PER_SECTION;
382 int nid = pgdat->node_id;
383 int zone_type;
384 unsigned long flags;
385 int ret;
386
387 zone_type = zone - pgdat->node_zones;
388 ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
389 if (ret)
390 return ret;
391
392 pgdat_resize_lock(zone->zone_pgdat, &flags);
393 grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
394 grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
395 phys_start_pfn + nr_pages);
396 pgdat_resize_unlock(zone->zone_pgdat, &flags);
397 memmap_init_zone(nr_pages, nid, zone_type,
398 phys_start_pfn, MEMMAP_HOTPLUG);
399 return 0;
400}
401
402static int __meminit __add_section(int nid, struct zone *zone,
403 unsigned long phys_start_pfn)
404{
405 int nr_pages = PAGES_PER_SECTION;
406 int ret;
407
408 if (pfn_valid(phys_start_pfn))
409 return -EEXIST;
410
411 ret = sparse_add_one_section(zone, phys_start_pfn, nr_pages);
412
413 if (ret < 0)
414 return ret;
415
416 ret = __add_zone(zone, phys_start_pfn);
417
418 if (ret < 0)
419 return ret;
420
421 return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
422}
423
424
425
426
427
428
429
430int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
431 unsigned long nr_pages)
432{
433 unsigned long i;
434 int err = 0;
435 int start_sec, end_sec;
436
437 start_sec = pfn_to_section_nr(phys_start_pfn);
438 end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);
439
440 for (i = start_sec; i <= end_sec; i++) {
441 err = __add_section(nid, zone, i << PFN_SECTION_SHIFT);
442
443
444
445
446
447
448 if (err && (err != -EEXIST))
449 break;
450 err = 0;
451 }
452
453 return err;
454}
455EXPORT_SYMBOL_GPL(__add_pages);
456
457#ifdef CONFIG_MEMORY_HOTREMOVE
458
459static int find_smallest_section_pfn(int nid, struct zone *zone,
460 unsigned long start_pfn,
461 unsigned long end_pfn)
462{
463 struct mem_section *ms;
464
465 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
466 ms = __pfn_to_section(start_pfn);
467
468 if (unlikely(!valid_section(ms)))
469 continue;
470
471 if (unlikely(pfn_to_nid(start_pfn) != nid))
472 continue;
473
474 if (zone && zone != page_zone(pfn_to_page(start_pfn)))
475 continue;
476
477 return start_pfn;
478 }
479
480 return 0;
481}
482
483
484static int find_biggest_section_pfn(int nid, struct zone *zone,
485 unsigned long start_pfn,
486 unsigned long end_pfn)
487{
488 struct mem_section *ms;
489 unsigned long pfn;
490
491
492 pfn = end_pfn - 1;
493 for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
494 ms = __pfn_to_section(pfn);
495
496 if (unlikely(!valid_section(ms)))
497 continue;
498
499 if (unlikely(pfn_to_nid(pfn) != nid))
500 continue;
501
502 if (zone && zone != page_zone(pfn_to_page(pfn)))
503 continue;
504
505 return pfn;
506 }
507
508 return 0;
509}
510
511static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
512 unsigned long end_pfn)
513{
514 unsigned long zone_start_pfn = zone->zone_start_pfn;
515 unsigned long z = zone_end_pfn(zone);
516 unsigned long zone_end_pfn = z;
517 unsigned long pfn;
518 struct mem_section *ms;
519 int nid = zone_to_nid(zone);
520
521 zone_span_writelock(zone);
522 if (zone_start_pfn == start_pfn) {
523
524
525
526
527
528
529 pfn = find_smallest_section_pfn(nid, zone, end_pfn,
530 zone_end_pfn);
531 if (pfn) {
532 zone->zone_start_pfn = pfn;
533 zone->spanned_pages = zone_end_pfn - pfn;
534 }
535 } else if (zone_end_pfn == end_pfn) {
536
537
538
539
540
541
542 pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
543 start_pfn);
544 if (pfn)
545 zone->spanned_pages = pfn - zone_start_pfn + 1;
546 }
547
548
549
550
551
552
553
554 pfn = zone_start_pfn;
555 for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
556 ms = __pfn_to_section(pfn);
557
558 if (unlikely(!valid_section(ms)))
559 continue;
560
561 if (page_zone(pfn_to_page(pfn)) != zone)
562 continue;
563
564
565 if (start_pfn == pfn)
566 continue;
567
568
569 zone_span_writeunlock(zone);
570 return;
571 }
572
573
574 zone->zone_start_pfn = 0;
575 zone->spanned_pages = 0;
576 zone_span_writeunlock(zone);
577}
578
579static void shrink_pgdat_span(struct pglist_data *pgdat,
580 unsigned long start_pfn, unsigned long end_pfn)
581{
582 unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
583 unsigned long pgdat_end_pfn =
584 pgdat->node_start_pfn + pgdat->node_spanned_pages;
585 unsigned long pfn;
586 struct mem_section *ms;
587 int nid = pgdat->node_id;
588
589 if (pgdat_start_pfn == start_pfn) {
590
591
592
593
594
595
596 pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
597 pgdat_end_pfn);
598 if (pfn) {
599 pgdat->node_start_pfn = pfn;
600 pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
601 }
602 } else if (pgdat_end_pfn == end_pfn) {
603
604
605
606
607
608
609 pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
610 start_pfn);
611 if (pfn)
612 pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
613 }
614
615
616
617
618
619
620
621
622 pfn = pgdat_start_pfn;
623 for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
624 ms = __pfn_to_section(pfn);
625
626 if (unlikely(!valid_section(ms)))
627 continue;
628
629 if (pfn_to_nid(pfn) != nid)
630 continue;
631
632
633 if (start_pfn == pfn)
634 continue;
635
636
637 return;
638 }
639
640
641 pgdat->node_start_pfn = 0;
642 pgdat->node_spanned_pages = 0;
643}
644
645static void __remove_zone(struct zone *zone, unsigned long start_pfn)
646{
647 struct pglist_data *pgdat = zone->zone_pgdat;
648 int nr_pages = PAGES_PER_SECTION;
649 int zone_type;
650 unsigned long flags;
651
652 zone_type = zone - pgdat->node_zones;
653
654 pgdat_resize_lock(zone->zone_pgdat, &flags);
655 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
656 shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
657 pgdat_resize_unlock(zone->zone_pgdat, &flags);
658}
659
660static int __remove_section(struct zone *zone, struct mem_section *ms)
661{
662 unsigned long start_pfn;
663 int scn_nr;
664 int ret = -EINVAL;
665
666 if (!valid_section(ms))
667 return ret;
668
669 ret = unregister_memory_section(ms);
670 if (ret)
671 return ret;
672
673 scn_nr = __section_nr(ms);
674 start_pfn = section_nr_to_pfn(scn_nr);
675 __remove_zone(zone, start_pfn);
676
677 sparse_remove_one_section(zone, ms);
678 return 0;
679}
680
681
682
683
684
685
686
687
688
689
690
691
692int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
693 unsigned long nr_pages)
694{
695 unsigned long i;
696 int sections_to_remove;
697 resource_size_t start, size;
698 int ret = 0;
699
700
701
702
703 BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
704 BUG_ON(nr_pages % PAGES_PER_SECTION);
705
706 start = phys_start_pfn << PAGE_SHIFT;
707 size = nr_pages * PAGE_SIZE;
708 ret = release_mem_region_adjustable(&iomem_resource, start, size);
709 if (ret) {
710 resource_size_t endres = start + size - 1;
711
712 pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
713 &start, &endres, ret);
714 }
715
716 sections_to_remove = nr_pages / PAGES_PER_SECTION;
717 for (i = 0; i < sections_to_remove; i++) {
718 unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
719 ret = __remove_section(zone, __pfn_to_section(pfn));
720 if (ret)
721 break;
722 }
723 return ret;
724}
725EXPORT_SYMBOL_GPL(__remove_pages);
726#endif
727
728int set_online_page_callback(online_page_callback_t callback)
729{
730 int rc = -EINVAL;
731
732 lock_memory_hotplug();
733
734 if (online_page_callback == generic_online_page) {
735 online_page_callback = callback;
736 rc = 0;
737 }
738
739 unlock_memory_hotplug();
740
741 return rc;
742}
743EXPORT_SYMBOL_GPL(set_online_page_callback);
744
745int restore_online_page_callback(online_page_callback_t callback)
746{
747 int rc = -EINVAL;
748
749 lock_memory_hotplug();
750
751 if (online_page_callback == callback) {
752 online_page_callback = generic_online_page;
753 rc = 0;
754 }
755
756 unlock_memory_hotplug();
757
758 return rc;
759}
760EXPORT_SYMBOL_GPL(restore_online_page_callback);
761
762void __online_page_set_limits(struct page *page)
763{
764}
765EXPORT_SYMBOL_GPL(__online_page_set_limits);
766
767void __online_page_increment_counters(struct page *page)
768{
769 adjust_managed_page_count(page, 1);
770}
771EXPORT_SYMBOL_GPL(__online_page_increment_counters);
772
773void __online_page_free(struct page *page)
774{
775 __free_reserved_page(page);
776}
777EXPORT_SYMBOL_GPL(__online_page_free);
778
779static void generic_online_page(struct page *page)
780{
781 __online_page_set_limits(page);
782 __online_page_increment_counters(page);
783 __online_page_free(page);
784}
785
786static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
787 void *arg)
788{
789 unsigned long i;
790 unsigned long onlined_pages = *(unsigned long *)arg;
791 struct page *page;
792 if (PageReserved(pfn_to_page(start_pfn)))
793 for (i = 0; i < nr_pages; i++) {
794 page = pfn_to_page(start_pfn + i);
795 (*online_page_callback)(page);
796 onlined_pages++;
797 }
798 *(unsigned long *)arg = onlined_pages;
799 return 0;
800}
801
802#ifdef CONFIG_MOVABLE_NODE
803
804
805
806
807static bool can_online_high_movable(struct zone *zone)
808{
809 return true;
810}
811#else
812
813static bool can_online_high_movable(struct zone *zone)
814{
815 return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
816}
817#endif
818
819
820static void node_states_check_changes_online(unsigned long nr_pages,
821 struct zone *zone, struct memory_notify *arg)
822{
823 int nid = zone_to_nid(zone);
824 enum zone_type zone_last = ZONE_NORMAL;
825
826
827
828
829
830
831
832
833
834
835 if (N_MEMORY == N_NORMAL_MEMORY)
836 zone_last = ZONE_MOVABLE;
837
838
839
840
841
842
843
844 if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
845 arg->status_change_nid_normal = nid;
846 else
847 arg->status_change_nid_normal = -1;
848
849#ifdef CONFIG_HIGHMEM
850
851
852
853
854
855
856
857
858
859 zone_last = ZONE_HIGHMEM;
860 if (N_MEMORY == N_HIGH_MEMORY)
861 zone_last = ZONE_MOVABLE;
862
863 if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
864 arg->status_change_nid_high = nid;
865 else
866 arg->status_change_nid_high = -1;
867#else
868 arg->status_change_nid_high = arg->status_change_nid_normal;
869#endif
870
871
872
873
874
875
876 if (!node_state(nid, N_MEMORY))
877 arg->status_change_nid = nid;
878 else
879 arg->status_change_nid = -1;
880}
881
882static void node_states_set_node(int node, struct memory_notify *arg)
883{
884 if (arg->status_change_nid_normal >= 0)
885 node_set_state(node, N_NORMAL_MEMORY);
886
887 if (arg->status_change_nid_high >= 0)
888 node_set_state(node, N_HIGH_MEMORY);
889
890 node_set_state(node, N_MEMORY);
891}
892
893
894int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
895{
896 unsigned long flags;
897 unsigned long onlined_pages = 0;
898 struct zone *zone;
899 int need_zonelists_rebuild = 0;
900 int nid;
901 int ret;
902 struct memory_notify arg;
903
904 lock_memory_hotplug();
905
906
907
908
909
910 zone = page_zone(pfn_to_page(pfn));
911
912 if ((zone_idx(zone) > ZONE_NORMAL || online_type == ONLINE_MOVABLE) &&
913 !can_online_high_movable(zone)) {
914 unlock_memory_hotplug();
915 return -EINVAL;
916 }
917
918 if (online_type == ONLINE_KERNEL && zone_idx(zone) == ZONE_MOVABLE) {
919 if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) {
920 unlock_memory_hotplug();
921 return -EINVAL;
922 }
923 }
924 if (online_type == ONLINE_MOVABLE && zone_idx(zone) == ZONE_MOVABLE - 1) {
925 if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) {
926 unlock_memory_hotplug();
927 return -EINVAL;
928 }
929 }
930
931
932 zone = page_zone(pfn_to_page(pfn));
933
934 arg.start_pfn = pfn;
935 arg.nr_pages = nr_pages;
936 node_states_check_changes_online(nr_pages, zone, &arg);
937
938 nid = page_to_nid(pfn_to_page(pfn));
939
940 ret = memory_notify(MEM_GOING_ONLINE, &arg);
941 ret = notifier_to_errno(ret);
942 if (ret) {
943 memory_notify(MEM_CANCEL_ONLINE, &arg);
944 unlock_memory_hotplug();
945 return ret;
946 }
947
948
949
950
951
952 mutex_lock(&zonelists_mutex);
953 if (!populated_zone(zone)) {
954 need_zonelists_rebuild = 1;
955 build_all_zonelists(NULL, zone);
956 }
957
958 ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
959 online_pages_range);
960 if (ret) {
961 if (need_zonelists_rebuild)
962 zone_pcp_reset(zone);
963 mutex_unlock(&zonelists_mutex);
964 printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",
965 (unsigned long long) pfn << PAGE_SHIFT,
966 (((unsigned long long) pfn + nr_pages)
967 << PAGE_SHIFT) - 1);
968 memory_notify(MEM_CANCEL_ONLINE, &arg);
969 unlock_memory_hotplug();
970 return ret;
971 }
972
973 zone->present_pages += onlined_pages;
974
975 pgdat_resize_lock(zone->zone_pgdat, &flags);
976 zone->zone_pgdat->node_present_pages += onlined_pages;
977 pgdat_resize_unlock(zone->zone_pgdat, &flags);
978
979 if (onlined_pages) {
980 node_states_set_node(zone_to_nid(zone), &arg);
981 if (need_zonelists_rebuild)
982 build_all_zonelists(NULL, NULL);
983 else
984 zone_pcp_update(zone);
985 }
986
987 mutex_unlock(&zonelists_mutex);
988
989 init_per_zone_wmark_min();
990
991 if (onlined_pages)
992 kswapd_run(zone_to_nid(zone));
993
994 vm_total_pages = nr_free_pagecache_pages();
995
996 writeback_set_ratelimit();
997
998 if (onlined_pages)
999 memory_notify(MEM_ONLINE, &arg);
1000 unlock_memory_hotplug();
1001
1002 return 0;
1003}
1004#endif
1005
1006
1007static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
1008{
1009 struct pglist_data *pgdat;
1010 unsigned long zones_size[MAX_NR_ZONES] = {0};
1011 unsigned long zholes_size[MAX_NR_ZONES] = {0};
1012 unsigned long start_pfn = start >> PAGE_SHIFT;
1013
1014 pgdat = NODE_DATA(nid);
1015 if (!pgdat) {
1016 pgdat = arch_alloc_nodedata(nid);
1017 if (!pgdat)
1018 return NULL;
1019
1020 arch_refresh_nodedata(nid, pgdat);
1021 }
1022
1023
1024
1025
1026 free_area_init_node(nid, zones_size, start_pfn, zholes_size);
1027
1028
1029
1030
1031
1032 mutex_lock(&zonelists_mutex);
1033 build_all_zonelists(pgdat, NULL);
1034 mutex_unlock(&zonelists_mutex);
1035
1036 return pgdat;
1037}
1038
1039static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
1040{
1041 arch_refresh_nodedata(nid, NULL);
1042 arch_free_nodedata(pgdat);
1043 return;
1044}
1045
1046
1047
1048
1049
1050int mem_online_node(int nid)
1051{
1052 pg_data_t *pgdat;
1053 int ret;
1054
1055 lock_memory_hotplug();
1056 pgdat = hotadd_new_pgdat(nid, 0);
1057 if (!pgdat) {
1058 ret = -ENOMEM;
1059 goto out;
1060 }
1061 node_set_online(nid);
1062 ret = register_one_node(nid);
1063 BUG_ON(ret);
1064
1065out:
1066 unlock_memory_hotplug();
1067 return ret;
1068}
1069
1070static int check_hotplug_memory_range(u64 start, u64 size)
1071{
1072 u64 start_pfn = start >> PAGE_SHIFT;
1073 u64 nr_pages = size >> PAGE_SHIFT;
1074
1075
1076 if ((start_pfn & ~PAGE_SECTION_MASK) ||
1077 (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
1078 pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
1079 (unsigned long long)start,
1080 (unsigned long long)size);
1081 return -EINVAL;
1082 }
1083
1084 return 0;
1085}
1086
1087
1088int __ref add_memory(int nid, u64 start, u64 size)
1089{
1090 pg_data_t *pgdat = NULL;
1091 bool new_pgdat;
1092 bool new_node;
1093 struct resource *res;
1094 int ret;
1095
1096 ret = check_hotplug_memory_range(start, size);
1097 if (ret)
1098 return ret;
1099
1100 lock_memory_hotplug();
1101
1102 res = register_memory_resource(start, size);
1103 ret = -EEXIST;
1104 if (!res)
1105 goto out;
1106
1107 {
1108 void *p = NODE_DATA(nid);
1109 new_pgdat = !p;
1110 }
1111 new_node = !node_online(nid);
1112 if (new_node) {
1113 pgdat = hotadd_new_pgdat(nid, start);
1114 ret = -ENOMEM;
1115 if (!pgdat)
1116 goto error;
1117 }
1118
1119
1120 ret = arch_add_memory(nid, start, size);
1121
1122 if (ret < 0)
1123 goto error;
1124
1125
1126 node_set_online(nid);
1127
1128 if (new_node) {
1129 ret = register_one_node(nid);
1130
1131
1132
1133
1134
1135 BUG_ON(ret);
1136 }
1137
1138
1139 firmware_map_add_hotplug(start, start + size, "System RAM");
1140
1141 goto out;
1142
1143error:
1144
1145 if (new_pgdat)
1146 rollback_node_hotadd(nid, pgdat);
1147 release_memory_resource(res);
1148
1149out:
1150 unlock_memory_hotplug();
1151 return ret;
1152}
1153EXPORT_SYMBOL_GPL(add_memory);
1154
1155#ifdef CONFIG_MEMORY_HOTREMOVE
1156
1157
1158
1159
1160
1161
1162
1163static inline int pageblock_free(struct page *page)
1164{
1165 return PageBuddy(page) && page_order(page) >= pageblock_order;
1166}
1167
1168
1169static struct page *next_active_pageblock(struct page *page)
1170{
1171
1172 BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));
1173
1174
1175 if (pageblock_free(page)) {
1176 int order;
1177
1178 order = page_order(page);
1179 if ((order < MAX_ORDER) && (order >= pageblock_order))
1180 return page + (1 << order);
1181 }
1182
1183 return page + pageblock_nr_pages;
1184}
1185
1186
1187int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1188{
1189 struct page *page = pfn_to_page(start_pfn);
1190 struct page *end_page = page + nr_pages;
1191
1192
1193 for (; page < end_page; page = next_active_pageblock(page)) {
1194 if (!is_pageblock_removable_nolock(page))
1195 return 0;
1196 cond_resched();
1197 }
1198
1199
1200 return 1;
1201}
1202
1203
1204
1205
1206static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
1207{
1208 unsigned long pfn;
1209 struct zone *zone = NULL;
1210 struct page *page;
1211 int i;
1212 for (pfn = start_pfn;
1213 pfn < end_pfn;
1214 pfn += MAX_ORDER_NR_PAGES) {
1215 i = 0;
1216
1217 while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i))
1218 i++;
1219 if (i == MAX_ORDER_NR_PAGES)
1220 continue;
1221 page = pfn_to_page(pfn + i);
1222 if (zone && page_zone(page) != zone)
1223 return 0;
1224 zone = page_zone(page);
1225 }
1226 return 1;
1227}
1228
1229
1230
1231
1232
1233
1234
1235static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
1236{
1237 unsigned long pfn;
1238 struct page *page;
1239 for (pfn = start; pfn < end; pfn++) {
1240 if (pfn_valid(pfn)) {
1241 page = pfn_to_page(pfn);
1242 if (PageLRU(page))
1243 return pfn;
1244 if (PageHuge(page)) {
1245 if (is_hugepage_active(page))
1246 return pfn;
1247 else
1248 pfn = round_up(pfn + 1,
1249 1 << compound_order(page)) - 1;
1250 }
1251 }
1252 }
1253 return 0;
1254}
1255
1256#define NR_OFFLINE_AT_ONCE_PAGES (256)
1257static int
1258do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
1259{
1260 unsigned long pfn;
1261 struct page *page;
1262 int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
1263 int not_managed = 0;
1264 int ret = 0;
1265 LIST_HEAD(source);
1266
1267 for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
1268 if (!pfn_valid(pfn))
1269 continue;
1270 page = pfn_to_page(pfn);
1271
1272 if (PageHuge(page)) {
1273 struct page *head = compound_head(page);
1274 pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
1275 if (compound_order(head) > PFN_SECTION_SHIFT) {
1276 ret = -EBUSY;
1277 break;
1278 }
1279 if (isolate_huge_page(page, &source))
1280 move_pages -= 1 << compound_order(head);
1281 continue;
1282 }
1283
1284 if (!get_page_unless_zero(page))
1285 continue;
1286
1287
1288
1289
1290 ret = isolate_lru_page(page);
1291 if (!ret) {
1292 put_page(page);
1293 list_add_tail(&page->lru, &source);
1294 move_pages--;
1295 inc_zone_page_state(page, NR_ISOLATED_ANON +
1296 page_is_file_cache(page));
1297
1298 } else {
1299#ifdef CONFIG_DEBUG_VM
1300 printk(KERN_ALERT "removing pfn %lx from LRU failed\n",
1301 pfn);
1302 dump_page(page);
1303#endif
1304 put_page(page);
1305
1306
1307 if (page_count(page)) {
1308 not_managed++;
1309 ret = -EBUSY;
1310 break;
1311 }
1312 }
1313 }
1314 if (!list_empty(&source)) {
1315 if (not_managed) {
1316 putback_movable_pages(&source);
1317 goto out;
1318 }
1319
1320
1321
1322
1323
1324 ret = migrate_pages(&source, alloc_migrate_target, 0,
1325 MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1326 if (ret)
1327 putback_movable_pages(&source);
1328 }
1329out:
1330 return ret;
1331}
1332
1333
1334
1335
1336static int
1337offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
1338 void *data)
1339{
1340 __offline_isolated_pages(start, start + nr_pages);
1341 return 0;
1342}
1343
1344static void
1345offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
1346{
1347 walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
1348 offline_isolated_pages_cb);
1349}
1350
1351
1352
1353
1354static int
1355check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
1356 void *data)
1357{
1358 int ret;
1359 long offlined = *(long *)data;
1360 ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
1361 offlined = nr_pages;
1362 if (!ret)
1363 *(long *)data += offlined;
1364 return ret;
1365}
1366
1367static long
1368check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
1369{
1370 long offlined = 0;
1371 int ret;
1372
1373 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
1374 check_pages_isolated_cb);
1375 if (ret < 0)
1376 offlined = (long)ret;
1377 return offlined;
1378}
1379
1380#ifdef CONFIG_MOVABLE_NODE
1381
1382
1383
1384
1385static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
1386{
1387 return true;
1388}
1389#else
1390
1391static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
1392{
1393 struct pglist_data *pgdat = zone->zone_pgdat;
1394 unsigned long present_pages = 0;
1395 enum zone_type zt;
1396
1397 for (zt = 0; zt <= ZONE_NORMAL; zt++)
1398 present_pages += pgdat->node_zones[zt].present_pages;
1399
1400 if (present_pages > nr_pages)
1401 return true;
1402
1403 present_pages = 0;
1404 for (; zt <= ZONE_MOVABLE; zt++)
1405 present_pages += pgdat->node_zones[zt].present_pages;
1406
1407
1408
1409
1410
1411 return present_pages == 0;
1412}
1413#endif
1414
1415
1416static void node_states_check_changes_offline(unsigned long nr_pages,
1417 struct zone *zone, struct memory_notify *arg)
1418{
1419 struct pglist_data *pgdat = zone->zone_pgdat;
1420 unsigned long present_pages = 0;
1421 enum zone_type zt, zone_last = ZONE_NORMAL;
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432 if (N_MEMORY == N_NORMAL_MEMORY)
1433 zone_last = ZONE_MOVABLE;
1434
1435
1436
1437
1438
1439
1440
1441
1442 for (zt = 0; zt <= zone_last; zt++)
1443 present_pages += pgdat->node_zones[zt].present_pages;
1444 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
1445 arg->status_change_nid_normal = zone_to_nid(zone);
1446 else
1447 arg->status_change_nid_normal = -1;
1448
1449#ifdef CONFIG_HIGHMEM
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459 zone_last = ZONE_HIGHMEM;
1460 if (N_MEMORY == N_HIGH_MEMORY)
1461 zone_last = ZONE_MOVABLE;
1462
1463 for (; zt <= zone_last; zt++)
1464 present_pages += pgdat->node_zones[zt].present_pages;
1465 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
1466 arg->status_change_nid_high = zone_to_nid(zone);
1467 else
1468 arg->status_change_nid_high = -1;
1469#else
1470 arg->status_change_nid_high = arg->status_change_nid_normal;
1471#endif
1472
1473
1474
1475
1476 zone_last = ZONE_MOVABLE;
1477
1478
1479
1480
1481
1482
1483
1484 for (; zt <= zone_last; zt++)
1485 present_pages += pgdat->node_zones[zt].present_pages;
1486 if (nr_pages >= present_pages)
1487 arg->status_change_nid = zone_to_nid(zone);
1488 else
1489 arg->status_change_nid = -1;
1490}
1491
1492static void node_states_clear_node(int node, struct memory_notify *arg)
1493{
1494 if (arg->status_change_nid_normal >= 0)
1495 node_clear_state(node, N_NORMAL_MEMORY);
1496
1497 if ((N_MEMORY != N_NORMAL_MEMORY) &&
1498 (arg->status_change_nid_high >= 0))
1499 node_clear_state(node, N_HIGH_MEMORY);
1500
1501 if ((N_MEMORY != N_HIGH_MEMORY) &&
1502 (arg->status_change_nid >= 0))
1503 node_clear_state(node, N_MEMORY);
1504}
1505
1506static int __ref __offline_pages(unsigned long start_pfn,
1507 unsigned long end_pfn, unsigned long timeout)
1508{
1509 unsigned long pfn, nr_pages, expire;
1510 long offlined_pages;
1511 int ret, drain, retry_max, node;
1512 unsigned long flags;
1513 struct zone *zone;
1514 struct memory_notify arg;
1515
1516
1517 if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
1518 return -EINVAL;
1519 if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
1520 return -EINVAL;
1521
1522
1523 if (!test_pages_in_a_zone(start_pfn, end_pfn))
1524 return -EINVAL;
1525
1526 lock_memory_hotplug();
1527
1528 zone = page_zone(pfn_to_page(start_pfn));
1529 node = zone_to_nid(zone);
1530 nr_pages = end_pfn - start_pfn;
1531
1532 ret = -EINVAL;
1533 if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
1534 goto out;
1535
1536
1537 ret = start_isolate_page_range(start_pfn, end_pfn,
1538 MIGRATE_MOVABLE, true);
1539 if (ret)
1540 goto out;
1541
1542 arg.start_pfn = start_pfn;
1543 arg.nr_pages = nr_pages;
1544 node_states_check_changes_offline(nr_pages, zone, &arg);
1545
1546 ret = memory_notify(MEM_GOING_OFFLINE, &arg);
1547 ret = notifier_to_errno(ret);
1548 if (ret)
1549 goto failed_removal;
1550
1551 pfn = start_pfn;
1552 expire = jiffies + timeout;
1553 drain = 0;
1554 retry_max = 5;
1555repeat:
1556
1557 ret = -EAGAIN;
1558 if (time_after(jiffies, expire))
1559 goto failed_removal;
1560 ret = -EINTR;
1561 if (signal_pending(current))
1562 goto failed_removal;
1563 ret = 0;
1564 if (drain) {
1565 lru_add_drain_all();
1566 cond_resched();
1567 drain_all_pages();
1568 }
1569
1570 pfn = scan_movable_pages(start_pfn, end_pfn);
1571 if (pfn) {
1572 ret = do_migrate_range(pfn, end_pfn);
1573 if (!ret) {
1574 drain = 1;
1575 goto repeat;
1576 } else {
1577 if (ret < 0)
1578 if (--retry_max == 0)
1579 goto failed_removal;
1580 yield();
1581 drain = 1;
1582 goto repeat;
1583 }
1584 }
1585
1586 lru_add_drain_all();
1587 yield();
1588
1589 drain_all_pages();
1590
1591
1592
1593
1594 dissolve_free_huge_pages(start_pfn, end_pfn);
1595
1596 offlined_pages = check_pages_isolated(start_pfn, end_pfn);
1597 if (offlined_pages < 0) {
1598 ret = -EBUSY;
1599 goto failed_removal;
1600 }
1601 printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);
1602
1603
1604 offline_isolated_pages(start_pfn, end_pfn);
1605
1606 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1607
1608 adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
1609 zone->present_pages -= offlined_pages;
1610
1611 pgdat_resize_lock(zone->zone_pgdat, &flags);
1612 zone->zone_pgdat->node_present_pages -= offlined_pages;
1613 pgdat_resize_unlock(zone->zone_pgdat, &flags);
1614
1615 init_per_zone_wmark_min();
1616
1617 if (!populated_zone(zone)) {
1618 zone_pcp_reset(zone);
1619 mutex_lock(&zonelists_mutex);
1620 build_all_zonelists(NULL, NULL);
1621 mutex_unlock(&zonelists_mutex);
1622 } else
1623 zone_pcp_update(zone);
1624
1625 node_states_clear_node(node, &arg);
1626 if (arg.status_change_nid >= 0)
1627 kswapd_stop(node);
1628
1629 vm_total_pages = nr_free_pagecache_pages();
1630 writeback_set_ratelimit();
1631
1632 memory_notify(MEM_OFFLINE, &arg);
1633 unlock_memory_hotplug();
1634 return 0;
1635
1636failed_removal:
1637 printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",
1638 (unsigned long long) start_pfn << PAGE_SHIFT,
1639 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1640 memory_notify(MEM_CANCEL_OFFLINE, &arg);
1641
1642 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1643
1644out:
1645 unlock_memory_hotplug();
1646 return ret;
1647}
1648
1649int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
1650{
1651 return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
1652}
1653#endif
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
1668 void *arg, int (*func)(struct memory_block *, void *))
1669{
1670 struct memory_block *mem = NULL;
1671 struct mem_section *section;
1672 unsigned long pfn, section_nr;
1673 int ret;
1674
1675 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1676 section_nr = pfn_to_section_nr(pfn);
1677 if (!present_section_nr(section_nr))
1678 continue;
1679
1680 section = __nr_to_section(section_nr);
1681
1682 if (mem)
1683 if ((section_nr >= mem->start_section_nr) &&
1684 (section_nr <= mem->end_section_nr))
1685 continue;
1686
1687 mem = find_memory_block_hinted(section, mem);
1688 if (!mem)
1689 continue;
1690
1691 ret = func(mem, arg);
1692 if (ret) {
1693 kobject_put(&mem->dev.kobj);
1694 return ret;
1695 }
1696 }
1697
1698 if (mem)
1699 kobject_put(&mem->dev.kobj);
1700
1701 return 0;
1702}
1703
1704#ifdef CONFIG_MEMORY_HOTREMOVE
1705static int is_memblock_offlined_cb(struct memory_block *mem, void *arg)
1706{
1707 int ret = !is_memblock_offlined(mem);
1708
1709 if (unlikely(ret)) {
1710 phys_addr_t beginpa, endpa;
1711
1712 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
1713 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
1714 pr_warn("removing memory fails, because memory "
1715 "[%pa-%pa] is onlined\n",
1716 &beginpa, &endpa);
1717 }
1718
1719 return ret;
1720}
1721
1722static int check_cpu_on_node(pg_data_t *pgdat)
1723{
1724 int cpu;
1725
1726 for_each_present_cpu(cpu) {
1727 if (cpu_to_node(cpu) == pgdat->node_id)
1728
1729
1730
1731
1732 return -EBUSY;
1733 }
1734
1735 return 0;
1736}
1737
1738static void unmap_cpu_on_node(pg_data_t *pgdat)
1739{
1740#ifdef CONFIG_ACPI_NUMA
1741 int cpu;
1742
1743 for_each_possible_cpu(cpu)
1744 if (cpu_to_node(cpu) == pgdat->node_id)
1745 numa_clear_node(cpu);
1746#endif
1747}
1748
1749static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
1750{
1751 int ret;
1752
1753 ret = check_cpu_on_node(pgdat);
1754 if (ret)
1755 return ret;
1756
1757
1758
1759
1760
1761
1762 unmap_cpu_on_node(pgdat);
1763 return 0;
1764}
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774void try_offline_node(int nid)
1775{
1776 pg_data_t *pgdat = NODE_DATA(nid);
1777 unsigned long start_pfn = pgdat->node_start_pfn;
1778 unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
1779 unsigned long pfn;
1780 struct page *pgdat_page = virt_to_page(pgdat);
1781 int i;
1782
1783 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1784 unsigned long section_nr = pfn_to_section_nr(pfn);
1785
1786 if (!present_section_nr(section_nr))
1787 continue;
1788
1789 if (pfn_to_nid(pfn) != nid)
1790 continue;
1791
1792
1793
1794
1795
1796 return;
1797 }
1798
1799 if (check_and_unmap_cpu_on_node(pgdat))
1800 return;
1801
1802
1803
1804
1805
1806 node_set_offline(nid);
1807 unregister_one_node(nid);
1808
1809 if (!PageSlab(pgdat_page) && !PageCompound(pgdat_page))
1810
1811 return;
1812
1813
1814 for (i = 0; i < MAX_NR_ZONES; i++) {
1815 struct zone *zone = pgdat->node_zones + i;
1816
1817
1818
1819
1820
1821 if (is_vmalloc_addr(zone->wait_table))
1822 vfree(zone->wait_table);
1823 }
1824
1825
1826
1827
1828
1829
1830
1831
1832 memset(pgdat, 0, sizeof(*pgdat));
1833}
1834EXPORT_SYMBOL(try_offline_node);
1835
1836
1837
1838
1839
1840
1841
1842
1843void __ref remove_memory(int nid, u64 start, u64 size)
1844{
1845 int ret;
1846
1847 BUG_ON(check_hotplug_memory_range(start, size));
1848
1849 lock_memory_hotplug();
1850
1851
1852
1853
1854
1855
1856 ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
1857 is_memblock_offlined_cb);
1858 if (ret) {
1859 unlock_memory_hotplug();
1860 BUG();
1861 }
1862
1863
1864 firmware_map_remove(start, start + size, "System RAM");
1865
1866 arch_remove_memory(start, size);
1867
1868 try_offline_node(nid);
1869
1870 unlock_memory_hotplug();
1871}
1872EXPORT_SYMBOL_GPL(remove_memory);
1873#endif
1874