1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/kernel.h>
19#include <linux/mm.h>
20#include <linux/nmi.h>
21#include <linux/swap.h>
22#include <linux/memblock.h>
23#include <linux/acpi.h>
24#include <linux/efi.h>
25#include <linux/nodemask.h>
26#include <linux/slab.h>
27#include <asm/pgalloc.h>
28#include <asm/tlb.h>
29#include <asm/meminit.h>
30#include <asm/numa.h>
31#include <asm/sections.h>
32
33
34
35
36
37struct early_node_data {
38 struct ia64_node_data *node_data;
39 unsigned long pernode_addr;
40 unsigned long pernode_size;
41 unsigned long min_pfn;
42 unsigned long max_pfn;
43};
44
45static struct early_node_data mem_data[MAX_NUMNODES] __initdata;
46static nodemask_t memory_less_mask __initdata;
47
48pg_data_t *pgdat_list[MAX_NUMNODES];
49
50
51
52
53
54#define MAX_NODE_ALIGN_OFFSET (32 * 1024 * 1024)
55#define NODEDATA_ALIGN(addr, node) \
56 ((((addr) + 1024*1024-1) & ~(1024*1024-1)) + \
57 (((node)*PERCPU_PAGE_SIZE) & (MAX_NODE_ALIGN_OFFSET - 1)))
58
59
60
61
62
63
64
65
66
67
68
69
70
71static int __init build_node_maps(unsigned long start, unsigned long len,
72 int node)
73{
74 unsigned long spfn, epfn, end = start + len;
75
76 epfn = GRANULEROUNDUP(end) >> PAGE_SHIFT;
77 spfn = GRANULEROUNDDOWN(start) >> PAGE_SHIFT;
78
79 if (!mem_data[node].min_pfn) {
80 mem_data[node].min_pfn = spfn;
81 mem_data[node].max_pfn = epfn;
82 } else {
83 mem_data[node].min_pfn = min(spfn, mem_data[node].min_pfn);
84 mem_data[node].max_pfn = max(epfn, mem_data[node].max_pfn);
85 }
86
87 return 0;
88}
89
90
91
92
93
94
95
96
97
98static int __meminit early_nr_cpus_node(int node)
99{
100 int cpu, n = 0;
101
102 for_each_possible_early_cpu(cpu)
103 if (node == node_cpuid[cpu].nid)
104 n++;
105
106 return n;
107}
108
109
110
111
112
113static unsigned long __meminit compute_pernodesize(int node)
114{
115 unsigned long pernodesize = 0, cpus;
116
117 cpus = early_nr_cpus_node(node);
118 pernodesize += PERCPU_PAGE_SIZE * cpus;
119 pernodesize += node * L1_CACHE_BYTES;
120 pernodesize += L1_CACHE_ALIGN(sizeof(pg_data_t));
121 pernodesize += L1_CACHE_ALIGN(sizeof(struct ia64_node_data));
122 pernodesize += L1_CACHE_ALIGN(sizeof(pg_data_t));
123 pernodesize = PAGE_ALIGN(pernodesize);
124 return pernodesize;
125}
126
127
128
129
130
131
132
133
134
135
136static void *per_cpu_node_setup(void *cpu_data, int node)
137{
138#ifdef CONFIG_SMP
139 int cpu;
140
141 for_each_possible_early_cpu(cpu) {
142 void *src = cpu == 0 ? __cpu0_per_cpu : __phys_per_cpu_start;
143
144 if (node != node_cpuid[cpu].nid)
145 continue;
146
147 memcpy(__va(cpu_data), src, __per_cpu_end - __per_cpu_start);
148 __per_cpu_offset[cpu] = (char *)__va(cpu_data) -
149 __per_cpu_start;
150
151
152
153
154
155
156
157
158
159
160 if (cpu == 0)
161 ia64_set_kr(IA64_KR_PER_CPU_DATA,
162 (unsigned long)cpu_data -
163 (unsigned long)__per_cpu_start);
164
165 cpu_data += PERCPU_PAGE_SIZE;
166 }
167#endif
168 return cpu_data;
169}
170
171#ifdef CONFIG_SMP
172
173
174
175
176
177
178
179
180void __init setup_per_cpu_areas(void)
181{
182 struct pcpu_alloc_info *ai;
183 struct pcpu_group_info *uninitialized_var(gi);
184 unsigned int *cpu_map;
185 void *base;
186 unsigned long base_offset;
187 unsigned int cpu;
188 ssize_t static_size, reserved_size, dyn_size;
189 int node, prev_node, unit, nr_units, rc;
190
191 ai = pcpu_alloc_alloc_info(MAX_NUMNODES, nr_cpu_ids);
192 if (!ai)
193 panic("failed to allocate pcpu_alloc_info");
194 cpu_map = ai->groups[0].cpu_map;
195
196
197 base = (void *)ULONG_MAX;
198 for_each_possible_cpu(cpu)
199 base = min(base,
200 (void *)(__per_cpu_offset[cpu] + __per_cpu_start));
201 base_offset = (void *)__per_cpu_start - base;
202
203
204 unit = 0;
205 for_each_node(node)
206 for_each_possible_cpu(cpu)
207 if (node == node_cpuid[cpu].nid)
208 cpu_map[unit++] = cpu;
209 nr_units = unit;
210
211
212 static_size = __per_cpu_end - __per_cpu_start;
213 reserved_size = PERCPU_MODULE_RESERVE;
214 dyn_size = PERCPU_PAGE_SIZE - static_size - reserved_size;
215 if (dyn_size < 0)
216 panic("percpu area overflow static=%zd reserved=%zd\n",
217 static_size, reserved_size);
218
219 ai->static_size = static_size;
220 ai->reserved_size = reserved_size;
221 ai->dyn_size = dyn_size;
222 ai->unit_size = PERCPU_PAGE_SIZE;
223 ai->atom_size = PAGE_SIZE;
224 ai->alloc_size = PERCPU_PAGE_SIZE;
225
226
227
228
229
230 prev_node = NUMA_NO_NODE;
231 ai->nr_groups = 0;
232 for (unit = 0; unit < nr_units; unit++) {
233 cpu = cpu_map[unit];
234 node = node_cpuid[cpu].nid;
235
236 if (node == prev_node) {
237 gi->nr_units++;
238 continue;
239 }
240 prev_node = node;
241
242 gi = &ai->groups[ai->nr_groups++];
243 gi->nr_units = 1;
244 gi->base_offset = __per_cpu_offset[cpu] + base_offset;
245 gi->cpu_map = &cpu_map[unit];
246 }
247
248 rc = pcpu_setup_first_chunk(ai, base);
249 if (rc)
250 panic("failed to setup percpu area (err=%d)", rc);
251
252 pcpu_free_alloc_info(ai);
253}
254#endif
255
256
257
258
259
260
261
262static void __init fill_pernode(int node, unsigned long pernode,
263 unsigned long pernodesize)
264{
265 void *cpu_data;
266 int cpus = early_nr_cpus_node(node);
267
268 mem_data[node].pernode_addr = pernode;
269 mem_data[node].pernode_size = pernodesize;
270 memset(__va(pernode), 0, pernodesize);
271
272 cpu_data = (void *)pernode;
273 pernode += PERCPU_PAGE_SIZE * cpus;
274 pernode += node * L1_CACHE_BYTES;
275
276 pgdat_list[node] = __va(pernode);
277 pernode += L1_CACHE_ALIGN(sizeof(pg_data_t));
278
279 mem_data[node].node_data = __va(pernode);
280 pernode += L1_CACHE_ALIGN(sizeof(struct ia64_node_data));
281 pernode += L1_CACHE_ALIGN(sizeof(pg_data_t));
282
283 cpu_data = per_cpu_node_setup(cpu_data, node);
284
285 return;
286}
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316static int __init find_pernode_space(unsigned long start, unsigned long len,
317 int node)
318{
319 unsigned long spfn, epfn;
320 unsigned long pernodesize = 0, pernode;
321
322 spfn = start >> PAGE_SHIFT;
323 epfn = (start + len) >> PAGE_SHIFT;
324
325
326
327
328
329 if (spfn < mem_data[node].min_pfn || epfn > mem_data[node].max_pfn)
330 return 0;
331
332
333 if (mem_data[node].pernode_addr)
334 return 0;
335
336
337
338
339
340 pernodesize = compute_pernodesize(node);
341 pernode = NODEDATA_ALIGN(start, node);
342
343
344 if (start + len > (pernode + pernodesize))
345 fill_pernode(node, pernode, pernodesize);
346
347 return 0;
348}
349
350
351
352
353
354
355
356
357static void __init reserve_pernode_space(void)
358{
359 unsigned long base, size;
360 int node;
361
362 for_each_online_node(node) {
363 if (node_isset(node, memory_less_mask))
364 continue;
365
366
367 size = mem_data[node].pernode_size;
368 base = __pa(mem_data[node].pernode_addr);
369 memblock_reserve(base, size);
370 }
371}
372
373static void __meminit scatter_node_data(void)
374{
375 pg_data_t **dst;
376 int node;
377
378
379
380
381
382
383
384
385
386 for_each_node(node) {
387 if (pgdat_list[node]) {
388 dst = LOCAL_DATA_ADDR(pgdat_list[node])->pg_data_ptrs;
389 memcpy(dst, pgdat_list, sizeof(pgdat_list));
390 }
391 }
392}
393
394
395
396
397
398
399
400
401
402static void __init initialize_pernode_data(void)
403{
404 int cpu, node;
405
406 scatter_node_data();
407
408#ifdef CONFIG_SMP
409
410 for_each_possible_early_cpu(cpu) {
411 node = node_cpuid[cpu].nid;
412 per_cpu(ia64_cpu_info, cpu).node_data =
413 mem_data[node].node_data;
414 }
415#else
416 {
417 struct cpuinfo_ia64 *cpu0_cpu_info;
418 cpu = 0;
419 node = node_cpuid[cpu].nid;
420 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start +
421 ((char *)&ia64_cpu_info - __per_cpu_start));
422 cpu0_cpu_info->node_data = mem_data[node].node_data;
423 }
424#endif
425}
426
427
428
429
430
431
432
433
434static void __init *memory_less_node_alloc(int nid, unsigned long pernodesize)
435{
436 void *ptr = NULL;
437 u8 best = 0xff;
438 int bestnode = NUMA_NO_NODE, node, anynode = 0;
439
440 for_each_online_node(node) {
441 if (node_isset(node, memory_less_mask))
442 continue;
443 else if (node_distance(nid, node) < best) {
444 best = node_distance(nid, node);
445 bestnode = node;
446 }
447 anynode = node;
448 }
449
450 if (bestnode == NUMA_NO_NODE)
451 bestnode = anynode;
452
453 ptr = memblock_alloc_try_nid(pernodesize, PERCPU_PAGE_SIZE,
454 __pa(MAX_DMA_ADDRESS),
455 MEMBLOCK_ALLOC_ACCESSIBLE,
456 bestnode);
457 if (!ptr)
458 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%lx\n",
459 __func__, pernodesize, PERCPU_PAGE_SIZE, bestnode,
460 __pa(MAX_DMA_ADDRESS));
461
462 return ptr;
463}
464
465
466
467
468
469static void __init memory_less_nodes(void)
470{
471 unsigned long pernodesize;
472 void *pernode;
473 int node;
474
475 for_each_node_mask(node, memory_less_mask) {
476 pernodesize = compute_pernodesize(node);
477 pernode = memory_less_node_alloc(node, pernodesize);
478 fill_pernode(node, __pa(pernode), pernodesize);
479 }
480
481 return;
482}
483
484
485
486
487
488
489
490void __init find_memory(void)
491{
492 int node;
493
494 reserve_memory();
495 efi_memmap_walk(filter_memory, register_active_ranges);
496
497 if (num_online_nodes() == 0) {
498 printk(KERN_ERR "node info missing!\n");
499 node_set_online(0);
500 }
501
502 nodes_or(memory_less_mask, memory_less_mask, node_online_map);
503 min_low_pfn = -1;
504 max_low_pfn = 0;
505
506
507 efi_memmap_walk(filter_rsvd_memory, build_node_maps);
508 efi_memmap_walk(filter_rsvd_memory, find_pernode_space);
509 efi_memmap_walk(find_max_min_low_pfn, NULL);
510
511 for_each_online_node(node)
512 if (mem_data[node].min_pfn)
513 node_clear(node, memory_less_mask);
514
515 reserve_pernode_space();
516 memory_less_nodes();
517 initialize_pernode_data();
518
519 max_pfn = max_low_pfn;
520
521 find_initrd();
522}
523
524#ifdef CONFIG_SMP
525
526
527
528
529
530
531void *per_cpu_init(void)
532{
533 int cpu;
534 static int first_time = 1;
535
536 if (first_time) {
537 first_time = 0;
538 for_each_possible_early_cpu(cpu)
539 per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu];
540 }
541
542 return __per_cpu_start + __per_cpu_offset[smp_processor_id()];
543}
544#endif
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559void call_pernode_memory(unsigned long start, unsigned long len, void *arg)
560{
561 unsigned long rs, re, end = start + len;
562 void (*func)(unsigned long, unsigned long, int);
563 int i;
564
565 start = PAGE_ALIGN(start);
566 end &= PAGE_MASK;
567 if (start >= end)
568 return;
569
570 func = arg;
571
572 if (!num_node_memblks) {
573
574 if (start < end)
575 (*func)(start, end - start, 0);
576 return;
577 }
578
579 for (i = 0; i < num_node_memblks; i++) {
580 rs = max(start, node_memblk[i].start_paddr);
581 re = min(end, node_memblk[i].start_paddr +
582 node_memblk[i].size);
583
584 if (rs < re)
585 (*func)(rs, re - rs, node_memblk[i].nid);
586
587 if (re == end)
588 break;
589 }
590}
591
592
593
594
595
596
597
598void __init paging_init(void)
599{
600 unsigned long max_dma;
601 unsigned long pfn_offset = 0;
602 unsigned long max_pfn = 0;
603 int node;
604 unsigned long max_zone_pfns[MAX_NR_ZONES];
605
606 max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT;
607
608 sparse_memory_present_with_active_regions(MAX_NUMNODES);
609 sparse_init();
610
611#ifdef CONFIG_VIRTUAL_MEM_MAP
612 VMALLOC_END -= PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) *
613 sizeof(struct page));
614 vmem_map = (struct page *) VMALLOC_END;
615 efi_memmap_walk(create_mem_map_page_table, NULL);
616 printk("Virtual mem_map starts at 0x%p\n", vmem_map);
617#endif
618
619 for_each_online_node(node) {
620 pfn_offset = mem_data[node].min_pfn;
621
622#ifdef CONFIG_VIRTUAL_MEM_MAP
623 NODE_DATA(node)->node_mem_map = vmem_map + pfn_offset;
624#endif
625 if (mem_data[node].max_pfn > max_pfn)
626 max_pfn = mem_data[node].max_pfn;
627 }
628
629 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
630#ifdef CONFIG_ZONE_DMA32
631 max_zone_pfns[ZONE_DMA32] = max_dma;
632#endif
633 max_zone_pfns[ZONE_NORMAL] = max_pfn;
634 free_area_init_nodes(max_zone_pfns);
635
636 zero_page_memmap_ptr = virt_to_page(ia64_imva(empty_zero_page));
637}
638
639#ifdef CONFIG_MEMORY_HOTPLUG
640pg_data_t *arch_alloc_nodedata(int nid)
641{
642 unsigned long size = compute_pernodesize(nid);
643
644 return kzalloc(size, GFP_KERNEL);
645}
646
647void arch_free_nodedata(pg_data_t *pgdat)
648{
649 kfree(pgdat);
650}
651
652void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat)
653{
654 pgdat_list[update_node] = update_pgdat;
655 scatter_node_data();
656}
657#endif
658
659#ifdef CONFIG_SPARSEMEM_VMEMMAP
660int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
661 struct vmem_altmap *altmap)
662{
663 return vmemmap_populate_basepages(start, end, node);
664}
665
666void vmemmap_free(unsigned long start, unsigned long end,
667 struct vmem_altmap *altmap)
668{
669}
670#endif
671