1
2
3
4
5
6
7#include <linux/kernel.h>
8#include <linux/init.h>
9
10#include <linux/bootmem.h>
11#include <linux/efi.h>
12#include <linux/elf.h>
13#include <linux/mm.h>
14#include <linux/mmzone.h>
15#include <linux/module.h>
16#include <linux/personality.h>
17#include <linux/reboot.h>
18#include <linux/slab.h>
19#include <linux/swap.h>
20#include <linux/proc_fs.h>
21#include <linux/bitops.h>
22#include <linux/kexec.h>
23
24#include <asm/dma.h>
25#include <asm/ia32.h>
26#include <asm/io.h>
27#include <asm/machvec.h>
28#include <asm/numa.h>
29#include <asm/patch.h>
30#include <asm/pgalloc.h>
31#include <asm/sal.h>
32#include <asm/sections.h>
33#include <asm/system.h>
34#include <asm/tlb.h>
35#include <asm/uaccess.h>
36#include <asm/unistd.h>
37#include <asm/mca.h>
38#include <asm/paravirt.h>
39
40DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
41
42extern void ia64_tlb_init (void);
43
44unsigned long MAX_DMA_ADDRESS = PAGE_OFFSET + 0x100000000UL;
45
46#ifdef CONFIG_VIRTUAL_MEM_MAP
47unsigned long vmalloc_end = VMALLOC_END_INIT;
48EXPORT_SYMBOL(vmalloc_end);
49struct page *vmem_map;
50EXPORT_SYMBOL(vmem_map);
51#endif
52
53struct page *zero_page_memmap_ptr;
54EXPORT_SYMBOL(zero_page_memmap_ptr);
55
56void
57__ia64_sync_icache_dcache (pte_t pte)
58{
59 unsigned long addr;
60 struct page *page;
61
62 page = pte_page(pte);
63 addr = (unsigned long) page_address(page);
64
65 if (test_bit(PG_arch_1, &page->flags))
66 return;
67
68 flush_icache_range(addr, addr + (PAGE_SIZE << compound_order(page)));
69 set_bit(PG_arch_1, &page->flags);
70}
71
72
73
74
75
76
77void
78dma_mark_clean(void *addr, size_t size)
79{
80 unsigned long pg_addr, end;
81
82 pg_addr = PAGE_ALIGN((unsigned long) addr);
83 end = (unsigned long) addr + size;
84 while (pg_addr + PAGE_SIZE <= end) {
85 struct page *page = virt_to_page(pg_addr);
86 set_bit(PG_arch_1, &page->flags);
87 pg_addr += PAGE_SIZE;
88 }
89}
90
91inline void
92ia64_set_rbs_bot (void)
93{
94 unsigned long stack_size = current->signal->rlim[RLIMIT_STACK].rlim_max & -16;
95
96 if (stack_size > MAX_USER_STACK_SIZE)
97 stack_size = MAX_USER_STACK_SIZE;
98 current->thread.rbs_bot = PAGE_ALIGN(current->mm->start_stack - stack_size);
99}
100
101
102
103
104
105
106
107void
108ia64_init_addr_space (void)
109{
110 struct vm_area_struct *vma;
111
112 ia64_set_rbs_bot();
113
114
115
116
117
118
119 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
120 if (vma) {
121 vma->vm_mm = current->mm;
122 vma->vm_start = current->thread.rbs_bot & PAGE_MASK;
123 vma->vm_end = vma->vm_start + PAGE_SIZE;
124 vma->vm_flags = VM_DATA_DEFAULT_FLAGS|VM_GROWSUP|VM_ACCOUNT;
125 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
126 down_write(¤t->mm->mmap_sem);
127 if (insert_vm_struct(current->mm, vma)) {
128 up_write(¤t->mm->mmap_sem);
129 kmem_cache_free(vm_area_cachep, vma);
130 return;
131 }
132 up_write(¤t->mm->mmap_sem);
133 }
134
135
136 if (!(current->personality & MMAP_PAGE_ZERO)) {
137 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
138 if (vma) {
139 vma->vm_mm = current->mm;
140 vma->vm_end = PAGE_SIZE;
141 vma->vm_page_prot = __pgprot(pgprot_val(PAGE_READONLY) | _PAGE_MA_NAT);
142 vma->vm_flags = VM_READ | VM_MAYREAD | VM_IO | VM_RESERVED;
143 down_write(¤t->mm->mmap_sem);
144 if (insert_vm_struct(current->mm, vma)) {
145 up_write(¤t->mm->mmap_sem);
146 kmem_cache_free(vm_area_cachep, vma);
147 return;
148 }
149 up_write(¤t->mm->mmap_sem);
150 }
151 }
152}
153
154void
155free_initmem (void)
156{
157 unsigned long addr, eaddr;
158
159 addr = (unsigned long) ia64_imva(__init_begin);
160 eaddr = (unsigned long) ia64_imva(__init_end);
161 while (addr < eaddr) {
162 ClearPageReserved(virt_to_page(addr));
163 init_page_count(virt_to_page(addr));
164 free_page(addr);
165 ++totalram_pages;
166 addr += PAGE_SIZE;
167 }
168 printk(KERN_INFO "Freeing unused kernel memory: %ldkB freed\n",
169 (__init_end - __init_begin) >> 10);
170}
171
172void __init
173free_initrd_mem (unsigned long start, unsigned long end)
174{
175 struct page *page;
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207 start = PAGE_ALIGN(start);
208 end = end & PAGE_MASK;
209
210 if (start < end)
211 printk(KERN_INFO "Freeing initrd memory: %ldkB freed\n", (end - start) >> 10);
212
213 for (; start < end; start += PAGE_SIZE) {
214 if (!virt_addr_valid(start))
215 continue;
216 page = virt_to_page(start);
217 ClearPageReserved(page);
218 init_page_count(page);
219 free_page(start);
220 ++totalram_pages;
221 }
222}
223
224
225
226
227static struct page * __init
228put_kernel_page (struct page *page, unsigned long address, pgprot_t pgprot)
229{
230 pgd_t *pgd;
231 pud_t *pud;
232 pmd_t *pmd;
233 pte_t *pte;
234
235 if (!PageReserved(page))
236 printk(KERN_ERR "put_kernel_page: page at 0x%p not in reserved memory\n",
237 page_address(page));
238
239 pgd = pgd_offset_k(address);
240
241 {
242 pud = pud_alloc(&init_mm, pgd, address);
243 if (!pud)
244 goto out;
245 pmd = pmd_alloc(&init_mm, pud, address);
246 if (!pmd)
247 goto out;
248 pte = pte_alloc_kernel(pmd, address);
249 if (!pte)
250 goto out;
251 if (!pte_none(*pte))
252 goto out;
253 set_pte(pte, mk_pte(page, pgprot));
254 }
255 out:
256
257 return page;
258}
259
260static void __init
261setup_gate (void)
262{
263 void *gate_section;
264 struct page *page;
265
266
267
268
269
270
271 gate_section = paravirt_get_gate_section();
272 page = virt_to_page(ia64_imva(gate_section));
273 put_kernel_page(page, GATE_ADDR, PAGE_READONLY);
274#ifdef HAVE_BUGGY_SEGREL
275 page = virt_to_page(ia64_imva(gate_section + PAGE_SIZE));
276 put_kernel_page(page, GATE_ADDR + PAGE_SIZE, PAGE_GATE);
277#else
278 put_kernel_page(page, GATE_ADDR + PERCPU_PAGE_SIZE, PAGE_GATE);
279
280 {
281 unsigned long addr;
282
283 for (addr = GATE_ADDR + PAGE_SIZE;
284 addr < GATE_ADDR + PERCPU_PAGE_SIZE;
285 addr += PAGE_SIZE)
286 {
287 put_kernel_page(ZERO_PAGE(0), addr,
288 PAGE_READONLY);
289 put_kernel_page(ZERO_PAGE(0), addr + PERCPU_PAGE_SIZE,
290 PAGE_READONLY);
291 }
292 }
293#endif
294 ia64_patch_gate();
295}
296
297void __devinit
298ia64_mmu_init (void *my_cpu_data)
299{
300 unsigned long pta, impl_va_bits;
301 extern void __devinit tlb_init (void);
302
303#ifdef CONFIG_DISABLE_VHPT
304# define VHPT_ENABLE_BIT 0
305#else
306# define VHPT_ENABLE_BIT 1
307#endif
308
309
310
311
312
313
314
315
316
317
318
319
320# define pte_bits 3
321# define mapped_space_bits (3*(PAGE_SHIFT - pte_bits) + PAGE_SHIFT)
322
323
324
325
326
327
328
329# define vmlpt_bits (impl_va_bits - PAGE_SHIFT + pte_bits)
330# define POW2(n) (1ULL << (n))
331
332 impl_va_bits = ffz(~(local_cpu_data->unimpl_va_mask | (7UL << 61)));
333
334 if (impl_va_bits < 51 || impl_va_bits > 61)
335 panic("CPU has bogus IMPL_VA_MSB value of %lu!\n", impl_va_bits - 1);
336
337
338
339
340
341
342 if ((mapped_space_bits - PAGE_SHIFT > vmlpt_bits - pte_bits) ||
343 (mapped_space_bits > impl_va_bits - 1))
344 panic("Cannot build a big enough virtual-linear page table"
345 " to cover mapped address space.\n"
346 " Try using a smaller page size.\n");
347
348
349
350 pta = POW2(61) - POW2(vmlpt_bits);
351
352
353
354
355
356
357
358 ia64_set_pta(pta | (0 << 8) | (vmlpt_bits << 2) | VHPT_ENABLE_BIT);
359
360 ia64_tlb_init();
361
362#ifdef CONFIG_HUGETLB_PAGE
363 ia64_set_rr(HPAGE_REGION_BASE, HPAGE_SHIFT << 2);
364 ia64_srlz_d();
365#endif
366}
367
368#ifdef CONFIG_VIRTUAL_MEM_MAP
369int vmemmap_find_next_valid_pfn(int node, int i)
370{
371 unsigned long end_address, hole_next_pfn;
372 unsigned long stop_address;
373 pg_data_t *pgdat = NODE_DATA(node);
374
375 end_address = (unsigned long) &vmem_map[pgdat->node_start_pfn + i];
376 end_address = PAGE_ALIGN(end_address);
377
378 stop_address = (unsigned long) &vmem_map[
379 pgdat->node_start_pfn + pgdat->node_spanned_pages];
380
381 do {
382 pgd_t *pgd;
383 pud_t *pud;
384 pmd_t *pmd;
385 pte_t *pte;
386
387 pgd = pgd_offset_k(end_address);
388 if (pgd_none(*pgd)) {
389 end_address += PGDIR_SIZE;
390 continue;
391 }
392
393 pud = pud_offset(pgd, end_address);
394 if (pud_none(*pud)) {
395 end_address += PUD_SIZE;
396 continue;
397 }
398
399 pmd = pmd_offset(pud, end_address);
400 if (pmd_none(*pmd)) {
401 end_address += PMD_SIZE;
402 continue;
403 }
404
405 pte = pte_offset_kernel(pmd, end_address);
406retry_pte:
407 if (pte_none(*pte)) {
408 end_address += PAGE_SIZE;
409 pte++;
410 if ((end_address < stop_address) &&
411 (end_address != ALIGN(end_address, 1UL << PMD_SHIFT)))
412 goto retry_pte;
413 continue;
414 }
415
416 break;
417 } while (end_address < stop_address);
418
419 end_address = min(end_address, stop_address);
420 end_address = end_address - (unsigned long) vmem_map + sizeof(struct page) - 1;
421 hole_next_pfn = end_address / sizeof(struct page);
422 return hole_next_pfn - pgdat->node_start_pfn;
423}
424
425int __init create_mem_map_page_table(u64 start, u64 end, void *arg)
426{
427 unsigned long address, start_page, end_page;
428 struct page *map_start, *map_end;
429 int node;
430 pgd_t *pgd;
431 pud_t *pud;
432 pmd_t *pmd;
433 pte_t *pte;
434
435 map_start = vmem_map + (__pa(start) >> PAGE_SHIFT);
436 map_end = vmem_map + (__pa(end) >> PAGE_SHIFT);
437
438 start_page = (unsigned long) map_start & PAGE_MASK;
439 end_page = PAGE_ALIGN((unsigned long) map_end);
440 node = paddr_to_nid(__pa(start));
441
442 for (address = start_page; address < end_page; address += PAGE_SIZE) {
443 pgd = pgd_offset_k(address);
444 if (pgd_none(*pgd))
445 pgd_populate(&init_mm, pgd, alloc_bootmem_pages_node(NODE_DATA(node), PAGE_SIZE));
446 pud = pud_offset(pgd, address);
447
448 if (pud_none(*pud))
449 pud_populate(&init_mm, pud, alloc_bootmem_pages_node(NODE_DATA(node), PAGE_SIZE));
450 pmd = pmd_offset(pud, address);
451
452 if (pmd_none(*pmd))
453 pmd_populate_kernel(&init_mm, pmd, alloc_bootmem_pages_node(NODE_DATA(node), PAGE_SIZE));
454 pte = pte_offset_kernel(pmd, address);
455
456 if (pte_none(*pte))
457 set_pte(pte, pfn_pte(__pa(alloc_bootmem_pages_node(NODE_DATA(node), PAGE_SIZE)) >> PAGE_SHIFT,
458 PAGE_KERNEL));
459 }
460 return 0;
461}
462
463struct memmap_init_callback_data {
464 struct page *start;
465 struct page *end;
466 int nid;
467 unsigned long zone;
468};
469
470static int __meminit
471virtual_memmap_init(u64 start, u64 end, void *arg)
472{
473 struct memmap_init_callback_data *args;
474 struct page *map_start, *map_end;
475
476 args = (struct memmap_init_callback_data *) arg;
477 map_start = vmem_map + (__pa(start) >> PAGE_SHIFT);
478 map_end = vmem_map + (__pa(end) >> PAGE_SHIFT);
479
480 if (map_start < args->start)
481 map_start = args->start;
482 if (map_end > args->end)
483 map_end = args->end;
484
485
486
487
488
489
490 map_start -= ((unsigned long) map_start & (PAGE_SIZE - 1)) / sizeof(struct page);
491 map_end += ((PAGE_ALIGN((unsigned long) map_end) - (unsigned long) map_end)
492 / sizeof(struct page));
493
494 if (map_start < map_end)
495 memmap_init_zone((unsigned long)(map_end - map_start),
496 args->nid, args->zone, page_to_pfn(map_start),
497 MEMMAP_EARLY);
498 return 0;
499}
500
501void __meminit
502memmap_init (unsigned long size, int nid, unsigned long zone,
503 unsigned long start_pfn)
504{
505 if (!vmem_map)
506 memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY);
507 else {
508 struct page *start;
509 struct memmap_init_callback_data args;
510
511 start = pfn_to_page(start_pfn);
512 args.start = start;
513 args.end = start + size;
514 args.nid = nid;
515 args.zone = zone;
516
517 efi_memmap_walk(virtual_memmap_init, &args);
518 }
519}
520
521int
522ia64_pfn_valid (unsigned long pfn)
523{
524 char byte;
525 struct page *pg = pfn_to_page(pfn);
526
527 return (__get_user(byte, (char __user *) pg) == 0)
528 && ((((u64)pg & PAGE_MASK) == (((u64)(pg + 1) - 1) & PAGE_MASK))
529 || (__get_user(byte, (char __user *) (pg + 1) - 1) == 0));
530}
531EXPORT_SYMBOL(ia64_pfn_valid);
532
533int __init find_largest_hole(u64 start, u64 end, void *arg)
534{
535 u64 *max_gap = arg;
536
537 static u64 last_end = PAGE_OFFSET;
538
539
540
541 if (*max_gap < (start - last_end))
542 *max_gap = start - last_end;
543 last_end = end;
544 return 0;
545}
546
547#endif
548
549int __init register_active_ranges(u64 start, u64 len, int nid)
550{
551 u64 end = start + len;
552
553#ifdef CONFIG_KEXEC
554 if (start > crashk_res.start && start < crashk_res.end)
555 start = crashk_res.end;
556 if (end > crashk_res.start && end < crashk_res.end)
557 end = crashk_res.start;
558#endif
559
560 if (start < end)
561 add_active_range(nid, __pa(start) >> PAGE_SHIFT,
562 __pa(end) >> PAGE_SHIFT);
563 return 0;
564}
565
566static int __init
567count_reserved_pages(u64 start, u64 end, void *arg)
568{
569 unsigned long num_reserved = 0;
570 unsigned long *count = arg;
571
572 for (; start < end; start += PAGE_SIZE)
573 if (PageReserved(virt_to_page(start)))
574 ++num_reserved;
575 *count += num_reserved;
576 return 0;
577}
578
579int
580find_max_min_low_pfn (u64 start, u64 end, void *arg)
581{
582 unsigned long pfn_start, pfn_end;
583#ifdef CONFIG_FLATMEM
584 pfn_start = (PAGE_ALIGN(__pa(start))) >> PAGE_SHIFT;
585 pfn_end = (PAGE_ALIGN(__pa(end - 1))) >> PAGE_SHIFT;
586#else
587 pfn_start = GRANULEROUNDDOWN(__pa(start)) >> PAGE_SHIFT;
588 pfn_end = GRANULEROUNDUP(__pa(end - 1)) >> PAGE_SHIFT;
589#endif
590 min_low_pfn = min(min_low_pfn, pfn_start);
591 max_low_pfn = max(max_low_pfn, pfn_end);
592 return 0;
593}
594
595
596
597
598
599
600
601
602
603static int nolwsys __initdata;
604
605static int __init
606nolwsys_setup (char *s)
607{
608 nolwsys = 1;
609 return 1;
610}
611
612__setup("nolwsys", nolwsys_setup);
613
614void __init
615mem_init (void)
616{
617 long reserved_pages, codesize, datasize, initsize;
618 pg_data_t *pgdat;
619 int i;
620
621 BUG_ON(PTRS_PER_PGD * sizeof(pgd_t) != PAGE_SIZE);
622 BUG_ON(PTRS_PER_PMD * sizeof(pmd_t) != PAGE_SIZE);
623 BUG_ON(PTRS_PER_PTE * sizeof(pte_t) != PAGE_SIZE);
624
625#ifdef CONFIG_PCI
626
627
628
629
630
631 platform_dma_init();
632#endif
633
634#ifdef CONFIG_FLATMEM
635 BUG_ON(!mem_map);
636 max_mapnr = max_low_pfn;
637#endif
638
639 high_memory = __va(max_low_pfn * PAGE_SIZE);
640
641 for_each_online_pgdat(pgdat)
642 if (pgdat->bdata->node_bootmem_map)
643 totalram_pages += free_all_bootmem_node(pgdat);
644
645 reserved_pages = 0;
646 efi_memmap_walk(count_reserved_pages, &reserved_pages);
647
648 codesize = (unsigned long) _etext - (unsigned long) _stext;
649 datasize = (unsigned long) _edata - (unsigned long) _etext;
650 initsize = (unsigned long) __init_end - (unsigned long) __init_begin;
651
652 printk(KERN_INFO "Memory: %luk/%luk available (%luk code, %luk reserved, "
653 "%luk data, %luk init)\n", nr_free_pages() << (PAGE_SHIFT - 10),
654 num_physpages << (PAGE_SHIFT - 10), codesize >> 10,
655 reserved_pages << (PAGE_SHIFT - 10), datasize >> 10, initsize >> 10);
656
657
658
659
660
661
662
663 for (i = 0; i < NR_syscalls; ++i) {
664 extern unsigned long sys_call_table[NR_syscalls];
665 unsigned long *fsyscall_table = paravirt_get_fsyscall_table();
666
667 if (!fsyscall_table[i] || nolwsys)
668 fsyscall_table[i] = sys_call_table[i] | 1;
669 }
670 setup_gate();
671
672#ifdef CONFIG_IA32_SUPPORT
673 ia32_mem_init();
674#endif
675}
676
677#ifdef CONFIG_MEMORY_HOTPLUG
678int arch_add_memory(int nid, u64 start, u64 size)
679{
680 pg_data_t *pgdat;
681 struct zone *zone;
682 unsigned long start_pfn = start >> PAGE_SHIFT;
683 unsigned long nr_pages = size >> PAGE_SHIFT;
684 int ret;
685
686 pgdat = NODE_DATA(nid);
687
688 zone = pgdat->node_zones + ZONE_NORMAL;
689 ret = __add_pages(nid, zone, start_pfn, nr_pages);
690
691 if (ret)
692 printk("%s: Problem encountered in __add_pages() as ret=%d\n",
693 __func__, ret);
694
695 return ret;
696}
697#endif
698
699
700
701
702
703
704
705
706static struct exec_domain ia32_exec_domain;
707
708static int __init
709per_linux32_init(void)
710{
711 ia32_exec_domain.name = "Linux/x86";
712 ia32_exec_domain.handler = NULL;
713 ia32_exec_domain.pers_low = PER_LINUX32;
714 ia32_exec_domain.pers_high = PER_LINUX32;
715 ia32_exec_domain.signal_map = default_exec_domain.signal_map;
716 ia32_exec_domain.signal_invmap = default_exec_domain.signal_invmap;
717 register_exec_domain(&ia32_exec_domain);
718
719 return 0;
720}
721
722__initcall(per_linux32_init);
723