1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/export.h>
21#include <linux/sched.h>
22#include <linux/kernel.h>
23#include <linux/errno.h>
24#include <linux/string.h>
25#include <linux/gfp.h>
26#include <linux/types.h>
27#include <linux/mm.h>
28#include <linux/stddef.h>
29#include <linux/init.h>
30#include <linux/bootmem.h>
31#include <linux/highmem.h>
32#include <linux/initrd.h>
33#include <linux/pagemap.h>
34#include <linux/suspend.h>
35#include <linux/memblock.h>
36#include <linux/hugetlb.h>
37#include <linux/slab.h>
38#include <linux/vmalloc.h>
39
40#include <asm/pgalloc.h>
41#include <asm/prom.h>
42#include <asm/io.h>
43#include <asm/mmu_context.h>
44#include <asm/pgtable.h>
45#include <asm/mmu.h>
46#include <asm/smp.h>
47#include <asm/machdep.h>
48#include <asm/btext.h>
49#include <asm/tlb.h>
50#include <asm/sections.h>
51#include <asm/sparsemem.h>
52#include <asm/vdso.h>
53#include <asm/fixmap.h>
54#include <asm/swiotlb.h>
55#include <asm/rtas.h>
56
57#include "mmu_decl.h"
58
59#ifndef CPU_FTR_COHERENT_ICACHE
60#define CPU_FTR_COHERENT_ICACHE 0
61#define CPU_FTR_NOEXECUTE 0
62#endif
63
64int init_bootmem_done;
65int mem_init_done;
66unsigned long long memory_limit;
67bool init_mem_is_free;
68
69#ifdef CONFIG_HIGHMEM
70pte_t *kmap_pte;
71EXPORT_SYMBOL(kmap_pte);
72pgprot_t kmap_prot;
73EXPORT_SYMBOL(kmap_prot);
74
75static inline pte_t *virt_to_kpte(unsigned long vaddr)
76{
77 return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
78 vaddr), vaddr), vaddr);
79}
80#endif
81
82int page_is_ram(unsigned long pfn)
83{
84#ifndef CONFIG_PPC64
85 return pfn < max_pfn;
86#else
87 unsigned long paddr = (pfn << PAGE_SHIFT);
88 struct memblock_region *reg;
89
90 for_each_memblock(memory, reg)
91 if (paddr >= reg->base && paddr < (reg->base + reg->size))
92 return 1;
93 return 0;
94#endif
95}
96
97pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
98 unsigned long size, pgprot_t vma_prot)
99{
100 if (ppc_md.phys_mem_access_prot)
101 return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
102
103 if (!page_is_ram(pfn))
104 vma_prot = pgprot_noncached(vma_prot);
105
106 return vma_prot;
107}
108EXPORT_SYMBOL(phys_mem_access_prot);
109
110#ifdef CONFIG_MEMORY_HOTPLUG
111
112#ifdef CONFIG_NUMA
113int memory_add_physaddr_to_nid(u64 start)
114{
115 return hot_add_scn_to_nid(start);
116}
117#endif
118
119int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
120 bool for_device)
121{
122 struct pglist_data *pgdata;
123 struct zone *zone;
124 unsigned long start_pfn = start >> PAGE_SHIFT;
125 unsigned long nr_pages = size >> PAGE_SHIFT;
126
127 resize_hpt_for_hotplug(memblock_phys_mem_size());
128
129 pgdata = NODE_DATA(nid);
130
131 start = (unsigned long)__va(start);
132 if (create_section_mapping(start, start + size))
133 return -EINVAL;
134
135
136 zone = pgdata->node_zones +
137 zone_for_memory(nid, start, size, 0, for_device);
138
139 return __add_pages(nid, zone, start_pfn, nr_pages, altmap);
140}
141
142#ifdef CONFIG_MEMORY_HOTREMOVE
143int arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
144{
145 unsigned long start_pfn = start >> PAGE_SHIFT;
146 unsigned long nr_pages = size >> PAGE_SHIFT;
147 struct zone *zone;
148 int ret;
149
150 zone = page_zone(pfn_to_page(start_pfn));
151 ret = __remove_pages(zone, start_pfn, nr_pages, altmap);
152 if (ret)
153 return ret;
154
155
156 start = (unsigned long)__va(start);
157 ret = remove_section_mapping(start, start + size);
158
159
160
161
162 vm_unmap_aliases();
163
164 if (resize_hpt_for_hotplug(memblock_phys_mem_size()) == -ENOSPC)
165 pr_warn("Hash collision while resizing HPT\n");
166
167 return ret;
168}
169#endif
170#endif
171
172
173
174
175
176
177
178int
179walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
180 void *arg, int (*func)(unsigned long, unsigned long, void *))
181{
182 struct memblock_region *reg;
183 unsigned long end_pfn = start_pfn + nr_pages;
184 unsigned long tstart, tend;
185 int ret = -1;
186
187 for_each_memblock(memory, reg) {
188 tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
189 tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
190 if (tstart >= tend)
191 continue;
192 ret = (*func)(tstart, tend - tstart, arg);
193 if (ret)
194 break;
195 }
196 return ret;
197}
198EXPORT_SYMBOL_GPL(walk_system_ram_range);
199
200
201
202
203
204
205#ifndef CONFIG_NEED_MULTIPLE_NODES
206void __init do_init_bootmem(void)
207{
208 unsigned long start, bootmap_pages;
209 unsigned long total_pages;
210 struct memblock_region *reg;
211 int boot_mapsize;
212
213 max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
214 total_pages = (memblock_end_of_DRAM() - memstart_addr) >> PAGE_SHIFT;
215#ifdef CONFIG_HIGHMEM
216 total_pages = total_lowmem >> PAGE_SHIFT;
217 max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
218#endif
219
220
221
222
223
224
225 bootmap_pages = bootmem_bootmap_pages(total_pages);
226
227 start = memblock_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);
228
229 min_low_pfn = MEMORY_START >> PAGE_SHIFT;
230 boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);
231
232
233
234
235 memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
236
237
238
239
240#ifdef CONFIG_HIGHMEM
241 free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);
242
243
244 for_each_memblock(reserved, reg) {
245 unsigned long top = reg->base + reg->size - 1;
246 if (top < lowmem_end_addr)
247 reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
248 else if (reg->base < lowmem_end_addr) {
249 unsigned long trunc_size = lowmem_end_addr - reg->base;
250 reserve_bootmem(reg->base, trunc_size, BOOTMEM_DEFAULT);
251 }
252 }
253#else
254 free_bootmem_with_active_regions(0, max_pfn);
255
256
257 for_each_memblock(reserved, reg)
258 reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
259#endif
260
261 sparse_memory_present_with_active_regions(0);
262
263 init_bootmem_done = 1;
264}
265
266
267static int __init mark_nonram_nosave(void)
268{
269 struct memblock_region *reg, *prev = NULL;
270
271 for_each_memblock(memory, reg) {
272 if (prev &&
273 memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
274 register_nosave_region(memblock_region_memory_end_pfn(prev),
275 memblock_region_memory_base_pfn(reg));
276 prev = reg;
277 }
278 return 0;
279}
280
281
282
283
284void __init paging_init(void)
285{
286 unsigned long long total_ram = memblock_phys_mem_size();
287 phys_addr_t top_of_ram = memblock_end_of_DRAM();
288 unsigned long max_zone_pfns[MAX_NR_ZONES];
289
290#ifdef CONFIG_PPC32
291 unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
292 unsigned long end = __fix_to_virt(FIX_HOLE);
293
294 for (; v < end; v += PAGE_SIZE)
295 map_page(v, 0, 0);
296#endif
297
298#ifdef CONFIG_HIGHMEM
299 map_page(PKMAP_BASE, 0, 0);
300 pkmap_page_table = virt_to_kpte(PKMAP_BASE);
301
302 kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
303 kmap_prot = PAGE_KERNEL;
304#endif
305
306 printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
307 (unsigned long long)top_of_ram, total_ram);
308 printk(KERN_DEBUG "Memory hole size: %ldMB\n",
309 (long int)((top_of_ram - total_ram) >> 20));
310 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
311#ifdef CONFIG_HIGHMEM
312 max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
313 max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
314#else
315 max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
316#endif
317 free_area_init_nodes(max_zone_pfns);
318
319 mark_nonram_nosave();
320}
321#endif
322
323static void __init register_page_bootmem_info(void)
324{
325 int i;
326
327 for_each_online_node(i)
328 register_page_bootmem_info_node(NODE_DATA(i));
329}
330
331void __init mem_init(void)
332{
333#ifdef CONFIG_NEED_MULTIPLE_NODES
334 int nid;
335#endif
336 pg_data_t *pgdat;
337 unsigned long i;
338 struct page *page;
339 unsigned long reservedpages = 0, codesize, initsize, datasize, bsssize;
340
341#ifdef CONFIG_SWIOTLB
342 swiotlb_init(0);
343#endif
344
345 register_page_bootmem_info();
346 num_physpages = memblock_phys_mem_size() >> PAGE_SHIFT;
347 high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
348
349#ifdef CONFIG_NEED_MULTIPLE_NODES
350 for_each_online_node(nid) {
351 if (NODE_DATA(nid)->node_spanned_pages != 0) {
352 printk("freeing bootmem node %d\n", nid);
353 totalram_pages +=
354 free_all_bootmem_node(NODE_DATA(nid));
355 }
356 }
357#else
358 max_mapnr = max_pfn;
359 totalram_pages += free_all_bootmem();
360#endif
361 for_each_online_pgdat(pgdat) {
362 for (i = 0; i < pgdat->node_spanned_pages; i++) {
363 if (!pfn_valid(pgdat->node_start_pfn + i))
364 continue;
365 page = pgdat_page_nr(pgdat, i);
366 if (PageReserved(page))
367 reservedpages++;
368 }
369 }
370
371 codesize = (unsigned long)&_sdata - (unsigned long)&_stext;
372 datasize = (unsigned long)&_edata - (unsigned long)&_sdata;
373 initsize = (unsigned long)&__init_end - (unsigned long)&__init_begin;
374 bsssize = (unsigned long)&__bss_stop - (unsigned long)&__bss_start;
375
376#ifdef CONFIG_HIGHMEM
377 {
378 unsigned long pfn, highmem_mapnr;
379
380 highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
381 for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
382 phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
383 struct page *page = pfn_to_page(pfn);
384 if (memblock_is_reserved(paddr))
385 continue;
386 free_highmem_page(page);
387 reservedpages--;
388 }
389 printk(KERN_DEBUG "High memory: %luk\n",
390 totalhigh_pages << (PAGE_SHIFT-10));
391 }
392#endif
393
394#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
395
396
397
398
399 per_cpu(next_tlbcam_idx, smp_processor_id()) =
400 (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
401#endif
402
403 printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, "
404 "%luk reserved, %luk data, %luk bss, %luk init)\n",
405 nr_free_pages() << (PAGE_SHIFT-10),
406 num_physpages << (PAGE_SHIFT-10),
407 codesize >> 10,
408 reservedpages << (PAGE_SHIFT-10),
409 datasize >> 10,
410 bsssize >> 10,
411 initsize >> 10);
412
413#ifdef CONFIG_PPC32
414 pr_info("Kernel virtual memory layout:\n");
415 pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP);
416#ifdef CONFIG_HIGHMEM
417 pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n",
418 PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
419#endif
420#ifdef CONFIG_NOT_COHERENT_CACHE
421 pr_info(" * 0x%08lx..0x%08lx : consistent mem\n",
422 IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
423#endif
424 pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
425 ioremap_bot, IOREMAP_TOP);
426 pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n",
427 VMALLOC_START, VMALLOC_END);
428#endif
429
430 mem_init_done = 1;
431}
432
433void free_initmem(void)
434{
435 ppc_md.progress = ppc_printk_progress;
436 init_mem_is_free = true;
437 free_initmem_default(POISON_FREE_INITMEM);
438}
439
440#ifdef CONFIG_BLK_DEV_INITRD
441void __init free_initrd_mem(unsigned long start, unsigned long end)
442{
443 free_reserved_area(start, end, 0, "initrd");
444}
445#endif
446
447
448
449
450
451
452void flush_dcache_page(struct page *page)
453{
454 if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
455 return;
456
457 if (test_bit(PG_arch_1, &page->flags))
458 clear_bit(PG_arch_1, &page->flags);
459}
460EXPORT_SYMBOL(flush_dcache_page);
461
462void flush_dcache_icache_page(struct page *page)
463{
464#ifdef CONFIG_HUGETLB_PAGE
465 if (PageCompound(page)) {
466 flush_dcache_icache_hugepage(page);
467 return;
468 }
469#endif
470#ifdef CONFIG_BOOKE
471 {
472 void *start = kmap_atomic(page);
473 __flush_dcache_icache(start);
474 kunmap_atomic(start);
475 }
476#elif defined(CONFIG_8xx) || defined(CONFIG_PPC64)
477
478 __flush_dcache_icache(page_address(page));
479#else
480 __flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
481#endif
482}
483EXPORT_SYMBOL(flush_dcache_icache_page);
484
485void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
486{
487 clear_page(page);
488
489
490
491
492
493
494 flush_dcache_page(pg);
495}
496EXPORT_SYMBOL(clear_user_page);
497
498void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
499 struct page *pg)
500{
501 copy_page(vto, vfrom);
502
503
504
505
506
507
508
509
510
511
512
513#if 0
514 if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
515 return;
516#endif
517
518 flush_dcache_page(pg);
519}
520
521void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
522 unsigned long addr, int len)
523{
524 unsigned long maddr;
525
526 maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
527 flush_icache_range(maddr, maddr + len);
528 kunmap(page);
529}
530EXPORT_SYMBOL(flush_icache_user_range);
531
532
533
534
535
536
537
538
539
540void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
541 pte_t *ptep)
542{
543#ifdef CONFIG_PPC_STD_MMU
544
545
546
547
548 unsigned long access = 0, trap;
549
550
551 if (!pte_young(*ptep) || address >= TASK_SIZE)
552 return;
553
554
555
556
557
558
559
560
561 if (current->thread.regs == NULL)
562 return;
563 trap = TRAP(current->thread.regs);
564 if (trap == 0x400)
565 access |= _PAGE_EXEC;
566 else if (trap != 0x300)
567 return;
568 hash_preload(vma->vm_mm, address, access, trap);
569#endif
570#if (defined(CONFIG_PPC_BOOK3E_64) || defined(CONFIG_PPC_FSL_BOOK3E)) \
571 && defined(CONFIG_HUGETLB_PAGE)
572 if (is_vm_hugetlb_page(vma))
573 book3e_hugetlb_preload(vma, address, *ptep);
574#endif
575}
576
577
578
579
580
581static int __init add_system_ram_resources(void)
582{
583 struct memblock_region *reg;
584
585 for_each_memblock(memory, reg) {
586 struct resource *res;
587 unsigned long base = reg->base;
588 unsigned long size = reg->size;
589
590 res = kzalloc(sizeof(struct resource), GFP_KERNEL);
591 WARN_ON(!res);
592
593 if (res) {
594 res->name = "System RAM";
595 res->start = base;
596 res->end = base + size - 1;
597 res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
598 WARN_ON(request_resource(&iomem_resource, res) < 0);
599 }
600 }
601
602 return 0;
603}
604subsys_initcall(add_system_ram_resources);
605
606#ifdef CONFIG_STRICT_DEVMEM
607
608
609
610
611
612
613
614int devmem_is_allowed(unsigned long pfn)
615{
616 if (iomem_is_exclusive(pfn << PAGE_SHIFT))
617 return 0;
618 if (!page_is_ram(pfn))
619 return 1;
620 if (page_is_rtas_user_buf(pfn))
621 return 1;
622 return 0;
623}
624#endif
625