1
2
3
4
5
6
7
8
9
10
11#include <linux/bug.h>
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/signal.h>
15#include <linux/sched.h>
16#include <linux/smp.h>
17#include <linux/kernel.h>
18#include <linux/errno.h>
19#include <linux/string.h>
20#include <linux/types.h>
21#include <linux/pagemap.h>
22#include <linux/ptrace.h>
23#include <linux/mman.h>
24#include <linux/mm.h>
25#include <linux/bootmem.h>
26#include <linux/highmem.h>
27#include <linux/swap.h>
28#include <linux/proc_fs.h>
29#include <linux/pfn.h>
30#include <linux/hardirq.h>
31
32#include <asm/asm-offsets.h>
33#include <asm/bootinfo.h>
34#include <asm/cachectl.h>
35#include <asm/cpu.h>
36#include <asm/dma.h>
37#include <asm/kmap_types.h>
38#include <asm/mmu_context.h>
39#include <asm/sections.h>
40#include <asm/pgtable.h>
41#include <asm/pgalloc.h>
42#include <asm/tlb.h>
43#include <asm/fixmap.h>
44
45
46#ifdef CONFIG_MIPS_MT_SMTC
47
48#include <asm/mipsmtregs.h>
49
50#define ENTER_CRITICAL(flags) \
51 { \
52 unsigned int mvpflags; \
53 local_irq_save(flags);\
54 mvpflags = dvpe()
55#define EXIT_CRITICAL(flags) \
56 evpe(mvpflags); \
57 local_irq_restore(flags); \
58 }
59#else
60
61#define ENTER_CRITICAL(flags) local_irq_save(flags)
62#define EXIT_CRITICAL(flags) local_irq_restore(flags)
63
64#endif
65
66DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
67
68
69
70
71
72
73
74
75unsigned long empty_zero_page, zero_page_mask;
76EXPORT_SYMBOL_GPL(empty_zero_page);
77
78
79
80
81unsigned long setup_zero_pages(void)
82{
83 unsigned int order;
84 unsigned long size;
85 struct page *page;
86
87 if (cpu_has_vce)
88 order = 3;
89 else
90 order = 0;
91
92 empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
93 if (!empty_zero_page)
94 panic("Oh boy, that early out of memory?");
95
96 page = virt_to_page((void *)empty_zero_page);
97 split_page(page, order);
98 while (page < virt_to_page((void *)(empty_zero_page + (PAGE_SIZE << order)))) {
99 SetPageReserved(page);
100 page++;
101 }
102
103 size = PAGE_SIZE << order;
104 zero_page_mask = (size - 1) & PAGE_MASK;
105
106 return 1UL << order;
107}
108
109#ifdef CONFIG_MIPS_MT_SMTC
110static pte_t *kmap_coherent_pte;
111static void __init kmap_coherent_init(void)
112{
113 unsigned long vaddr;
114
115
116 vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
117 kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
118}
119#else
120static inline void kmap_coherent_init(void) {}
121#endif
122
123void *kmap_coherent(struct page *page, unsigned long addr)
124{
125 enum fixed_addresses idx;
126 unsigned long vaddr, flags, entrylo;
127 unsigned long old_ctx;
128 pte_t pte;
129 int tlbidx;
130
131 BUG_ON(Page_dcache_dirty(page));
132
133 inc_preempt_count();
134 idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
135#ifdef CONFIG_MIPS_MT_SMTC
136 idx += FIX_N_COLOURS * smp_processor_id() +
137 (in_interrupt() ? (FIX_N_COLOURS * NR_CPUS) : 0);
138#else
139 idx += in_interrupt() ? FIX_N_COLOURS : 0;
140#endif
141 vaddr = __fix_to_virt(FIX_CMAP_END - idx);
142 pte = mk_pte(page, PAGE_KERNEL);
143#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
144 entrylo = pte.pte_high;
145#else
146 entrylo = pte_val(pte) >> 6;
147#endif
148
149 ENTER_CRITICAL(flags);
150 old_ctx = read_c0_entryhi();
151 write_c0_entryhi(vaddr & (PAGE_MASK << 1));
152 write_c0_entrylo0(entrylo);
153 write_c0_entrylo1(entrylo);
154#ifdef CONFIG_MIPS_MT_SMTC
155 set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
156
157 mtc0_tlbw_hazard();
158 tlb_probe();
159 tlb_probe_hazard();
160 tlbidx = read_c0_index();
161 mtc0_tlbw_hazard();
162 if (tlbidx < 0)
163 tlb_write_random();
164 else
165 tlb_write_indexed();
166#else
167 tlbidx = read_c0_wired();
168 write_c0_wired(tlbidx + 1);
169 write_c0_index(tlbidx);
170 mtc0_tlbw_hazard();
171 tlb_write_indexed();
172#endif
173 tlbw_use_hazard();
174 write_c0_entryhi(old_ctx);
175 EXIT_CRITICAL(flags);
176
177 return (void*) vaddr;
178}
179
180#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
181
182void kunmap_coherent(void)
183{
184#ifndef CONFIG_MIPS_MT_SMTC
185 unsigned int wired;
186 unsigned long flags, old_ctx;
187
188 ENTER_CRITICAL(flags);
189 old_ctx = read_c0_entryhi();
190 wired = read_c0_wired() - 1;
191 write_c0_wired(wired);
192 write_c0_index(wired);
193 write_c0_entryhi(UNIQUE_ENTRYHI(wired));
194 write_c0_entrylo0(0);
195 write_c0_entrylo1(0);
196 mtc0_tlbw_hazard();
197 tlb_write_indexed();
198 tlbw_use_hazard();
199 write_c0_entryhi(old_ctx);
200 EXIT_CRITICAL(flags);
201#endif
202 dec_preempt_count();
203 preempt_check_resched();
204}
205
206void copy_user_highpage(struct page *to, struct page *from,
207 unsigned long vaddr, struct vm_area_struct *vma)
208{
209 void *vfrom, *vto;
210
211 vto = kmap_atomic(to, KM_USER1);
212 if (cpu_has_dc_aliases &&
213 page_mapped(from) && !Page_dcache_dirty(from)) {
214 vfrom = kmap_coherent(from, vaddr);
215 copy_page(vto, vfrom);
216 kunmap_coherent();
217 } else {
218 vfrom = kmap_atomic(from, KM_USER0);
219 copy_page(vto, vfrom);
220 kunmap_atomic(vfrom, KM_USER0);
221 }
222 if ((!cpu_has_ic_fills_f_dc) ||
223 pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
224 flush_data_cache_page((unsigned long)vto);
225 kunmap_atomic(vto, KM_USER1);
226
227 smp_wmb();
228}
229
230void copy_to_user_page(struct vm_area_struct *vma,
231 struct page *page, unsigned long vaddr, void *dst, const void *src,
232 unsigned long len)
233{
234 if (cpu_has_dc_aliases &&
235 page_mapped(page) && !Page_dcache_dirty(page)) {
236 void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
237 memcpy(vto, src, len);
238 kunmap_coherent();
239 } else {
240 memcpy(dst, src, len);
241 if (cpu_has_dc_aliases)
242 SetPageDcacheDirty(page);
243 }
244 if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
245 flush_cache_page(vma, vaddr, page_to_pfn(page));
246}
247
248void copy_from_user_page(struct vm_area_struct *vma,
249 struct page *page, unsigned long vaddr, void *dst, const void *src,
250 unsigned long len)
251{
252 if (cpu_has_dc_aliases &&
253 page_mapped(page) && !Page_dcache_dirty(page)) {
254 void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
255 memcpy(dst, vfrom, len);
256 kunmap_coherent();
257 } else {
258 memcpy(dst, src, len);
259 if (cpu_has_dc_aliases)
260 SetPageDcacheDirty(page);
261 }
262}
263
264void __init fixrange_init(unsigned long start, unsigned long end,
265 pgd_t *pgd_base)
266{
267#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
268 pgd_t *pgd;
269 pud_t *pud;
270 pmd_t *pmd;
271 pte_t *pte;
272 int i, j, k;
273 unsigned long vaddr;
274
275 vaddr = start;
276 i = __pgd_offset(vaddr);
277 j = __pud_offset(vaddr);
278 k = __pmd_offset(vaddr);
279 pgd = pgd_base + i;
280
281 for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
282 pud = (pud_t *)pgd;
283 for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {
284 pmd = (pmd_t *)pud;
285 for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
286 if (pmd_none(*pmd)) {
287 pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
288 set_pmd(pmd, __pmd((unsigned long)pte));
289 BUG_ON(pte != pte_offset_kernel(pmd, 0));
290 }
291 vaddr += PMD_SIZE;
292 }
293 k = 0;
294 }
295 j = 0;
296 }
297#endif
298}
299
300#ifndef CONFIG_NEED_MULTIPLE_NODES
301static int __init page_is_ram(unsigned long pagenr)
302{
303 int i;
304
305 for (i = 0; i < boot_mem_map.nr_map; i++) {
306 unsigned long addr, end;
307
308 if (boot_mem_map.map[i].type != BOOT_MEM_RAM)
309
310 continue;
311
312 addr = PFN_UP(boot_mem_map.map[i].addr);
313 end = PFN_DOWN(boot_mem_map.map[i].addr +
314 boot_mem_map.map[i].size);
315
316 if (pagenr >= addr && pagenr < end)
317 return 1;
318 }
319
320 return 0;
321}
322
323void __init paging_init(void)
324{
325 unsigned long max_zone_pfns[MAX_NR_ZONES];
326 unsigned long lastpfn;
327
328 pagetable_init();
329
330#ifdef CONFIG_HIGHMEM
331 kmap_init();
332#endif
333 kmap_coherent_init();
334
335#ifdef CONFIG_ZONE_DMA
336 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
337#endif
338#ifdef CONFIG_ZONE_DMA32
339 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
340#endif
341 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
342 lastpfn = max_low_pfn;
343#ifdef CONFIG_HIGHMEM
344 max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
345 lastpfn = highend_pfn;
346
347 if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
348 printk(KERN_WARNING "This processor doesn't support highmem."
349 " %ldk highmem ignored\n",
350 (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
351 max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
352 lastpfn = max_low_pfn;
353 }
354#endif
355
356 free_area_init_nodes(max_zone_pfns);
357}
358
359#ifdef CONFIG_64BIT
360static struct kcore_list kcore_kseg0;
361#endif
362
363void __init mem_init(void)
364{
365 unsigned long codesize, reservedpages, datasize, initsize;
366 unsigned long tmp, ram;
367
368#ifdef CONFIG_HIGHMEM
369#ifdef CONFIG_DISCONTIGMEM
370#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
371#endif
372 max_mapnr = highend_pfn;
373#else
374 max_mapnr = max_low_pfn;
375#endif
376 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
377
378 totalram_pages += free_all_bootmem();
379 totalram_pages -= setup_zero_pages();
380
381 reservedpages = ram = 0;
382 for (tmp = 0; tmp < max_low_pfn; tmp++)
383 if (page_is_ram(tmp)) {
384 ram++;
385 if (PageReserved(pfn_to_page(tmp)))
386 reservedpages++;
387 }
388 num_physpages = ram;
389
390#ifdef CONFIG_HIGHMEM
391 for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
392 struct page *page = pfn_to_page(tmp);
393
394 if (!page_is_ram(tmp)) {
395 SetPageReserved(page);
396 continue;
397 }
398 ClearPageReserved(page);
399 init_page_count(page);
400 __free_page(page);
401 totalhigh_pages++;
402 }
403 totalram_pages += totalhigh_pages;
404 num_physpages += totalhigh_pages;
405#endif
406
407 codesize = (unsigned long) &_etext - (unsigned long) &_text;
408 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
409 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
410
411#ifdef CONFIG_64BIT
412 if ((unsigned long) &_text > (unsigned long) CKSEG0)
413
414
415 kclist_add(&kcore_kseg0, (void *) CKSEG0,
416 0x80000000 - 4, KCORE_TEXT);
417#endif
418
419 printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
420 "%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",
421 nr_free_pages() << (PAGE_SHIFT-10),
422 ram << (PAGE_SHIFT-10),
423 codesize >> 10,
424 reservedpages << (PAGE_SHIFT-10),
425 datasize >> 10,
426 initsize >> 10,
427 (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
428}
429#endif
430
431void free_init_pages(const char *what, unsigned long begin, unsigned long end)
432{
433 unsigned long pfn;
434
435 for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
436 struct page *page = pfn_to_page(pfn);
437 void *addr = phys_to_virt(PFN_PHYS(pfn));
438
439 ClearPageReserved(page);
440 init_page_count(page);
441 memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
442 __free_page(page);
443 totalram_pages++;
444 }
445 printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
446}
447
448#ifdef CONFIG_BLK_DEV_INITRD
449void free_initrd_mem(unsigned long start, unsigned long end)
450{
451 free_init_pages("initrd memory",
452 virt_to_phys((void *)start),
453 virt_to_phys((void *)end));
454}
455#endif
456
457void __init_refok free_initmem(void)
458{
459 prom_free_prom_memory();
460 free_init_pages("unused kernel memory",
461 __pa_symbol(&__init_begin),
462 __pa_symbol(&__init_end));
463}
464
465unsigned long pgd_current[NR_CPUS];
466
467
468
469
470#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))
471
472
473
474
475
476
477pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);
478#ifdef CONFIG_64BIT
479pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);
480#endif
481pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);
482
