1#include <linux/initrd.h>
2#include <linux/ioport.h>
3#include <linux/swap.h>
4
5#include <asm/cacheflush.h>
6#include <asm/e820.h>
7#include <asm/init.h>
8#include <asm/page.h>
9#include <asm/page_types.h>
10#include <asm/sections.h>
11#include <asm/setup.h>
12#include <asm/system.h>
13#include <asm/tlbflush.h>
14#include <asm/tlb.h>
15#include <asm/proto.h>
16
17DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
18
19unsigned long __initdata e820_table_start;
20unsigned long __meminitdata e820_table_end;
21unsigned long __meminitdata e820_table_top;
22
23int after_bootmem;
24
25int direct_gbpages
26#ifdef CONFIG_DIRECT_GBPAGES
27 = 1
28#endif
29;
30
31static void __init find_early_table_space(unsigned long end, int use_pse,
32 int use_gbpages)
33{
34 unsigned long puds, pmds, ptes, tables, start;
35
36 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
37 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
38
39 if (use_gbpages) {
40 unsigned long extra;
41
42 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT);
43 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT;
44 } else
45 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
46
47 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
48
49 if (use_pse) {
50 unsigned long extra;
51
52 extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT);
53#ifdef CONFIG_X86_32
54 extra += PMD_SIZE;
55#endif
56 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
57 } else
58 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
59
60 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
61
62#ifdef CONFIG_X86_32
63
64 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
65#endif
66
67
68
69
70
71
72#ifdef CONFIG_X86_32
73 start = 0x7000;
74#else
75 start = 0x8000;
76#endif
77 e820_table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
78 tables, PAGE_SIZE);
79 if (e820_table_start == -1UL)
80 panic("Cannot find space for the kernel page tables");
81
82 e820_table_start >>= PAGE_SHIFT;
83 e820_table_end = e820_table_start;
84 e820_table_top = e820_table_start + (tables >> PAGE_SHIFT);
85
86 printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n",
87 end, e820_table_start << PAGE_SHIFT, e820_table_top << PAGE_SHIFT);
88}
89
90struct map_range {
91 unsigned long start;
92 unsigned long end;
93 unsigned page_size_mask;
94};
95
96#ifdef CONFIG_X86_32
97#define NR_RANGE_MR 3
98#else
99#define NR_RANGE_MR 5
100#endif
101
102static int __meminit save_mr(struct map_range *mr, int nr_range,
103 unsigned long start_pfn, unsigned long end_pfn,
104 unsigned long page_size_mask)
105{
106 if (start_pfn < end_pfn) {
107 if (nr_range >= NR_RANGE_MR)
108 panic("run out of range for init_memory_mapping\n");
109 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
110 mr[nr_range].end = end_pfn<<PAGE_SHIFT;
111 mr[nr_range].page_size_mask = page_size_mask;
112 nr_range++;
113 }
114
115 return nr_range;
116}
117
118
119
120
121
122
123unsigned long __init_refok init_memory_mapping(unsigned long start,
124 unsigned long end)
125{
126 unsigned long page_size_mask = 0;
127 unsigned long start_pfn, end_pfn;
128 unsigned long ret = 0;
129 unsigned long pos;
130
131 struct map_range mr[NR_RANGE_MR];
132 int nr_range, i;
133 int use_pse, use_gbpages;
134
135 printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end);
136
137#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
138
139
140
141
142
143 use_pse = use_gbpages = 0;
144#else
145 use_pse = cpu_has_pse;
146 use_gbpages = direct_gbpages;
147#endif
148
149 set_nx();
150 if (nx_enabled)
151 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
152
153
154 if (cpu_has_pse)
155 set_in_cr4(X86_CR4_PSE);
156
157
158 if (cpu_has_pge) {
159 set_in_cr4(X86_CR4_PGE);
160 __supported_pte_mask |= _PAGE_GLOBAL;
161 }
162
163 if (use_gbpages)
164 page_size_mask |= 1 << PG_LEVEL_1G;
165 if (use_pse)
166 page_size_mask |= 1 << PG_LEVEL_2M;
167
168 memset(mr, 0, sizeof(mr));
169 nr_range = 0;
170
171
172 start_pfn = start >> PAGE_SHIFT;
173 pos = start_pfn << PAGE_SHIFT;
174#ifdef CONFIG_X86_32
175
176
177
178
179
180
181 if (pos == 0)
182 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
183 else
184 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
185 << (PMD_SHIFT - PAGE_SHIFT);
186#else
187 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
188 << (PMD_SHIFT - PAGE_SHIFT);
189#endif
190 if (end_pfn > (end >> PAGE_SHIFT))
191 end_pfn = end >> PAGE_SHIFT;
192 if (start_pfn < end_pfn) {
193 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
194 pos = end_pfn << PAGE_SHIFT;
195 }
196
197
198 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
199 << (PMD_SHIFT - PAGE_SHIFT);
200#ifdef CONFIG_X86_32
201 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
202#else
203 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
204 << (PUD_SHIFT - PAGE_SHIFT);
205 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
206 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
207#endif
208
209 if (start_pfn < end_pfn) {
210 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
211 page_size_mask & (1<<PG_LEVEL_2M));
212 pos = end_pfn << PAGE_SHIFT;
213 }
214
215#ifdef CONFIG_X86_64
216
217 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
218 << (PUD_SHIFT - PAGE_SHIFT);
219 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
220 if (start_pfn < end_pfn) {
221 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
222 page_size_mask &
223 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
224 pos = end_pfn << PAGE_SHIFT;
225 }
226
227
228 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
229 << (PMD_SHIFT - PAGE_SHIFT);
230 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
231 if (start_pfn < end_pfn) {
232 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
233 page_size_mask & (1<<PG_LEVEL_2M));
234 pos = end_pfn << PAGE_SHIFT;
235 }
236#endif
237
238
239 start_pfn = pos>>PAGE_SHIFT;
240 end_pfn = end>>PAGE_SHIFT;
241 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
242
243
244 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
245 unsigned long old_start;
246 if (mr[i].end != mr[i+1].start ||
247 mr[i].page_size_mask != mr[i+1].page_size_mask)
248 continue;
249
250 old_start = mr[i].start;
251 memmove(&mr[i], &mr[i+1],
252 (nr_range - 1 - i) * sizeof(struct map_range));
253 mr[i--].start = old_start;
254 nr_range--;
255 }
256
257 for (i = 0; i < nr_range; i++)
258 printk(KERN_DEBUG " %010lx - %010lx page %s\n",
259 mr[i].start, mr[i].end,
260 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
261 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
262
263
264
265
266
267
268
269
270 if (!after_bootmem)
271 find_early_table_space(end, use_pse, use_gbpages);
272
273#ifdef CONFIG_X86_32
274 for (i = 0; i < nr_range; i++)
275 kernel_physical_mapping_init(mr[i].start, mr[i].end,
276 mr[i].page_size_mask);
277 ret = end;
278#else
279 for (i = 0; i < nr_range; i++)
280 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
281 mr[i].page_size_mask);
282#endif
283
284#ifdef CONFIG_X86_32
285 early_ioremap_page_table_range_init();
286
287 load_cr3(swapper_pg_dir);
288#endif
289
290#ifdef CONFIG_X86_64
291 if (!after_bootmem && !start) {
292 pud_t *pud;
293 pmd_t *pmd;
294
295 mmu_cr4_features = read_cr4();
296
297
298
299
300
301
302
303 pud = pud_offset(pgd_offset_k(_brk_end), _brk_end);
304 pmd = pmd_offset(pud, _brk_end - 1);
305 while (++pmd <= pmd_offset(pud, (unsigned long)_end - 1))
306 pmd_clear(pmd);
307 }
308#endif
309 __flush_tlb_all();
310
311 if (!after_bootmem && e820_table_end > e820_table_start)
312 reserve_early(e820_table_start << PAGE_SHIFT,
313 e820_table_end << PAGE_SHIFT, "PGTABLE");
314
315 if (!after_bootmem)
316 early_memtest(start, end);
317
318 return ret >> PAGE_SHIFT;
319}
320
321
322
323
324
325
326
327
328
329
330
331
332int devmem_is_allowed(unsigned long pagenr)
333{
334 if (pagenr <= 256)
335 return 1;
336 if (iomem_is_exclusive(pagenr << PAGE_SHIFT))
337 return 0;
338 if (!page_is_ram(pagenr))
339 return 1;
340 return 0;
341}
342
343void free_init_pages(char *what, unsigned long begin, unsigned long end)
344{
345 unsigned long addr = begin;
346
347 if (addr >= end)
348 return;
349
350
351
352
353
354
355#ifdef CONFIG_DEBUG_PAGEALLOC
356 printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n",
357 begin, PAGE_ALIGN(end));
358 set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
359#else
360
361
362
363
364
365 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
366
367 printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
368
369 for (; addr < end; addr += PAGE_SIZE) {
370 ClearPageReserved(virt_to_page(addr));
371 init_page_count(virt_to_page(addr));
372 memset((void *)(addr & ~(PAGE_SIZE-1)),
373 POISON_FREE_INITMEM, PAGE_SIZE);
374 free_page(addr);
375 totalram_pages++;
376 }
377#endif
378}
379
380void free_initmem(void)
381{
382 free_init_pages("unused kernel memory",
383 (unsigned long)(&__init_begin),
384 (unsigned long)(&__init_end));
385}
386
387#ifdef CONFIG_BLK_DEV_INITRD
388void free_initrd_mem(unsigned long start, unsigned long end)
389{
390 free_init_pages("initrd memory", start, end);
391}
392#endif
393