1#ifndef _ASM_X86_PGTABLE_H
2#define _ASM_X86_PGTABLE_H
3
4#include <asm/page.h>
5#include <asm/e820.h>
6
7#include <asm/pgtable_types.h>
8
9
10
11
12#define pgprot_noncached(prot) \
13 ((boot_cpu_data.x86 > 3) \
14 ? (__pgprot(pgprot_val(prot) | _PAGE_CACHE_UC_MINUS)) \
15 : (prot))
16
17#ifndef __ASSEMBLY__
18
19
20
21
22
23extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
24#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
25
26extern spinlock_t pgd_lock;
27extern struct list_head pgd_list;
28
29#ifdef CONFIG_PARAVIRT
30#include <asm/paravirt.h>
31#else
32#define set_pte(ptep, pte) native_set_pte(ptep, pte)
33#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
34
35#define set_pte_atomic(ptep, pte) \
36 native_set_pte_atomic(ptep, pte)
37
38#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
39
40#ifndef __PAGETABLE_PUD_FOLDED
41#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
42#define pgd_clear(pgd) native_pgd_clear(pgd)
43#endif
44
45#ifndef set_pud
46# define set_pud(pudp, pud) native_set_pud(pudp, pud)
47#endif
48
49#ifndef __PAGETABLE_PMD_FOLDED
50#define pud_clear(pud) native_pud_clear(pud)
51#endif
52
53#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
54#define pmd_clear(pmd) native_pmd_clear(pmd)
55
56#define pte_update(mm, addr, ptep) do { } while (0)
57#define pte_update_defer(mm, addr, ptep) do { } while (0)
58
59#define pgd_val(x) native_pgd_val(x)
60#define __pgd(x) native_make_pgd(x)
61
62#ifndef __PAGETABLE_PUD_FOLDED
63#define pud_val(x) native_pud_val(x)
64#define __pud(x) native_make_pud(x)
65#endif
66
67#ifndef __PAGETABLE_PMD_FOLDED
68#define pmd_val(x) native_pmd_val(x)
69#define __pmd(x) native_make_pmd(x)
70#endif
71
72#define pte_val(x) native_pte_val(x)
73#define __pte(x) native_make_pte(x)
74
75#define arch_end_context_switch(prev) do {} while(0)
76
77#endif
78
79
80
81
82
83static inline int pte_dirty(pte_t pte)
84{
85 return pte_flags(pte) & _PAGE_DIRTY;
86}
87
88static inline int pte_young(pte_t pte)
89{
90 return pte_flags(pte) & _PAGE_ACCESSED;
91}
92
93static inline int pte_write(pte_t pte)
94{
95 return pte_flags(pte) & _PAGE_RW;
96}
97
98static inline int pte_file(pte_t pte)
99{
100 return pte_flags(pte) & _PAGE_FILE;
101}
102
103static inline int pte_huge(pte_t pte)
104{
105 return pte_flags(pte) & _PAGE_PSE;
106}
107
108static inline int pte_global(pte_t pte)
109{
110 return pte_flags(pte) & _PAGE_GLOBAL;
111}
112
113static inline int pte_exec(pte_t pte)
114{
115 return !(pte_flags(pte) & _PAGE_NX);
116}
117
118static inline int pte_special(pte_t pte)
119{
120 return pte_flags(pte) & _PAGE_SPECIAL;
121}
122
123static inline unsigned long pte_pfn(pte_t pte)
124{
125 return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
126}
127
128static inline unsigned long pmd_pfn(pmd_t pmd)
129{
130 return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
131}
132
133#define pte_page(pte) pfn_to_page(pte_pfn(pte))
134
135static inline int pmd_large(pmd_t pte)
136{
137 return (pmd_flags(pte) & (_PAGE_PSE | _PAGE_PRESENT)) ==
138 (_PAGE_PSE | _PAGE_PRESENT);
139}
140
141static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
142{
143 pteval_t v = native_pte_val(pte);
144
145 return native_make_pte(v | set);
146}
147
148static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
149{
150 pteval_t v = native_pte_val(pte);
151
152 return native_make_pte(v & ~clear);
153}
154
155static inline pte_t pte_mkclean(pte_t pte)
156{
157 return pte_clear_flags(pte, _PAGE_DIRTY);
158}
159
160static inline pte_t pte_mkold(pte_t pte)
161{
162 return pte_clear_flags(pte, _PAGE_ACCESSED);
163}
164
165static inline pte_t pte_wrprotect(pte_t pte)
166{
167 return pte_clear_flags(pte, _PAGE_RW);
168}
169
170static inline pte_t pte_mkexec(pte_t pte)
171{
172 return pte_clear_flags(pte, _PAGE_NX);
173}
174
175static inline pte_t pte_mkdirty(pte_t pte)
176{
177 return pte_set_flags(pte, _PAGE_DIRTY);
178}
179
180static inline pte_t pte_mkyoung(pte_t pte)
181{
182 return pte_set_flags(pte, _PAGE_ACCESSED);
183}
184
185static inline pte_t pte_mkwrite(pte_t pte)
186{
187 return pte_set_flags(pte, _PAGE_RW);
188}
189
190static inline pte_t pte_mkhuge(pte_t pte)
191{
192 return pte_set_flags(pte, _PAGE_PSE);
193}
194
195static inline pte_t pte_clrhuge(pte_t pte)
196{
197 return pte_clear_flags(pte, _PAGE_PSE);
198}
199
200static inline pte_t pte_mkglobal(pte_t pte)
201{
202 return pte_set_flags(pte, _PAGE_GLOBAL);
203}
204
205static inline pte_t pte_clrglobal(pte_t pte)
206{
207 return pte_clear_flags(pte, _PAGE_GLOBAL);
208}
209
210static inline pte_t pte_mkspecial(pte_t pte)
211{
212 return pte_set_flags(pte, _PAGE_SPECIAL);
213}
214
215
216
217
218
219static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
220{
221 pgprotval_t protval = pgprot_val(pgprot);
222
223 if (protval & _PAGE_PRESENT)
224 protval &= __supported_pte_mask;
225
226 return protval;
227}
228
229static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
230{
231 return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) |
232 massage_pgprot(pgprot));
233}
234
235static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
236{
237 return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) |
238 massage_pgprot(pgprot));
239}
240
241static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
242{
243 pteval_t val = pte_val(pte);
244
245
246
247
248
249 val &= _PAGE_CHG_MASK;
250 val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK;
251
252 return __pte(val);
253}
254
255
256#define pgprot_modify pgprot_modify
257static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
258{
259 pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
260 pgprotval_t addbits = pgprot_val(newprot);
261 return __pgprot(preservebits | addbits);
262}
263
264#define pte_pgprot(x) __pgprot(pte_flags(x) & PTE_FLAGS_MASK)
265
266#define canon_pgprot(p) __pgprot(massage_pgprot(p))
267
268static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
269 unsigned long flags,
270 unsigned long new_flags)
271{
272
273
274
275 if (is_ISA_range(paddr, paddr + size - 1))
276 return 1;
277
278
279
280
281
282
283
284 if ((flags == _PAGE_CACHE_UC_MINUS &&
285 new_flags == _PAGE_CACHE_WB) ||
286 (flags == _PAGE_CACHE_WC &&
287 new_flags == _PAGE_CACHE_WB)) {
288 return 0;
289 }
290
291 return 1;
292}
293
294pmd_t *populate_extra_pmd(unsigned long vaddr);
295pte_t *populate_extra_pte(unsigned long vaddr);
296#endif
297
298#ifdef CONFIG_X86_32
299# include "pgtable_32.h"
300#else
301# include "pgtable_64.h"
302#endif
303
304#ifndef __ASSEMBLY__
305#include <linux/mm_types.h>
306
307static inline int pte_none(pte_t pte)
308{
309 return !pte.pte;
310}
311
312#define __HAVE_ARCH_PTE_SAME
313static inline int pte_same(pte_t a, pte_t b)
314{
315 return a.pte == b.pte;
316}
317
318static inline int pte_present(pte_t a)
319{
320 return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
321}
322
323static inline int pte_hidden(pte_t pte)
324{
325 return pte_flags(pte) & _PAGE_HIDDEN;
326}
327
328static inline int pmd_present(pmd_t pmd)
329{
330 return pmd_flags(pmd) & _PAGE_PRESENT;
331}
332
333static inline int pmd_none(pmd_t pmd)
334{
335
336
337 return (unsigned long)native_pmd_val(pmd) == 0;
338}
339
340static inline unsigned long pmd_page_vaddr(pmd_t pmd)
341{
342 return (unsigned long)__va(pmd_val(pmd) & PTE_PFN_MASK);
343}
344
345
346
347
348
349#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
350
351
352
353
354
355
356
357static inline unsigned long pmd_index(unsigned long address)
358{
359 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
360}
361
362
363
364
365
366
367
368
369#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
370
371
372
373
374
375
376
377static inline unsigned long pte_index(unsigned long address)
378{
379 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
380}
381
382static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
383{
384 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
385}
386
387static inline int pmd_bad(pmd_t pmd)
388{
389 return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
390}
391
392static inline unsigned long pages_to_mb(unsigned long npg)
393{
394 return npg >> (20 - PAGE_SHIFT);
395}
396
397#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \
398 remap_pfn_range(vma, vaddr, pfn, size, prot)
399
400#if PAGETABLE_LEVELS > 2
401static inline int pud_none(pud_t pud)
402{
403 return native_pud_val(pud) == 0;
404}
405
406static inline int pud_present(pud_t pud)
407{
408 return pud_flags(pud) & _PAGE_PRESENT;
409}
410
411static inline unsigned long pud_page_vaddr(pud_t pud)
412{
413 return (unsigned long)__va((unsigned long)pud_val(pud) & PTE_PFN_MASK);
414}
415
416
417
418
419
420#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT)
421
422
423static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
424{
425 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
426}
427
428static inline int pud_large(pud_t pud)
429{
430 return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
431 (_PAGE_PSE | _PAGE_PRESENT);
432}
433
434static inline int pud_bad(pud_t pud)
435{
436 return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
437}
438#else
439static inline int pud_large(pud_t pud)
440{
441 return 0;
442}
443#endif
444
445#if PAGETABLE_LEVELS > 3
446static inline int pgd_present(pgd_t pgd)
447{
448 return pgd_flags(pgd) & _PAGE_PRESENT;
449}
450
451static inline unsigned long pgd_page_vaddr(pgd_t pgd)
452{
453 return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
454}
455
456
457
458
459
460#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
461
462
463static inline unsigned long pud_index(unsigned long address)
464{
465 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
466}
467
468static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
469{
470 return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address);
471}
472
473static inline int pgd_bad(pgd_t pgd)
474{
475 return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE;
476}
477
478static inline int pgd_none(pgd_t pgd)
479{
480 return !native_pgd_val(pgd);
481}
482#endif
483
484#endif
485
486
487
488
489
490
491
492#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
493
494
495
496
497
498#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address)))
499
500
501
502
503#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
504
505
506#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
507#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
508
509#ifndef __ASSEMBLY__
510
511extern int direct_gbpages;
512
513
514static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
515{
516 pte_t res = *ptep;
517
518
519 native_pte_clear(NULL, 0, ptep);
520 return res;
521}
522
523static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
524 pte_t *ptep , pte_t pte)
525{
526 native_set_pte(ptep, pte);
527}
528
529#ifndef CONFIG_PARAVIRT
530
531
532
533
534
535
536
537
538
539
540
541
542
543#define pte_update(mm, addr, ptep) do { } while (0)
544#define pte_update_defer(mm, addr, ptep) do { } while (0)
545#endif
546
547
548
549
550
551
552
553
554struct vm_area_struct;
555
556#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
557extern int ptep_set_access_flags(struct vm_area_struct *vma,
558 unsigned long address, pte_t *ptep,
559 pte_t entry, int dirty);
560
561#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
562extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
563 unsigned long addr, pte_t *ptep);
564
565#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
566extern int ptep_clear_flush_young(struct vm_area_struct *vma,
567 unsigned long address, pte_t *ptep);
568
569#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
570static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
571 pte_t *ptep)
572{
573 pte_t pte = native_ptep_get_and_clear(ptep);
574 pte_update(mm, addr, ptep);
575 return pte;
576}
577
578#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
579static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
580 unsigned long addr, pte_t *ptep,
581 int full)
582{
583 pte_t pte;
584 if (full) {
585
586
587
588
589 pte = native_local_ptep_get_and_clear(ptep);
590 } else {
591 pte = ptep_get_and_clear(mm, addr, ptep);
592 }
593 return pte;
594}
595
596#define __HAVE_ARCH_PTEP_SET_WRPROTECT
597static inline void ptep_set_wrprotect(struct mm_struct *mm,
598 unsigned long addr, pte_t *ptep)
599{
600 clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
601 pte_update(mm, addr, ptep);
602}
603
604
605
606
607
608
609
610
611
612
613
614static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
615{
616 memcpy(dst, src, count * sizeof(pgd_t));
617}
618
619
620#include <asm-generic/pgtable.h>
621#endif
622
623#endif
624