1#ifndef _ASM_POWERPC_PGTABLE_PPC64_H_
2#define _ASM_POWERPC_PGTABLE_PPC64_H_
3
4
5
6
7
8#ifdef CONFIG_PPC_64K_PAGES
9#include <asm/pgtable-ppc64-64k.h>
10#else
11#include <asm/pgtable-ppc64-4k.h>
12#endif
13#include <asm/barrier.h>
14
15#define FIRST_USER_ADDRESS 0
16
17
18
19
20#define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \
21 PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT)
22#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE)
23
24#ifdef CONFIG_TRANSPARENT_HUGEPAGE
25#define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1)
26#else
27#define PMD_CACHE_INDEX PMD_INDEX_SIZE
28#endif
29
30
31
32
33#ifdef CONFIG_PPC_BOOK3E
34#define KERN_VIRT_START ASM_CONST(0x8000000000000000)
35#else
36#define KERN_VIRT_START ASM_CONST(0xD000000000000000)
37#endif
38#define KERN_VIRT_SIZE ASM_CONST(0x0000100000000000)
39
40
41
42
43
44
45#define VMALLOC_START KERN_VIRT_START
46#ifdef CONFIG_PPC_BOOK3E
47#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 2)
48#else
49#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
50#endif
51#define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE)
52
53
54
55
56
57
58
59
60
61
62#define KERN_IO_START (KERN_VIRT_START + (KERN_VIRT_SIZE >> 1))
63#define FULL_IO_SIZE 0x80000000ul
64#define ISA_IO_BASE (KERN_IO_START)
65#define ISA_IO_END (KERN_IO_START + 0x10000ul)
66#define PHB_IO_BASE (ISA_IO_END)
67#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
68#define IOREMAP_BASE (PHB_IO_END)
69#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE)
70
71
72
73
74
75#define REGION_SHIFT 60UL
76#define REGION_MASK (0xfUL << REGION_SHIFT)
77#define REGION_ID(ea) (((unsigned long)(ea)) >> REGION_SHIFT)
78
79#define VMALLOC_REGION_ID (REGION_ID(VMALLOC_START))
80#define KERNEL_REGION_ID (REGION_ID(PAGE_OFFSET))
81#define VMEMMAP_REGION_ID (0xfUL)
82#define USER_REGION_ID (0UL)
83
84
85
86
87
88#ifdef CONFIG_PPC_BOOK3E
89#define VMEMMAP_BASE VMALLOC_END
90#define VMEMMAP_END KERN_IO_START
91#else
92#define VMEMMAP_BASE (VMEMMAP_REGION_ID << REGION_SHIFT)
93#endif
94#define vmemmap ((struct page *)VMEMMAP_BASE)
95
96
97
98
99
100#ifdef CONFIG_PPC_BOOK3S
101#include <asm/pte-hash64.h>
102#else
103#include <asm/pte-book3e.h>
104#endif
105#include <asm/pte-common.h>
106
107#ifdef CONFIG_PPC_MM_SLICES
108#define HAVE_ARCH_UNMAPPED_AREA
109#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
110#endif
111
112#ifndef __ASSEMBLY__
113
114
115
116
117
118
119#ifndef __real_pte
120
121#ifdef STRICT_MM_TYPECHECKS
122#define __real_pte(e,p) ((real_pte_t){(e)})
123#define __rpte_to_pte(r) ((r).pte)
124#else
125#define __real_pte(e,p) (e)
126#define __rpte_to_pte(r) (__pte(r))
127#endif
128#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> 12)
129
130#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \
131 do { \
132 index = 0; \
133 shift = mmu_psize_defs[psize].shift; \
134
135#define pte_iterate_hashed_end() } while(0)
136
137#ifdef CONFIG_PPC_HAS_HASH_64K
138#define pte_pagesize_index(mm, addr, pte) get_slice_psize(mm, addr)
139#else
140#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K
141#endif
142
143#endif
144
145
146
147
148#define PMD_BAD_BITS (PTE_TABLE_SIZE-1)
149#define PUD_BAD_BITS (PMD_TABLE_SIZE-1)
150
151#define pmd_set(pmdp, pmdval) (pmd_val(*(pmdp)) = (pmdval))
152#define pmd_none(pmd) (!pmd_val(pmd))
153#define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \
154 || (pmd_val(pmd) & PMD_BAD_BITS))
155#define pmd_present(pmd) (pmd_val(pmd) != 0)
156#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0)
157#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS)
158extern struct page *pmd_page(pmd_t pmd);
159
160#define pud_set(pudp, pudval) (pud_val(*(pudp)) = (pudval))
161#define pud_none(pud) (!pud_val(pud))
162#define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \
163 || (pud_val(pud) & PUD_BAD_BITS))
164#define pud_present(pud) (pud_val(pud) != 0)
165#define pud_clear(pudp) (pud_val(*(pudp)) = 0)
166#define pud_page_vaddr(pud) (pud_val(pud) & ~PUD_MASKED_BITS)
167#define pud_page(pud) virt_to_page(pud_page_vaddr(pud))
168
169#define pgd_set(pgdp, pudp) ({pgd_val(*(pgdp)) = (unsigned long)(pudp);})
170
171
172
173
174
175#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
176
177#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
178
179#define pmd_offset(pudp,addr) \
180 (((pmd_t *) pud_page_vaddr(*(pudp))) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)))
181
182#define pte_offset_kernel(dir,addr) \
183 (((pte_t *) pmd_page_vaddr(*(dir))) + (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)))
184
185#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
186#define pte_unmap(pte) do { } while(0)
187
188
189
190#define pgd_offset_k(address) pgd_offset(&init_mm, address)
191extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
192 pte_t *ptep, unsigned long pte, int huge);
193
194
195static inline unsigned long pte_update(struct mm_struct *mm,
196 unsigned long addr,
197 pte_t *ptep, unsigned long clr,
198 int huge)
199{
200#ifdef PTE_ATOMIC_UPDATES
201 unsigned long old, tmp;
202
203 __asm__ __volatile__(
204 "1: ldarx %0,0,%3 # pte_update\n\
205 andi. %1,%0,%6\n\
206 bne- 1b \n\
207 andc %1,%0,%4 \n\
208 stdcx. %1,0,%3 \n\
209 bne- 1b"
210 : "=&r" (old), "=&r" (tmp), "=m" (*ptep)
211 : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY)
212 : "cc" );
213#else
214 unsigned long old = pte_val(*ptep);
215 *ptep = __pte(old & ~clr);
216#endif
217
218 if (!huge)
219 assert_pte_locked(mm, addr);
220
221#ifdef CONFIG_PPC_STD_MMU_64
222 if (old & _PAGE_HASHPTE)
223 hpte_need_flush(mm, addr, ptep, old, huge);
224#endif
225
226 return old;
227}
228
229static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
230 unsigned long addr, pte_t *ptep)
231{
232 unsigned long old;
233
234 if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
235 return 0;
236 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0);
237 return (old & _PAGE_ACCESSED) != 0;
238}
239#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
240#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
241({ \
242 int __r; \
243 __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
244 __r; \
245})
246
247#define __HAVE_ARCH_PTEP_SET_WRPROTECT
248static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
249 pte_t *ptep)
250{
251
252 if ((pte_val(*ptep) & _PAGE_RW) == 0)
253 return;
254
255 pte_update(mm, addr, ptep, _PAGE_RW, 0);
256}
257
258static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
259 unsigned long addr, pte_t *ptep)
260{
261 if ((pte_val(*ptep) & _PAGE_RW) == 0)
262 return;
263
264 pte_update(mm, addr, ptep, _PAGE_RW, 1);
265}
266
267
268
269
270
271
272
273
274
275#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
276#define ptep_clear_flush_young(__vma, __address, __ptep) \
277({ \
278 int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \
279 __ptep); \
280 __young; \
281})
282
283#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
284static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
285 unsigned long addr, pte_t *ptep)
286{
287 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0);
288 return __pte(old);
289}
290
291static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
292 pte_t * ptep)
293{
294 pte_update(mm, addr, ptep, ~0UL, 0);
295}
296
297
298
299
300
301static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry)
302{
303 unsigned long bits = pte_val(entry) &
304 (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
305
306#ifdef PTE_ATOMIC_UPDATES
307 unsigned long old, tmp;
308
309 __asm__ __volatile__(
310 "1: ldarx %0,0,%4\n\
311 andi. %1,%0,%6\n\
312 bne- 1b \n\
313 or %0,%3,%0\n\
314 stdcx. %0,0,%4\n\
315 bne- 1b"
316 :"=&r" (old), "=&r" (tmp), "=m" (*ptep)
317 :"r" (bits), "r" (ptep), "m" (*ptep), "i" (_PAGE_BUSY)
318 :"cc");
319#else
320 unsigned long old = pte_val(*ptep);
321 *ptep = __pte(old | bits);
322#endif
323}
324
325#define __HAVE_ARCH_PTE_SAME
326#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)
327
328#define pte_ERROR(e) \
329 printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
330#define pmd_ERROR(e) \
331 printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
332#define pgd_ERROR(e) \
333 printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
334
335
336#define __swp_type(entry) (((entry).val >> 1) & 0x3f)
337#define __swp_offset(entry) ((entry).val >> 8)
338#define __swp_entry(type, offset) ((swp_entry_t){((type)<< 1)|((offset)<<8)})
339#define __pte_to_swp_entry(pte) ((swp_entry_t){pte_val(pte) >> PTE_RPN_SHIFT})
340#define __swp_entry_to_pte(x) ((pte_t) { (x).val << PTE_RPN_SHIFT })
341#define pte_to_pgoff(pte) (pte_val(pte) >> PTE_RPN_SHIFT)
342#define pgoff_to_pte(off) ((pte_t) {((off) << PTE_RPN_SHIFT)|_PAGE_FILE})
343#define PTE_FILE_MAX_BITS (BITS_PER_LONG - PTE_RPN_SHIFT)
344
345void pgtable_cache_add(unsigned shift, void (*ctor)(void *));
346void pgtable_cache_init(void);
347#endif
348
349
350
351
352#define _PAGE_SPLITTING _PAGE_SPECIAL
353
354
355
356
357
358#define _PAGE_THP_HUGE _PAGE_4K_PFN
359
360
361
362
363#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | \
364 _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPLITTING | \
365 _PAGE_THP_HUGE)
366
367#ifndef __ASSEMBLY__
368
369
370
371
372
373
374
375
376
377
378
379
380static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
381{
382 return (hpte_slot_array[index] >> 3) & 0x1;
383}
384
385static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
386 int index)
387{
388 return hpte_slot_array[index] >> 4;
389}
390
391static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
392 unsigned int index, unsigned int hidx)
393{
394 hpte_slot_array[index] = hidx << 4 | 0x1 << 3;
395}
396
397static inline char *get_hpte_slot_array(pmd_t *pmdp)
398{
399
400
401
402
403
404
405 smp_rmb();
406 return *(char **)(pmdp + PTRS_PER_PMD);
407
408
409}
410
411extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
412 pmd_t *pmdp);
413#ifdef CONFIG_TRANSPARENT_HUGEPAGE
414extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
415extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
416extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
417extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
418 pmd_t *pmdp, pmd_t pmd);
419extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
420 pmd_t *pmd);
421
422static inline int pmd_trans_huge(pmd_t pmd)
423{
424
425
426
427 return (pmd_val(pmd) & 0x3) && (pmd_val(pmd) & _PAGE_THP_HUGE);
428}
429
430static inline int pmd_large(pmd_t pmd)
431{
432
433
434
435 if (pmd_trans_huge(pmd))
436 return pmd_val(pmd) & _PAGE_PRESENT;
437 return 0;
438}
439
440static inline int pmd_trans_splitting(pmd_t pmd)
441{
442 if (pmd_trans_huge(pmd))
443 return pmd_val(pmd) & _PAGE_SPLITTING;
444 return 0;
445}
446
447extern int has_transparent_hugepage(void);
448#endif
449
450static inline pte_t pmd_pte(pmd_t pmd)
451{
452 return __pte(pmd_val(pmd));
453}
454
455static inline pmd_t pte_pmd(pte_t pte)
456{
457 return __pmd(pte_val(pte));
458}
459
460static inline pte_t *pmdp_ptep(pmd_t *pmd)
461{
462 return (pte_t *)pmd;
463}
464
465#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
466#define pmd_young(pmd) pte_young(pmd_pte(pmd))
467#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
468#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
469#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
470#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
471#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
472
473#define __HAVE_ARCH_PMD_WRITE
474#define pmd_write(pmd) pte_write(pmd_pte(pmd))
475
476static inline pmd_t pmd_mkhuge(pmd_t pmd)
477{
478
479 return pmd;
480}
481
482static inline pmd_t pmd_mknotpresent(pmd_t pmd)
483{
484 pmd_val(pmd) &= ~_PAGE_PRESENT;
485 return pmd;
486}
487
488static inline pmd_t pmd_mksplitting(pmd_t pmd)
489{
490 pmd_val(pmd) |= _PAGE_SPLITTING;
491 return pmd;
492}
493
494#define __HAVE_ARCH_PMD_SAME
495static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
496{
497 return (((pmd_val(pmd_a) ^ pmd_val(pmd_b)) & ~_PAGE_HPTEFLAGS) == 0);
498}
499
500#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
501extern int pmdp_set_access_flags(struct vm_area_struct *vma,
502 unsigned long address, pmd_t *pmdp,
503 pmd_t entry, int dirty);
504
505extern unsigned long pmd_hugepage_update(struct mm_struct *mm,
506 unsigned long addr,
507 pmd_t *pmdp, unsigned long clr);
508
509static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
510 unsigned long addr, pmd_t *pmdp)
511{
512 unsigned long old;
513
514 if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
515 return 0;
516 old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED);
517 return ((old & _PAGE_ACCESSED) != 0);
518}
519
520#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
521extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
522 unsigned long address, pmd_t *pmdp);
523#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
524extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
525 unsigned long address, pmd_t *pmdp);
526
527#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
528extern pmd_t pmdp_get_and_clear(struct mm_struct *mm,
529 unsigned long addr, pmd_t *pmdp);
530
531#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
532extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address,
533 pmd_t *pmdp);
534
535#define __HAVE_ARCH_PMDP_SET_WRPROTECT
536static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
537 pmd_t *pmdp)
538{
539
540 if ((pmd_val(*pmdp) & _PAGE_RW) == 0)
541 return;
542
543 pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW);
544}
545
546#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
547extern void pmdp_splitting_flush(struct vm_area_struct *vma,
548 unsigned long address, pmd_t *pmdp);
549
550#define __HAVE_ARCH_PGTABLE_DEPOSIT
551extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
552 pgtable_t pgtable);
553#define __HAVE_ARCH_PGTABLE_WITHDRAW
554extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
555
556#define __HAVE_ARCH_PMDP_INVALIDATE
557extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
558 pmd_t *pmdp);
559#endif
560#endif
561