1#ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H
2#define _ASM_POWERPC_BOOK3S_64_HASH_64K_H
3
4#define H_PTE_INDEX_SIZE 8
5#define H_PMD_INDEX_SIZE 5
6#define H_PUD_INDEX_SIZE 5
7#define H_PGD_INDEX_SIZE 12
8
9#define H_PAGE_COMBO 0x00001000
10#define H_PAGE_4K_PFN 0x00002000
11
12
13
14
15#define H_PAGE_THP_HUGE H_PAGE_4K_PFN
16
17
18
19
20
21#define H_PAGE_COMBO_VALID (H_PAGE_F_GIX | H_PAGE_F_SECOND)
22
23
24#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \
25 H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO)
26
27
28
29#define H_PTE_FRAG_NR 16
30
31
32
33
34#define H_PTE_FRAG_SIZE_SHIFT 12
35#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
36
37#ifndef __ASSEMBLY__
38#include <asm/errno.h>
39
40
41
42
43
44
45
46#define __real_pte __real_pte
47static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
48{
49 real_pte_t rpte;
50 unsigned long *hidxp;
51
52 rpte.pte = pte;
53 rpte.hidx = 0;
54 if (pte_val(pte) & H_PAGE_COMBO) {
55
56
57
58
59 smp_rmb();
60 hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
61 rpte.hidx = *hidxp;
62 }
63 return rpte;
64}
65
66static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
67{
68 if ((pte_val(rpte.pte) & H_PAGE_COMBO))
69 return (rpte.hidx >> (index<<2)) & 0xf;
70 return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf;
71}
72
73#define __rpte_to_pte(r) ((r).pte)
74extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index);
75
76
77
78
79#define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \
80 do { \
81 unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \
82 unsigned __split = (psize == MMU_PAGE_4K || \
83 psize == MMU_PAGE_64K_AP); \
84 shift = mmu_psize_defs[psize].shift; \
85 for (index = 0; vpn < __end; index++, \
86 vpn += (1L << (shift - VPN_SHIFT))) { \
87 if (!__split || __rpte_sub_valid(rpte, index)) \
88 do {
89
90#define pte_iterate_hashed_end() } while(0); } } while(0)
91
92#define pte_pagesize_index(mm, addr, pte) \
93 (((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K)
94
95extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
96 unsigned long pfn, unsigned long size, pgprot_t);
97static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
98 unsigned long pfn, pgprot_t prot)
99{
100 if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) {
101 WARN(1, "remap_4k_pfn called with wrong pfn value\n");
102 return -EINVAL;
103 }
104 return remap_pfn_range(vma, addr, pfn, PAGE_SIZE,
105 __pgprot(pgprot_val(prot) | H_PAGE_4K_PFN));
106}
107
108#define H_PTE_TABLE_SIZE PTE_FRAG_SIZE
109#ifdef CONFIG_TRANSPARENT_HUGEPAGE
110#define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \
111 (sizeof(unsigned long) << PMD_INDEX_SIZE))
112#else
113#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
114#endif
115#define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
116#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
117
118#ifdef CONFIG_TRANSPARENT_HUGEPAGE
119static inline char *get_hpte_slot_array(pmd_t *pmdp)
120{
121
122
123
124
125
126
127 smp_rmb();
128 return *(char **)(pmdp + PTRS_PER_PMD);
129
130
131}
132
133
134
135
136
137
138
139
140
141
142
143
144static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
145{
146 return hpte_slot_array[index] & 0x1;
147}
148
149static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
150 int index)
151{
152 return hpte_slot_array[index] >> 1;
153}
154
155static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
156 unsigned int index, unsigned int hidx)
157{
158 hpte_slot_array[index] = (hidx << 1) | 0x1;
159}
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177static inline int hash__pmd_trans_huge(pmd_t pmd)
178{
179 return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) ==
180 (_PAGE_PTE | H_PAGE_THP_HUGE));
181}
182
183static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
184{
185 return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0);
186}
187
188static inline pmd_t hash__pmd_mkhuge(pmd_t pmd)
189{
190 return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE));
191}
192
193extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm,
194 unsigned long addr, pmd_t *pmdp,
195 unsigned long clr, unsigned long set);
196extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
197 unsigned long address, pmd_t *pmdp);
198extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
199 pgtable_t pgtable);
200extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
201extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
202 unsigned long address, pmd_t *pmdp);
203extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
204 unsigned long addr, pmd_t *pmdp);
205extern int hash__has_transparent_hugepage(void);
206#endif
207#endif
208
209#endif
210