1
2#ifndef _ASM_X86_PGTABLE_3LEVEL_H
3#define _ASM_X86_PGTABLE_3LEVEL_H
4
5#include <asm/atomic64_32.h>
6
7
8
9
10
11
12
13
14#define pte_ERROR(e) \
15 pr_err("%s:%d: bad pte %p(%08lx%08lx)\n", \
16 __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low)
17#define pmd_ERROR(e) \
18 pr_err("%s:%d: bad pmd %p(%016Lx)\n", \
19 __FILE__, __LINE__, &(e), pmd_val(e))
20#define pgd_ERROR(e) \
21 pr_err("%s:%d: bad pgd %p(%016Lx)\n", \
22 __FILE__, __LINE__, &(e), pgd_val(e))
23
24
25
26
27
28
29
30static inline void native_set_pte(pte_t *ptep, pte_t pte)
31{
32 ptep->pte_high = pte.pte_high;
33 smp_wmb();
34 ptep->pte_low = pte.pte_low;
35}
36
37#define pmd_read_atomic pmd_read_atomic
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
76{
77 pmdval_t ret;
78 u32 *tmp = (u32 *)pmdp;
79
80 ret = (pmdval_t) (*tmp);
81 if (ret) {
82
83
84
85
86 smp_rmb();
87 ret |= ((pmdval_t)*(tmp + 1)) << 32;
88 }
89
90 return (pmd_t) { ret };
91}
92
93static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
94{
95 set_64bit((unsigned long long *)(ptep), native_pte_val(pte));
96}
97
98static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
99{
100 set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd));
101}
102
103static inline void native_set_pud(pud_t *pudp, pud_t pud)
104{
105#ifdef CONFIG_PAGE_TABLE_ISOLATION
106 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd);
107#endif
108 set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
109}
110
111
112
113
114
115
116static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
117 pte_t *ptep)
118{
119 ptep->pte_low = 0;
120 smp_wmb();
121 ptep->pte_high = 0;
122}
123
124static inline void native_pmd_clear(pmd_t *pmd)
125{
126 u32 *tmp = (u32 *)pmd;
127 *tmp = 0;
128 smp_wmb();
129 *(tmp + 1) = 0;
130}
131
132static inline void native_pud_clear(pud_t *pudp)
133{
134}
135
136static inline void pud_clear(pud_t *pudp)
137{
138 set_pud(pudp, __pud(0));
139
140
141
142
143
144
145
146
147
148
149
150}
151
152#ifdef CONFIG_SMP
153static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
154{
155 pte_t res;
156
157 res.pte = (pteval_t)arch_atomic64_xchg((atomic64_t *)ptep, 0);
158
159 return res;
160}
161#else
162#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
163#endif
164
165union split_pmd {
166 struct {
167 u32 pmd_low;
168 u32 pmd_high;
169 };
170 pmd_t pmd;
171};
172
173#ifdef CONFIG_SMP
174static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
175{
176 union split_pmd res, *orig = (union split_pmd *)pmdp;
177
178
179 res.pmd_low = xchg(&orig->pmd_low, 0);
180 res.pmd_high = orig->pmd_high;
181 orig->pmd_high = 0;
182
183 return res.pmd;
184}
185#else
186#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
187#endif
188
189#ifndef pmdp_establish
190#define pmdp_establish pmdp_establish
191static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
192 unsigned long address, pmd_t *pmdp, pmd_t pmd)
193{
194 pmd_t old;
195
196
197
198
199
200
201 if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
202 union split_pmd old, new, *ptr;
203
204 ptr = (union split_pmd *)pmdp;
205
206 new.pmd = pmd;
207
208
209 old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);
210 old.pmd_high = ptr->pmd_high;
211 ptr->pmd_high = new.pmd_high;
212 return old.pmd;
213 }
214
215 do {
216 old = *pmdp;
217 } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
218
219 return old;
220}
221#endif
222
223#ifdef CONFIG_SMP
224union split_pud {
225 struct {
226 u32 pud_low;
227 u32 pud_high;
228 };
229 pud_t pud;
230};
231
232static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
233{
234 union split_pud res, *orig = (union split_pud *)pudp;
235
236#ifdef CONFIG_PAGE_TABLE_ISOLATION
237 pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0));
238#endif
239
240
241 res.pud_low = xchg(&orig->pud_low, 0);
242 res.pud_high = orig->pud_high;
243 orig->pud_high = 0;
244
245 return res.pud;
246}
247#else
248#define native_pudp_get_and_clear(xp) native_local_pudp_get_and_clear(xp)
249#endif
250
251
252#define SWP_TYPE_BITS 5
253
254#define SWP_OFFSET_FIRST_BIT (_PAGE_BIT_PROTNONE + 1)
255
256
257#define SWP_OFFSET_SHIFT (SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS)
258
259#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5)
260#define __swp_type(x) (((x).val) & 0x1f)
261#define __swp_offset(x) ((x).val >> 5)
262#define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << 5})
263
264
265
266
267
268
269
270
271
272#define __swp_pteval_entry(type, offset) ((pteval_t) { \
273 (~(pteval_t)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
274 | ((pteval_t)(type) << (64 - SWP_TYPE_BITS)) })
275
276#define __swp_entry_to_pte(x) ((pte_t){ .pte = \
277 __swp_pteval_entry(__swp_type(x), __swp_offset(x)) })
278
279
280
281
282
283
284#define __pteval_swp_type(x) ((unsigned long)((x).pte >> (64 - SWP_TYPE_BITS)))
285#define __pteval_swp_offset(x) ((unsigned long)(~((x).pte) << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT))
286
287#define __pte_to_swp_entry(pte) (__swp_entry(__pteval_swp_type(pte), \
288 __pteval_swp_offset(pte)))
289
290#include <asm/pgtable-invert.h>
291
292#endif
293