1#ifndef _ASM_GENERIC_PGTABLE_H
2#define _ASM_GENERIC_PGTABLE_H
3
4#ifndef __ASSEMBLY__
5#ifdef CONFIG_MMU
6
7#include <linux/mm_types.h>
8#include <linux/bug.h>
9
10
11
12
13
14
15
16#ifndef USER_PGTABLES_CEILING
17#define USER_PGTABLES_CEILING 0UL
18#endif
19
20#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
21extern int ptep_set_access_flags(struct vm_area_struct *vma,
22 unsigned long address, pte_t *ptep,
23 pte_t entry, int dirty);
24#endif
25
26#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
27extern int pmdp_set_access_flags(struct vm_area_struct *vma,
28 unsigned long address, pmd_t *pmdp,
29 pmd_t entry, int dirty);
30#endif
31
32#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
33static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
34 unsigned long address,
35 pte_t *ptep)
36{
37 pte_t pte = *ptep;
38 int r = 1;
39 if (!pte_young(pte))
40 r = 0;
41 else
42 set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
43 return r;
44}
45#endif
46
47#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
48#ifdef CONFIG_TRANSPARENT_HUGEPAGE
49static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
50 unsigned long address,
51 pmd_t *pmdp)
52{
53 pmd_t pmd = *pmdp;
54 int r = 1;
55 if (!pmd_young(pmd))
56 r = 0;
57 else
58 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
59 return r;
60}
61#else
62static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
63 unsigned long address,
64 pmd_t *pmdp)
65{
66 BUG();
67 return 0;
68}
69#endif
70#endif
71
72#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
73int ptep_clear_flush_young(struct vm_area_struct *vma,
74 unsigned long address, pte_t *ptep);
75#endif
76
77#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
78int pmdp_clear_flush_young(struct vm_area_struct *vma,
79 unsigned long address, pmd_t *pmdp);
80#endif
81
82#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
83static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
84 unsigned long address,
85 pte_t *ptep)
86{
87 pte_t pte = *ptep;
88 pte_clear(mm, address, ptep);
89 return pte;
90}
91#endif
92
93#ifndef __HAVE_ARCH_PMDP_GET_AND_CLEAR
94#ifdef CONFIG_TRANSPARENT_HUGEPAGE
95static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
96 unsigned long address,
97 pmd_t *pmdp)
98{
99 pmd_t pmd = *pmdp;
100 pmd_clear(pmdp);
101 return pmd;
102}
103#endif
104#endif
105
106#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
107static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
108 unsigned long address, pte_t *ptep,
109 int full)
110{
111 pte_t pte;
112 pte = ptep_get_and_clear(mm, address, ptep);
113 return pte;
114}
115#endif
116
117
118
119
120
121
122#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
123static inline void pte_clear_not_present_full(struct mm_struct *mm,
124 unsigned long address,
125 pte_t *ptep,
126 int full)
127{
128 pte_clear(mm, address, ptep);
129}
130#endif
131
132#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
133extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
134 unsigned long address,
135 pte_t *ptep);
136#endif
137
138#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH
139extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
140 unsigned long address,
141 pmd_t *pmdp);
142#endif
143
144#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
145struct mm_struct;
146static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
147{
148 pte_t old_pte = *ptep;
149 set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
150}
151#endif
152
153#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
154#ifdef CONFIG_TRANSPARENT_HUGEPAGE
155static inline void pmdp_set_wrprotect(struct mm_struct *mm,
156 unsigned long address, pmd_t *pmdp)
157{
158 pmd_t old_pmd = *pmdp;
159 set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
160}
161#else
162static inline void pmdp_set_wrprotect(struct mm_struct *mm,
163 unsigned long address, pmd_t *pmdp)
164{
165 BUG();
166}
167#endif
168#endif
169
170#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
171extern void pmdp_splitting_flush(struct vm_area_struct *vma,
172 unsigned long address, pmd_t *pmdp);
173#endif
174
175#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
176extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
177 pgtable_t pgtable);
178#endif
179
180#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
181extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
182#endif
183
184#ifndef __HAVE_ARCH_PMDP_INVALIDATE
185extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
186 pmd_t *pmdp);
187#endif
188
189#ifndef __HAVE_ARCH_PTE_SAME
190static inline int pte_same(pte_t pte_a, pte_t pte_b)
191{
192 return pte_val(pte_a) == pte_val(pte_b);
193}
194#endif
195
196#ifndef __HAVE_ARCH_PMD_SAME
197#ifdef CONFIG_TRANSPARENT_HUGEPAGE
198static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
199{
200 return pmd_val(pmd_a) == pmd_val(pmd_b);
201}
202#else
203static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
204{
205 BUG();
206 return 0;
207}
208#endif
209#endif
210
211#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
212#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
213#endif
214
215#ifndef __HAVE_ARCH_MOVE_PTE
216#define move_pte(pte, prot, old_addr, new_addr) (pte)
217#endif
218
219#ifndef pte_accessible
220# define pte_accessible(pte) ((void)(pte),1)
221#endif
222
223#ifndef flush_tlb_fix_spurious_fault
224#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address)
225#endif
226
227#ifndef pgprot_noncached
228#define pgprot_noncached(prot) (prot)
229#endif
230
231#ifndef pgprot_writecombine
232#define pgprot_writecombine pgprot_noncached
233#endif
234
235
236
237
238
239
240
241#define pgd_addr_end(addr, end) \
242({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
243 (__boundary - 1 < (end) - 1)? __boundary: (end); \
244})
245
246#ifndef pud_addr_end
247#define pud_addr_end(addr, end) \
248({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
249 (__boundary - 1 < (end) - 1)? __boundary: (end); \
250})
251#endif
252
253#ifndef pmd_addr_end
254#define pmd_addr_end(addr, end) \
255({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
256 (__boundary - 1 < (end) - 1)? __boundary: (end); \
257})
258#endif
259
260
261
262
263
264
265void pgd_clear_bad(pgd_t *);
266void pud_clear_bad(pud_t *);
267void pmd_clear_bad(pmd_t *);
268
269static inline int pgd_none_or_clear_bad(pgd_t *pgd)
270{
271 if (pgd_none(*pgd))
272 return 1;
273 if (unlikely(pgd_bad(*pgd))) {
274 pgd_clear_bad(pgd);
275 return 1;
276 }
277 return 0;
278}
279
280static inline int pud_none_or_clear_bad(pud_t *pud)
281{
282 if (pud_none(*pud))
283 return 1;
284 if (unlikely(pud_bad(*pud))) {
285 pud_clear_bad(pud);
286 return 1;
287 }
288 return 0;
289}
290
291static inline int pmd_none_or_clear_bad(pmd_t *pmd)
292{
293 if (pmd_none(*pmd))
294 return 1;
295 if (unlikely(pmd_bad(*pmd))) {
296 pmd_clear_bad(pmd);
297 return 1;
298 }
299 return 0;
300}
301
302static inline pte_t __ptep_modify_prot_start(struct mm_struct *mm,
303 unsigned long addr,
304 pte_t *ptep)
305{
306
307
308
309
310
311 return ptep_get_and_clear(mm, addr, ptep);
312}
313
314static inline void __ptep_modify_prot_commit(struct mm_struct *mm,
315 unsigned long addr,
316 pte_t *ptep, pte_t pte)
317{
318
319
320
321
322 set_pte_at(mm, addr, ptep, pte);
323}
324
325#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
341 unsigned long addr,
342 pte_t *ptep)
343{
344 return __ptep_modify_prot_start(mm, addr, ptep);
345}
346
347
348
349
350
351static inline void ptep_modify_prot_commit(struct mm_struct *mm,
352 unsigned long addr,
353 pte_t *ptep, pte_t pte)
354{
355 __ptep_modify_prot_commit(mm, addr, ptep, pte);
356}
357#endif
358#endif
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
376#define arch_enter_lazy_mmu_mode() do {} while (0)
377#define arch_leave_lazy_mmu_mode() do {} while (0)
378#define arch_flush_lazy_mmu_mode() do {} while (0)
379#endif
380
381
382
383
384
385
386
387
388
389
390
391
392#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
393#define arch_start_context_switch(prev) do {} while (0)
394#endif
395
396#ifndef CONFIG_HAVE_ARCH_SOFT_DIRTY
397static inline int pte_soft_dirty(pte_t pte)
398{
399 return 0;
400}
401
402static inline int pmd_soft_dirty(pmd_t pmd)
403{
404 return 0;
405}
406
407static inline pte_t pte_mksoft_dirty(pte_t pte)
408{
409 return pte;
410}
411
412static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
413{
414 return pmd;
415}
416
417static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
418{
419 return pte;
420}
421
422static inline int pte_swp_soft_dirty(pte_t pte)
423{
424 return 0;
425}
426
427static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
428{
429 return pte;
430}
431
432static inline pte_t pte_file_clear_soft_dirty(pte_t pte)
433{
434 return pte;
435}
436
437static inline pte_t pte_file_mksoft_dirty(pte_t pte)
438{
439 return pte;
440}
441
442static inline int pte_file_soft_dirty(pte_t pte)
443{
444 return 0;
445}
446#endif
447
448#ifndef __HAVE_PFNMAP_TRACKING
449
450
451
452
453
454
455
456
457
458
459static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
460 unsigned long pfn, unsigned long addr,
461 unsigned long size)
462{
463 return 0;
464}
465
466
467
468
469
470static inline int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
471 unsigned long pfn)
472{
473 return 0;
474}
475
476
477
478
479
480static inline int track_pfn_copy(struct vm_area_struct *vma)
481{
482 return 0;
483}
484
485
486
487
488
489
490static inline void untrack_pfn(struct vm_area_struct *vma,
491 unsigned long pfn, unsigned long size)
492{
493}
494#else
495extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
496 unsigned long pfn, unsigned long addr,
497 unsigned long size);
498extern int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
499 unsigned long pfn);
500extern int track_pfn_copy(struct vm_area_struct *vma);
501extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
502 unsigned long size);
503#endif
504
505#ifdef __HAVE_COLOR_ZERO_PAGE
506static inline int is_zero_pfn(unsigned long pfn)
507{
508 extern unsigned long zero_pfn;
509 unsigned long offset_from_zero_pfn = pfn - zero_pfn;
510 return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
511}
512
513#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
514
515#else
516static inline int is_zero_pfn(unsigned long pfn)
517{
518 extern unsigned long zero_pfn;
519 return pfn == zero_pfn;
520}
521
522static inline unsigned long my_zero_pfn(unsigned long addr)
523{
524 extern unsigned long zero_pfn;
525 return zero_pfn;
526}
527#endif
528
529#ifdef CONFIG_MMU
530
531#ifndef CONFIG_TRANSPARENT_HUGEPAGE
532static inline int pmd_trans_huge(pmd_t pmd)
533{
534 return 0;
535}
536static inline int pmd_trans_splitting(pmd_t pmd)
537{
538 return 0;
539}
540#ifndef __HAVE_ARCH_PMD_WRITE
541static inline int pmd_write(pmd_t pmd)
542{
543 BUG();
544 return 0;
545}
546#endif
547#endif
548
549#ifndef pmd_read_atomic
550static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
551{
552
553
554
555
556
557 return *pmdp;
558}
559#endif
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
583{
584 pmd_t pmdval = pmd_read_atomic(pmd);
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599#ifdef CONFIG_TRANSPARENT_HUGEPAGE
600 barrier();
601#endif
602 if (pmd_none(pmdval))
603 return 1;
604 if (unlikely(pmd_bad(pmdval))) {
605 if (!pmd_trans_huge(pmdval))
606 pmd_clear_bad(pmd);
607 return 1;
608 }
609 return 0;
610}
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625static inline int pmd_trans_unstable(pmd_t *pmd)
626{
627#ifdef CONFIG_TRANSPARENT_HUGEPAGE
628 return pmd_none_or_trans_huge_or_clear_bad(pmd);
629#else
630 return 0;
631#endif
632}
633
634#ifdef CONFIG_NUMA_BALANCING
635#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
636
637
638
639
640
641
642
643
644
645#ifndef pte_numa
646static inline int pte_numa(pte_t pte)
647{
648 return (pte_flags(pte) &
649 (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA;
650}
651#endif
652
653#ifndef pmd_numa
654static inline int pmd_numa(pmd_t pmd)
655{
656 return (pmd_flags(pmd) &
657 (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA;
658}
659#endif
660
661
662
663
664
665
666
667
668
669#ifndef pte_mknonnuma
670static inline pte_t pte_mknonnuma(pte_t pte)
671{
672 pte = pte_clear_flags(pte, _PAGE_NUMA);
673 return pte_set_flags(pte, _PAGE_PRESENT|_PAGE_ACCESSED);
674}
675#endif
676
677#ifndef pmd_mknonnuma
678static inline pmd_t pmd_mknonnuma(pmd_t pmd)
679{
680 pmd = pmd_clear_flags(pmd, _PAGE_NUMA);
681 return pmd_set_flags(pmd, _PAGE_PRESENT|_PAGE_ACCESSED);
682}
683#endif
684
685#ifndef pte_mknuma
686static inline pte_t pte_mknuma(pte_t pte)
687{
688 pte = pte_set_flags(pte, _PAGE_NUMA);
689 return pte_clear_flags(pte, _PAGE_PRESENT);
690}
691#endif
692
693#ifndef pmd_mknuma
694static inline pmd_t pmd_mknuma(pmd_t pmd)
695{
696 pmd = pmd_set_flags(pmd, _PAGE_NUMA);
697 return pmd_clear_flags(pmd, _PAGE_PRESENT);
698}
699#endif
700#else
701extern int pte_numa(pte_t pte);
702extern int pmd_numa(pmd_t pmd);
703extern pte_t pte_mknonnuma(pte_t pte);
704extern pmd_t pmd_mknonnuma(pmd_t pmd);
705extern pte_t pte_mknuma(pte_t pte);
706extern pmd_t pmd_mknuma(pmd_t pmd);
707#endif
708#else
709static inline int pmd_numa(pmd_t pmd)
710{
711 return 0;
712}
713
714static inline int pte_numa(pte_t pte)
715{
716 return 0;
717}
718
719static inline pte_t pte_mknonnuma(pte_t pte)
720{
721 return pte;
722}
723
724static inline pmd_t pmd_mknonnuma(pmd_t pmd)
725{
726 return pmd;
727}
728
729static inline pte_t pte_mknuma(pte_t pte)
730{
731 return pte;
732}
733
734static inline pmd_t pmd_mknuma(pmd_t pmd)
735{
736 return pmd;
737}
738#endif
739
740#endif
741
742#endif
743
744#ifndef io_remap_pfn_range
745#define io_remap_pfn_range remap_pfn_range
746#endif
747
748#endif
749