1
2#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
3#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
4
5#include <asm-generic/5level-fixup.h>
6
7#ifndef __ASSEMBLY__
8#include <linux/mmdebug.h>
9#include <linux/bug.h>
10#endif
11
12
13
14
15#define _PAGE_BIT_SWAP_TYPE 0
16
17#define _PAGE_EXEC 0x00001
18#define _PAGE_WRITE 0x00002
19#define _PAGE_READ 0x00004
20#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE)
21#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)
22#define _PAGE_PRIVILEGED 0x00008
23#define _PAGE_SAO 0x00010
24#define _PAGE_NON_IDEMPOTENT 0x00020
25#define _PAGE_TOLERANT 0x00030
26#define _PAGE_DIRTY 0x00080
27#define _PAGE_ACCESSED 0x00100
28
29
30
31#define _RPAGE_SW0 0x2000000000000000UL
32#define _RPAGE_SW1 0x00800
33#define _RPAGE_SW2 0x00400
34#define _RPAGE_SW3 0x00200
35#define _RPAGE_RSV1 0x1000000000000000UL
36#define _RPAGE_RSV2 0x0800000000000000UL
37#define _RPAGE_RSV3 0x0400000000000000UL
38#define _RPAGE_RSV4 0x0200000000000000UL
39#define _RPAGE_RSV5 0x00040UL
40
41#define _PAGE_PTE 0x4000000000000000UL
42#define _PAGE_PRESENT 0x8000000000000000UL
43
44
45
46
47
48
49
50
51
52#define _PAGE_INVALID _RPAGE_SW0
53
54
55
56
57
58
59#define _RPAGE_RPN0 0x01000
60#define _RPAGE_RPN1 0x02000
61#define _RPAGE_RPN44 0x0100000000000000UL
62#define _RPAGE_RPN43 0x0080000000000000UL
63#define _RPAGE_RPN42 0x0040000000000000UL
64#define _RPAGE_RPN41 0x0020000000000000UL
65
66
67#define _RPAGE_PA_MAX 57
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88#define _PAGE_PA_MAX 53
89
90#define _PAGE_SOFT_DIRTY _RPAGE_SW3
91#define _PAGE_SPECIAL _RPAGE_SW2
92#define _PAGE_DEVMAP _RPAGE_SW1
93
94
95
96
97
98
99#define _PAGE_NO_CACHE _PAGE_TOLERANT
100
101
102
103
104
105#define PTE_RPN_MASK (((1UL << _PAGE_PA_MAX) - 1) & (PAGE_MASK))
106
107
108
109
110#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
111 _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \
112 _PAGE_SOFT_DIRTY | _PAGE_DEVMAP)
113
114
115
116#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY)
117#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ)
118#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \
119 _PAGE_RW | _PAGE_EXEC)
120
121
122
123
124#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
125 _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \
126 _PAGE_SOFT_DIRTY | _PAGE_DEVMAP)
127
128#define H_PTE_PKEY (H_PTE_PKEY_BIT0 | H_PTE_PKEY_BIT1 | H_PTE_PKEY_BIT2 | \
129 H_PTE_PKEY_BIT3 | H_PTE_PKEY_BIT4)
130
131
132
133
134
135
136#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED)
137#define _PAGE_BASE (_PAGE_BASE_NC)
138
139
140
141
142
143
144
145
146
147#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_PRIVILEGED)
148#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW)
149#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC)
150#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_READ)
151#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
152#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ)
153#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
154
155
156#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW)
157#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
158 _PAGE_TOLERANT)
159#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
160 _PAGE_NON_IDEMPOTENT)
161#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX)
162#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO)
163#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX)
164
165
166
167
168
169
170#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \
171 defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE)
172#define PAGE_KERNEL_TEXT PAGE_KERNEL_X
173#else
174#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX
175#endif
176
177
178#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
179#define PAGE_AGP (PAGE_KERNEL_NC)
180
181#ifndef __ASSEMBLY__
182
183
184
185extern unsigned long __pte_index_size;
186extern unsigned long __pmd_index_size;
187extern unsigned long __pud_index_size;
188extern unsigned long __pgd_index_size;
189extern unsigned long __pud_cache_index;
190#define PTE_INDEX_SIZE __pte_index_size
191#define PMD_INDEX_SIZE __pmd_index_size
192#define PUD_INDEX_SIZE __pud_index_size
193#define PGD_INDEX_SIZE __pgd_index_size
194
195#define PMD_CACHE_INDEX 0
196#define PUD_CACHE_INDEX __pud_cache_index
197
198
199
200
201extern unsigned long __pte_table_size;
202extern unsigned long __pmd_table_size;
203extern unsigned long __pud_table_size;
204extern unsigned long __pgd_table_size;
205#define PTE_TABLE_SIZE __pte_table_size
206#define PMD_TABLE_SIZE __pmd_table_size
207#define PUD_TABLE_SIZE __pud_table_size
208#define PGD_TABLE_SIZE __pgd_table_size
209
210extern unsigned long __pmd_val_bits;
211extern unsigned long __pud_val_bits;
212extern unsigned long __pgd_val_bits;
213#define PMD_VAL_BITS __pmd_val_bits
214#define PUD_VAL_BITS __pud_val_bits
215#define PGD_VAL_BITS __pgd_val_bits
216
217extern unsigned long __pte_frag_nr;
218#define PTE_FRAG_NR __pte_frag_nr
219extern unsigned long __pte_frag_size_shift;
220#define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift
221#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
222
223extern unsigned long __pmd_frag_nr;
224#define PMD_FRAG_NR __pmd_frag_nr
225extern unsigned long __pmd_frag_size_shift;
226#define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift
227#define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT)
228
229#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
230#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
231#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
232#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
233
234
235#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
236#define PMD_SIZE (1UL << PMD_SHIFT)
237#define PMD_MASK (~(PMD_SIZE-1))
238
239
240#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
241#define PUD_SIZE (1UL << PUD_SHIFT)
242#define PUD_MASK (~(PUD_SIZE-1))
243
244
245#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
246#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
247#define PGDIR_MASK (~(PGDIR_SIZE-1))
248
249
250#define PMD_MASKED_BITS 0xc0000000000000ffUL
251
252#define PUD_MASKED_BITS 0xc0000000000000ffUL
253
254#define PGD_MASKED_BITS 0xc0000000000000ffUL
255
256
257
258
259enum pgtable_index {
260 PTE_INDEX = 0,
261 PMD_INDEX,
262 PUD_INDEX,
263 PGD_INDEX,
264
265
266
267 HTLB_16M_INDEX,
268 HTLB_16G_INDEX,
269};
270
271extern unsigned long __vmalloc_start;
272extern unsigned long __vmalloc_end;
273#define VMALLOC_START __vmalloc_start
274#define VMALLOC_END __vmalloc_end
275
276static inline unsigned int ioremap_max_order(void)
277{
278 if (radix_enabled())
279 return PUD_SHIFT;
280 return 7 + PAGE_SHIFT;
281}
282#define IOREMAP_MAX_ORDER ioremap_max_order()
283
284extern unsigned long __kernel_virt_start;
285extern unsigned long __kernel_io_start;
286extern unsigned long __kernel_io_end;
287#define KERN_VIRT_START __kernel_virt_start
288#define KERN_IO_START __kernel_io_start
289#define KERN_IO_END __kernel_io_end
290
291extern struct page *vmemmap;
292extern unsigned long ioremap_bot;
293extern unsigned long pci_io_base;
294#endif
295
296#include <asm/book3s/64/hash.h>
297#include <asm/book3s/64/radix.h>
298
299#ifdef CONFIG_PPC_64K_PAGES
300#include <asm/book3s/64/pgtable-64k.h>
301#else
302#include <asm/book3s/64/pgtable-4k.h>
303#endif
304
305#include <asm/barrier.h>
306
307
308
309
310
311
312
313
314#define FULL_IO_SIZE 0x80000000ul
315#define ISA_IO_BASE (KERN_IO_START)
316#define ISA_IO_END (KERN_IO_START + 0x10000ul)
317#define PHB_IO_BASE (ISA_IO_END)
318#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
319#define IOREMAP_BASE (PHB_IO_END)
320#define IOREMAP_END (KERN_IO_END)
321
322
323#define HAVE_PAGE_AGP
324
325#ifndef __ASSEMBLY__
326
327
328
329
330
331
332#ifndef __real_pte
333
334#define __real_pte(e, p, o) ((real_pte_t){(e)})
335#define __rpte_to_pte(r) ((r).pte)
336#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)
337
338#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \
339 do { \
340 index = 0; \
341 shift = mmu_psize_defs[psize].shift; \
342
343#define pte_iterate_hashed_end() } while(0)
344
345
346
347
348
349#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K
350
351#endif
352
353static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr,
354 pte_t *ptep, unsigned long clr,
355 unsigned long set, int huge)
356{
357 if (radix_enabled())
358 return radix__pte_update(mm, addr, ptep, clr, set, huge);
359 return hash__pte_update(mm, addr, ptep, clr, set, huge);
360}
361
362
363
364
365
366
367
368
369
370
371static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
372 unsigned long addr, pte_t *ptep)
373{
374 unsigned long old;
375
376 if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
377 return 0;
378 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
379 return (old & _PAGE_ACCESSED) != 0;
380}
381
382#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
383#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
384({ \
385 int __r; \
386 __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
387 __r; \
388})
389
390static inline int __pte_write(pte_t pte)
391{
392 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
393}
394
395#ifdef CONFIG_NUMA_BALANCING
396#define pte_savedwrite pte_savedwrite
397static inline bool pte_savedwrite(pte_t pte)
398{
399
400
401
402
403
404
405
406 return !(pte_raw(pte) & cpu_to_be64(_PAGE_RWX | _PAGE_PRIVILEGED));
407}
408#else
409#define pte_savedwrite pte_savedwrite
410static inline bool pte_savedwrite(pte_t pte)
411{
412 return false;
413}
414#endif
415
416static inline int pte_write(pte_t pte)
417{
418 return __pte_write(pte) || pte_savedwrite(pte);
419}
420
421static inline int pte_read(pte_t pte)
422{
423 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_READ));
424}
425
426#define __HAVE_ARCH_PTEP_SET_WRPROTECT
427static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
428 pte_t *ptep)
429{
430 if (__pte_write(*ptep))
431 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
432 else if (unlikely(pte_savedwrite(*ptep)))
433 pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 0);
434}
435
436#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT
437static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
438 unsigned long addr, pte_t *ptep)
439{
440
441
442
443
444 if (__pte_write(*ptep))
445 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
446 else if (unlikely(pte_savedwrite(*ptep)))
447 pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 1);
448}
449
450#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
451static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
452 unsigned long addr, pte_t *ptep)
453{
454 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
455 return __pte(old);
456}
457
458#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
459static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
460 unsigned long addr,
461 pte_t *ptep, int full)
462{
463 if (full && radix_enabled()) {
464
465
466
467
468 return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
469 }
470 return ptep_get_and_clear(mm, addr, ptep);
471}
472
473
474static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
475 pte_t * ptep)
476{
477 pte_update(mm, addr, ptep, ~0UL, 0, 0);
478}
479
480static inline int pte_dirty(pte_t pte)
481{
482 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));
483}
484
485static inline int pte_young(pte_t pte)
486{
487 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED));
488}
489
490static inline int pte_special(pte_t pte)
491{
492 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL));
493}
494
495static inline bool pte_exec(pte_t pte)
496{
497 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_EXEC));
498}
499
500
501#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
502static inline bool pte_soft_dirty(pte_t pte)
503{
504 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY));
505}
506
507static inline pte_t pte_mksoft_dirty(pte_t pte)
508{
509 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SOFT_DIRTY));
510}
511
512static inline pte_t pte_clear_soft_dirty(pte_t pte)
513{
514 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SOFT_DIRTY));
515}
516#endif
517
518#ifdef CONFIG_NUMA_BALANCING
519static inline int pte_protnone(pte_t pte)
520{
521 return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) ==
522 cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE);
523}
524
525#define pte_mk_savedwrite pte_mk_savedwrite
526static inline pte_t pte_mk_savedwrite(pte_t pte)
527{
528
529
530
531
532
533 VM_BUG_ON((pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_RWX | _PAGE_PRIVILEGED)) !=
534 cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED));
535 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED));
536}
537
538#define pte_clear_savedwrite pte_clear_savedwrite
539static inline pte_t pte_clear_savedwrite(pte_t pte)
540{
541
542
543
544 VM_BUG_ON(!pte_protnone(pte));
545 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED));
546}
547#else
548#define pte_clear_savedwrite pte_clear_savedwrite
549static inline pte_t pte_clear_savedwrite(pte_t pte)
550{
551 VM_WARN_ON(1);
552 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE));
553}
554#endif
555
556static inline int pte_present(pte_t pte)
557{
558
559
560
561
562
563
564 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID));
565}
566
567static inline bool pte_hw_valid(pte_t pte)
568{
569 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));
570}
571
572#ifdef CONFIG_PPC_MEM_KEYS
573extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute);
574#else
575static inline bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
576{
577 return true;
578}
579#endif
580
581static inline bool pte_user(pte_t pte)
582{
583 return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED));
584}
585
586#define pte_access_permitted pte_access_permitted
587static inline bool pte_access_permitted(pte_t pte, bool write)
588{
589
590
591
592
593 if (!pte_present(pte) || !pte_user(pte) || !pte_read(pte))
594 return false;
595
596 if (write && !pte_write(pte))
597 return false;
598
599 return arch_pte_access_permitted(pte_val(pte), write, 0);
600}
601
602
603
604
605
606
607
608
609static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
610{
611 return __pte((((pte_basic_t)(pfn) << PAGE_SHIFT) & PTE_RPN_MASK) |
612 pgprot_val(pgprot));
613}
614
615static inline unsigned long pte_pfn(pte_t pte)
616{
617 return (pte_val(pte) & PTE_RPN_MASK) >> PAGE_SHIFT;
618}
619
620
621static inline pte_t pte_wrprotect(pte_t pte)
622{
623 if (unlikely(pte_savedwrite(pte)))
624 return pte_clear_savedwrite(pte);
625 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE));
626}
627
628static inline pte_t pte_exprotect(pte_t pte)
629{
630 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_EXEC));
631}
632
633static inline pte_t pte_mkclean(pte_t pte)
634{
635 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_DIRTY));
636}
637
638static inline pte_t pte_mkold(pte_t pte)
639{
640 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_ACCESSED));
641}
642
643static inline pte_t pte_mkexec(pte_t pte)
644{
645 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC));
646}
647
648static inline pte_t pte_mkpte(pte_t pte)
649{
650 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PTE));
651}
652
653static inline pte_t pte_mkwrite(pte_t pte)
654{
655
656
657
658 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_RW));
659}
660
661static inline pte_t pte_mkdirty(pte_t pte)
662{
663 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_DIRTY | _PAGE_SOFT_DIRTY));
664}
665
666static inline pte_t pte_mkyoung(pte_t pte)
667{
668 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_ACCESSED));
669}
670
671static inline pte_t pte_mkspecial(pte_t pte)
672{
673 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL));
674}
675
676static inline pte_t pte_mkhuge(pte_t pte)
677{
678 return pte;
679}
680
681static inline pte_t pte_mkdevmap(pte_t pte)
682{
683 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL | _PAGE_DEVMAP));
684}
685
686static inline pte_t pte_mkprivileged(pte_t pte)
687{
688 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED));
689}
690
691static inline pte_t pte_mkuser(pte_t pte)
692{
693 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED));
694}
695
696
697
698
699
700
701
702static inline int pte_devmap(pte_t pte)
703{
704 u64 mask = cpu_to_be64(_PAGE_DEVMAP | _PAGE_PTE);
705
706 return (pte_raw(pte) & mask) == mask;
707}
708
709static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
710{
711
712 return __pte_raw((pte_raw(pte) & cpu_to_be64(_PAGE_CHG_MASK)) |
713 cpu_to_be64(pgprot_val(newprot)));
714}
715
716
717#define MAX_SWAPFILES_CHECK() do { \
718 BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \
719
720
721
722 \
723 BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \
724 BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \
725 } while (0)
726
727#define SWP_TYPE_BITS 5
728#define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \
729 & ((1UL << SWP_TYPE_BITS) - 1))
730#define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PAGE_SHIFT)
731#define __swp_entry(type, offset) ((swp_entry_t) { \
732 ((type) << _PAGE_BIT_SWAP_TYPE) \
733 | (((offset) << PAGE_SHIFT) & PTE_RPN_MASK)})
734
735
736
737
738
739
740#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) & ~_PAGE_PTE })
741#define __swp_entry_to_pte(x) __pte((x).val | _PAGE_PTE)
742#define __pmd_to_swp_entry(pmd) (__pte_to_swp_entry(pmd_pte(pmd)))
743#define __swp_entry_to_pmd(x) (pte_pmd(__swp_entry_to_pte(x)))
744
745#ifdef CONFIG_MEM_SOFT_DIRTY
746#define _PAGE_SWP_SOFT_DIRTY (1UL << (SWP_TYPE_BITS + _PAGE_BIT_SWAP_TYPE))
747#else
748#define _PAGE_SWP_SOFT_DIRTY 0UL
749#endif
750
751#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
752static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
753{
754 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
755}
756
757static inline bool pte_swp_soft_dirty(pte_t pte)
758{
759 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
760}
761
762static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
763{
764 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SWP_SOFT_DIRTY));
765}
766#endif
767
768static inline bool check_pte_access(unsigned long access, unsigned long ptev)
769{
770
771
772
773 if (access & ~ptev)
774 return false;
775
776
777
778 if ((access & _PAGE_PRIVILEGED) != (ptev & _PAGE_PRIVILEGED))
779 return false;
780
781 return true;
782}
783
784
785
786
787static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
788 pte_t *ptep, pte_t entry,
789 unsigned long address,
790 int psize)
791{
792 if (radix_enabled())
793 return radix__ptep_set_access_flags(vma, ptep, entry,
794 address, psize);
795 return hash__ptep_set_access_flags(ptep, entry);
796}
797
798#define __HAVE_ARCH_PTE_SAME
799static inline int pte_same(pte_t pte_a, pte_t pte_b)
800{
801 if (radix_enabled())
802 return radix__pte_same(pte_a, pte_b);
803 return hash__pte_same(pte_a, pte_b);
804}
805
806static inline int pte_none(pte_t pte)
807{
808 if (radix_enabled())
809 return radix__pte_none(pte);
810 return hash__pte_none(pte);
811}
812
813static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
814 pte_t *ptep, pte_t pte, int percpu)
815{
816 if (radix_enabled())
817 return radix__set_pte_at(mm, addr, ptep, pte, percpu);
818 return hash__set_pte_at(mm, addr, ptep, pte, percpu);
819}
820
821#define _PAGE_CACHE_CTL (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
822
823#define pgprot_noncached pgprot_noncached
824static inline pgprot_t pgprot_noncached(pgprot_t prot)
825{
826 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
827 _PAGE_NON_IDEMPOTENT);
828}
829
830#define pgprot_noncached_wc pgprot_noncached_wc
831static inline pgprot_t pgprot_noncached_wc(pgprot_t prot)
832{
833 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
834 _PAGE_TOLERANT);
835}
836
837#define pgprot_cached pgprot_cached
838static inline pgprot_t pgprot_cached(pgprot_t prot)
839{
840 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL));
841}
842
843#define pgprot_writecombine pgprot_writecombine
844static inline pgprot_t pgprot_writecombine(pgprot_t prot)
845{
846 return pgprot_noncached_wc(prot);
847}
848
849
850
851static inline bool pte_ci(pte_t pte)
852{
853 __be64 pte_v = pte_raw(pte);
854
855 if (((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_TOLERANT)) ||
856 ((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_NON_IDEMPOTENT)))
857 return true;
858 return false;
859}
860
861static inline void pmd_clear(pmd_t *pmdp)
862{
863 *pmdp = __pmd(0);
864}
865
866static inline int pmd_none(pmd_t pmd)
867{
868 return !pmd_raw(pmd);
869}
870
871static inline int pmd_present(pmd_t pmd)
872{
873
874
875
876
877
878
879 if (pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID))
880 return true;
881
882 return false;
883}
884
885static inline int pmd_is_serializing(pmd_t pmd)
886{
887
888
889
890
891
892
893
894
895 if ((pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) ==
896 cpu_to_be64(_PAGE_INVALID))
897 return true;
898
899 return false;
900}
901
902static inline int pmd_bad(pmd_t pmd)
903{
904 if (radix_enabled())
905 return radix__pmd_bad(pmd);
906 return hash__pmd_bad(pmd);
907}
908
909static inline void pud_clear(pud_t *pudp)
910{
911 *pudp = __pud(0);
912}
913
914static inline int pud_none(pud_t pud)
915{
916 return !pud_raw(pud);
917}
918
919static inline int pud_present(pud_t pud)
920{
921 return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PRESENT));
922}
923
924extern struct page *pud_page(pud_t pud);
925extern struct page *pmd_page(pmd_t pmd);
926static inline pte_t pud_pte(pud_t pud)
927{
928 return __pte_raw(pud_raw(pud));
929}
930
931static inline pud_t pte_pud(pte_t pte)
932{
933 return __pud_raw(pte_raw(pte));
934}
935#define pud_write(pud) pte_write(pud_pte(pud))
936
937static inline int pud_bad(pud_t pud)
938{
939 if (radix_enabled())
940 return radix__pud_bad(pud);
941 return hash__pud_bad(pud);
942}
943
944#define pud_access_permitted pud_access_permitted
945static inline bool pud_access_permitted(pud_t pud, bool write)
946{
947 return pte_access_permitted(pud_pte(pud), write);
948}
949
950#define pgd_write(pgd) pte_write(pgd_pte(pgd))
951
952static inline void pgd_clear(pgd_t *pgdp)
953{
954 *pgdp = __pgd(0);
955}
956
957static inline int pgd_none(pgd_t pgd)
958{
959 return !pgd_raw(pgd);
960}
961
962static inline int pgd_present(pgd_t pgd)
963{
964 return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PRESENT));
965}
966
967static inline pte_t pgd_pte(pgd_t pgd)
968{
969 return __pte_raw(pgd_raw(pgd));
970}
971
972static inline pgd_t pte_pgd(pte_t pte)
973{
974 return __pgd_raw(pte_raw(pte));
975}
976
977static inline int pgd_bad(pgd_t pgd)
978{
979 if (radix_enabled())
980 return radix__pgd_bad(pgd);
981 return hash__pgd_bad(pgd);
982}
983
984#define pgd_access_permitted pgd_access_permitted
985static inline bool pgd_access_permitted(pgd_t pgd, bool write)
986{
987 return pte_access_permitted(pgd_pte(pgd), write);
988}
989
990extern struct page *pgd_page(pgd_t pgd);
991
992
993#define __pgtable_ptr_val(ptr) __pa(ptr)
994
995#define pmd_page_vaddr(pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)
996#define pud_page_vaddr(pud) __va(pud_val(pud) & ~PUD_MASKED_BITS)
997#define pgd_page_vaddr(pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS)
998
999#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
1000#define pud_index(address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))
1001#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
1002#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))
1003
1004
1005
1006
1007
1008
1009#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
1010
1011#define pud_offset(pgdp, addr) \
1012 (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr))
1013#define pmd_offset(pudp,addr) \
1014 (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))
1015#define pte_offset_kernel(dir,addr) \
1016 (((pte_t *) pmd_page_vaddr(*(dir))) + pte_index(addr))
1017
1018#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
1019
1020static inline void pte_unmap(pte_t *pte) { }
1021
1022
1023
1024#define pgd_offset_k(address) pgd_offset(&init_mm, address)
1025
1026#define pte_ERROR(e) \
1027 pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
1028#define pmd_ERROR(e) \
1029 pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
1030#define pud_ERROR(e) \
1031 pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
1032#define pgd_ERROR(e) \
1033 pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
1034
1035static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
1036{
1037 if (radix_enabled()) {
1038#if defined(CONFIG_PPC_RADIX_MMU) && defined(DEBUG_VM)
1039 unsigned long page_size = 1 << mmu_psize_defs[mmu_io_psize].shift;
1040 WARN((page_size != PAGE_SIZE), "I/O page size != PAGE_SIZE");
1041#endif
1042 return radix__map_kernel_page(ea, pa, prot, PAGE_SIZE);
1043 }
1044 return hash__map_kernel_page(ea, pa, prot);
1045}
1046
1047static inline int __meminit vmemmap_create_mapping(unsigned long start,
1048 unsigned long page_size,
1049 unsigned long phys)
1050{
1051 if (radix_enabled())
1052 return radix__vmemmap_create_mapping(start, page_size, phys);
1053 return hash__vmemmap_create_mapping(start, page_size, phys);
1054}
1055
1056#ifdef CONFIG_MEMORY_HOTPLUG
1057static inline void vmemmap_remove_mapping(unsigned long start,
1058 unsigned long page_size)
1059{
1060 if (radix_enabled())
1061 return radix__vmemmap_remove_mapping(start, page_size);
1062 return hash__vmemmap_remove_mapping(start, page_size);
1063}
1064#endif
1065
1066static inline pte_t pmd_pte(pmd_t pmd)
1067{
1068 return __pte_raw(pmd_raw(pmd));
1069}
1070
1071static inline pmd_t pte_pmd(pte_t pte)
1072{
1073 return __pmd_raw(pte_raw(pte));
1074}
1075
1076static inline pte_t *pmdp_ptep(pmd_t *pmd)
1077{
1078 return (pte_t *)pmd;
1079}
1080#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
1081#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
1082#define pmd_young(pmd) pte_young(pmd_pte(pmd))
1083#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
1084#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
1085#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
1086#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
1087#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
1088#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
1089#define pmd_mk_savedwrite(pmd) pte_pmd(pte_mk_savedwrite(pmd_pte(pmd)))
1090#define pmd_clear_savedwrite(pmd) pte_pmd(pte_clear_savedwrite(pmd_pte(pmd)))
1091
1092#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1093#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))
1094#define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd)))
1095#define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)))
1096
1097#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1098#define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd)))
1099#define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd))
1100#define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd)))
1101#endif
1102#endif
1103
1104#ifdef CONFIG_NUMA_BALANCING
1105static inline int pmd_protnone(pmd_t pmd)
1106{
1107 return pte_protnone(pmd_pte(pmd));
1108}
1109#endif
1110
1111#define pmd_write(pmd) pte_write(pmd_pte(pmd))
1112#define __pmd_write(pmd) __pte_write(pmd_pte(pmd))
1113#define pmd_savedwrite(pmd) pte_savedwrite(pmd_pte(pmd))
1114
1115#define pmd_access_permitted pmd_access_permitted
1116static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1117{
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128 if (pmd_is_serializing(pmd))
1129 return false;
1130
1131 return pte_access_permitted(pmd_pte(pmd), write);
1132}
1133
1134#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1135extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
1136extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
1137extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
1138extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1139 pmd_t *pmdp, pmd_t pmd);
1140extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
1141 pmd_t *pmd);
1142extern int hash__has_transparent_hugepage(void);
1143static inline int has_transparent_hugepage(void)
1144{
1145 if (radix_enabled())
1146 return radix__has_transparent_hugepage();
1147 return hash__has_transparent_hugepage();
1148}
1149#define has_transparent_hugepage has_transparent_hugepage
1150
1151static inline unsigned long
1152pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp,
1153 unsigned long clr, unsigned long set)
1154{
1155 if (radix_enabled())
1156 return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set);
1157 return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set);
1158}
1159
1160
1161
1162
1163
1164static inline int pmd_large(pmd_t pmd)
1165{
1166 return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
1167}
1168
1169static inline pmd_t pmd_mknotpresent(pmd_t pmd)
1170{
1171 return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT);
1172}
1173
1174
1175
1176
1177static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
1178 unsigned long addr, pmd_t *pmdp)
1179{
1180 unsigned long old;
1181
1182 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
1183 return 0;
1184 old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
1185 return ((old & _PAGE_ACCESSED) != 0);
1186}
1187
1188#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1189static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
1190 pmd_t *pmdp)
1191{
1192 if (__pmd_write((*pmdp)))
1193 pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);
1194 else if (unlikely(pmd_savedwrite(*pmdp)))
1195 pmd_hugepage_update(mm, addr, pmdp, 0, _PAGE_PRIVILEGED);
1196}
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209static inline int pmd_trans_huge(pmd_t pmd)
1210{
1211 if (!pmd_present(pmd))
1212 return false;
1213
1214 if (radix_enabled())
1215 return radix__pmd_trans_huge(pmd);
1216 return hash__pmd_trans_huge(pmd);
1217}
1218
1219#define __HAVE_ARCH_PMD_SAME
1220static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
1221{
1222 if (radix_enabled())
1223 return radix__pmd_same(pmd_a, pmd_b);
1224 return hash__pmd_same(pmd_a, pmd_b);
1225}
1226
1227static inline pmd_t pmd_mkhuge(pmd_t pmd)
1228{
1229 if (radix_enabled())
1230 return radix__pmd_mkhuge(pmd);
1231 return hash__pmd_mkhuge(pmd);
1232}
1233
1234#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1235extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1236 unsigned long address, pmd_t *pmdp,
1237 pmd_t entry, int dirty);
1238
1239#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1240extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1241 unsigned long address, pmd_t *pmdp);
1242
1243#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1244static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1245 unsigned long addr, pmd_t *pmdp)
1246{
1247 if (radix_enabled())
1248 return radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
1249 return hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
1250}
1251
1252static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1253 unsigned long address, pmd_t *pmdp)
1254{
1255 if (radix_enabled())
1256 return radix__pmdp_collapse_flush(vma, address, pmdp);
1257 return hash__pmdp_collapse_flush(vma, address, pmdp);
1258}
1259#define pmdp_collapse_flush pmdp_collapse_flush
1260
1261#define __HAVE_ARCH_PGTABLE_DEPOSIT
1262static inline void pgtable_trans_huge_deposit(struct mm_struct *mm,
1263 pmd_t *pmdp, pgtable_t pgtable)
1264{
1265 if (radix_enabled())
1266 return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
1267 return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
1268}
1269
1270#define __HAVE_ARCH_PGTABLE_WITHDRAW
1271static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm,
1272 pmd_t *pmdp)
1273{
1274 if (radix_enabled())
1275 return radix__pgtable_trans_huge_withdraw(mm, pmdp);
1276 return hash__pgtable_trans_huge_withdraw(mm, pmdp);
1277}
1278
1279#define __HAVE_ARCH_PMDP_INVALIDATE
1280extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
1281 pmd_t *pmdp);
1282
1283#define pmd_move_must_withdraw pmd_move_must_withdraw
1284struct spinlock;
1285extern int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
1286 struct spinlock *old_pmd_ptl,
1287 struct vm_area_struct *vma);
1288
1289
1290
1291
1292#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit
1293static inline bool arch_needs_pgtable_deposit(void)
1294{
1295 if (radix_enabled())
1296 return false;
1297 return true;
1298}
1299extern void serialize_against_pte_lookup(struct mm_struct *mm);
1300
1301
1302static inline pmd_t pmd_mkdevmap(pmd_t pmd)
1303{
1304 return __pmd(pmd_val(pmd) | (_PAGE_PTE | _PAGE_DEVMAP));
1305}
1306
1307static inline int pmd_devmap(pmd_t pmd)
1308{
1309 return pte_devmap(pmd_pte(pmd));
1310}
1311
1312static inline int pud_devmap(pud_t pud)
1313{
1314 return 0;
1315}
1316
1317static inline int pgd_devmap(pgd_t pgd)
1318{
1319 return 0;
1320}
1321#endif
1322
1323static inline int pud_pfn(pud_t pud)
1324{
1325
1326
1327
1328
1329
1330 BUILD_BUG();
1331 return 0;
1332}
1333#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1334pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *);
1335void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long,
1336 pte_t *, pte_t, pte_t);
1337
1338
1339
1340
1341static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_val)
1342{
1343 if (!(old_val & _PAGE_READ))
1344 return false;
1345
1346 if ((!(old_val & _PAGE_WRITE)) && (new_val & _PAGE_WRITE))
1347 return true;
1348
1349 return false;
1350}
1351
1352
1353
1354
1355#define pmd_is_leaf pmd_is_leaf
1356static inline bool pmd_is_leaf(pmd_t pmd)
1357{
1358 return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
1359}
1360
1361#define pud_is_leaf pud_is_leaf
1362static inline bool pud_is_leaf(pud_t pud)
1363{
1364 return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));
1365}
1366
1367#define pgd_is_leaf pgd_is_leaf
1368static inline bool pgd_is_leaf(pgd_t pgd)
1369{
1370 return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));
1371}
1372
1373#endif
1374#endif
1375