1
2#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
3#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
4
5#include <asm-generic/5level-fixup.h>
6
7#ifndef __ASSEMBLY__
8#include <linux/mmdebug.h>
9#include <linux/bug.h>
10#endif
11
12
13
14
15#define _PAGE_BIT_SWAP_TYPE 0
16
17#define _PAGE_EXEC 0x00001
18#define _PAGE_WRITE 0x00002
19#define _PAGE_READ 0x00004
20#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE)
21#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)
22#define _PAGE_PRIVILEGED 0x00008
23#define _PAGE_SAO 0x00010
24#define _PAGE_NON_IDEMPOTENT 0x00020
25#define _PAGE_TOLERANT 0x00030
26#define _PAGE_DIRTY 0x00080
27#define _PAGE_ACCESSED 0x00100
28
29
30
31#define _RPAGE_SW0 0x2000000000000000UL
32#define _RPAGE_SW1 0x00800
33#define _RPAGE_SW2 0x00400
34#define _RPAGE_SW3 0x00200
35#define _RPAGE_RSV1 0x1000000000000000UL
36#define _RPAGE_RSV2 0x0800000000000000UL
37#define _RPAGE_RSV3 0x0400000000000000UL
38#define _RPAGE_RSV4 0x0200000000000000UL
39#define _RPAGE_RSV5 0x00040UL
40
41#define _PAGE_PTE 0x4000000000000000UL
42#define _PAGE_PRESENT 0x8000000000000000UL
43
44
45
46
47
48
49
50
51
52#define _PAGE_INVALID _RPAGE_SW0
53
54
55
56
57
58
59#define _RPAGE_RPN0 0x01000
60#define _RPAGE_RPN1 0x02000
61#define _RPAGE_RPN44 0x0100000000000000UL
62#define _RPAGE_RPN43 0x0080000000000000UL
63#define _RPAGE_RPN42 0x0040000000000000UL
64#define _RPAGE_RPN41 0x0020000000000000UL
65
66
67#define _RPAGE_PA_MAX 57
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88#define _PAGE_PA_MAX 53
89
90#define _PAGE_SOFT_DIRTY _RPAGE_SW3
91#define _PAGE_SPECIAL _RPAGE_SW2
92#define _PAGE_DEVMAP _RPAGE_SW1
93#define __HAVE_ARCH_PTE_DEVMAP
94
95
96
97
98
99
100#define _PAGE_NO_CACHE _PAGE_TOLERANT
101
102
103
104
105
106#define PTE_RPN_MASK (((1UL << _PAGE_PA_MAX) - 1) & (PAGE_MASK))
107
108
109
110
111#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
112 _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \
113 _PAGE_SOFT_DIRTY | _PAGE_DEVMAP)
114
115
116
117#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY)
118#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ)
119#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \
120 _PAGE_RW | _PAGE_EXEC)
121
122
123
124
125#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
126 _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \
127 _PAGE_SOFT_DIRTY | _PAGE_DEVMAP)
128
129#define H_PTE_PKEY (H_PTE_PKEY_BIT0 | H_PTE_PKEY_BIT1 | H_PTE_PKEY_BIT2 | \
130 H_PTE_PKEY_BIT3 | H_PTE_PKEY_BIT4)
131
132
133
134
135
136
137#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED)
138#define _PAGE_BASE (_PAGE_BASE_NC)
139
140
141
142
143
144
145
146
147
148#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_PRIVILEGED)
149#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW)
150#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC)
151#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_READ)
152#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
153#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ)
154#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
155
156
157#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW)
158#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
159 _PAGE_TOLERANT)
160#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
161 _PAGE_NON_IDEMPOTENT)
162#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX)
163#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO)
164#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX)
165
166
167
168
169
170
171#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \
172 defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE)
173#define PAGE_KERNEL_TEXT PAGE_KERNEL_X
174#else
175#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX
176#endif
177
178
179#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
180#define PAGE_AGP (PAGE_KERNEL_NC)
181
182#ifndef __ASSEMBLY__
183
184
185
186extern unsigned long __pte_index_size;
187extern unsigned long __pmd_index_size;
188extern unsigned long __pud_index_size;
189extern unsigned long __pgd_index_size;
190extern unsigned long __pud_cache_index;
191#define PTE_INDEX_SIZE __pte_index_size
192#define PMD_INDEX_SIZE __pmd_index_size
193#define PUD_INDEX_SIZE __pud_index_size
194#define PGD_INDEX_SIZE __pgd_index_size
195
196#define PMD_CACHE_INDEX 0
197#define PUD_CACHE_INDEX __pud_cache_index
198
199
200
201
202extern unsigned long __pte_table_size;
203extern unsigned long __pmd_table_size;
204extern unsigned long __pud_table_size;
205extern unsigned long __pgd_table_size;
206#define PTE_TABLE_SIZE __pte_table_size
207#define PMD_TABLE_SIZE __pmd_table_size
208#define PUD_TABLE_SIZE __pud_table_size
209#define PGD_TABLE_SIZE __pgd_table_size
210
211extern unsigned long __pmd_val_bits;
212extern unsigned long __pud_val_bits;
213extern unsigned long __pgd_val_bits;
214#define PMD_VAL_BITS __pmd_val_bits
215#define PUD_VAL_BITS __pud_val_bits
216#define PGD_VAL_BITS __pgd_val_bits
217
218extern unsigned long __pte_frag_nr;
219#define PTE_FRAG_NR __pte_frag_nr
220extern unsigned long __pte_frag_size_shift;
221#define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift
222#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
223
224extern unsigned long __pmd_frag_nr;
225#define PMD_FRAG_NR __pmd_frag_nr
226extern unsigned long __pmd_frag_size_shift;
227#define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift
228#define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT)
229
230#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
231#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
232#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
233#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
234
235
236#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
237#define PMD_SIZE (1UL << PMD_SHIFT)
238#define PMD_MASK (~(PMD_SIZE-1))
239
240
241#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
242#define PUD_SIZE (1UL << PUD_SHIFT)
243#define PUD_MASK (~(PUD_SIZE-1))
244
245
246#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
247#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
248#define PGDIR_MASK (~(PGDIR_SIZE-1))
249
250
251#define PMD_MASKED_BITS 0xc0000000000000ffUL
252
253#define PUD_MASKED_BITS 0xc0000000000000ffUL
254
255#define PGD_MASKED_BITS 0xc0000000000000ffUL
256
257
258
259
260enum pgtable_index {
261 PTE_INDEX = 0,
262 PMD_INDEX,
263 PUD_INDEX,
264 PGD_INDEX,
265
266
267
268 HTLB_16M_INDEX,
269 HTLB_16G_INDEX,
270};
271
272extern unsigned long __vmalloc_start;
273extern unsigned long __vmalloc_end;
274#define VMALLOC_START __vmalloc_start
275#define VMALLOC_END __vmalloc_end
276
277extern unsigned long __kernel_virt_start;
278extern unsigned long __kernel_virt_size;
279extern unsigned long __kernel_io_start;
280extern unsigned long __kernel_io_end;
281#define KERN_VIRT_START __kernel_virt_start
282#define KERN_IO_START __kernel_io_start
283#define KERN_IO_END __kernel_io_end
284
285extern struct page *vmemmap;
286extern unsigned long ioremap_bot;
287extern unsigned long pci_io_base;
288#endif
289
290#include <asm/book3s/64/hash.h>
291#include <asm/book3s/64/radix.h>
292
293#ifdef CONFIG_PPC_64K_PAGES
294#include <asm/book3s/64/pgtable-64k.h>
295#else
296#include <asm/book3s/64/pgtable-4k.h>
297#endif
298
299#include <asm/barrier.h>
300
301
302
303
304
305
306
307
308#define FULL_IO_SIZE 0x80000000ul
309#define ISA_IO_BASE (KERN_IO_START)
310#define ISA_IO_END (KERN_IO_START + 0x10000ul)
311#define PHB_IO_BASE (ISA_IO_END)
312#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
313#define IOREMAP_BASE (PHB_IO_END)
314#define IOREMAP_END (KERN_IO_END)
315
316
317#define HAVE_PAGE_AGP
318
319#ifndef __ASSEMBLY__
320
321
322
323
324
325
326#ifndef __real_pte
327
328#define __real_pte(e, p, o) ((real_pte_t){(e)})
329#define __rpte_to_pte(r) ((r).pte)
330#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)
331
332#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \
333 do { \
334 index = 0; \
335 shift = mmu_psize_defs[psize].shift; \
336
337#define pte_iterate_hashed_end() } while(0)
338
339
340
341
342
343#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K
344
345#endif
346
347static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr,
348 pte_t *ptep, unsigned long clr,
349 unsigned long set, int huge)
350{
351 if (radix_enabled())
352 return radix__pte_update(mm, addr, ptep, clr, set, huge);
353 return hash__pte_update(mm, addr, ptep, clr, set, huge);
354}
355
356
357
358
359
360
361
362
363
364
365static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
366 unsigned long addr, pte_t *ptep)
367{
368 unsigned long old;
369
370 if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
371 return 0;
372 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
373 return (old & _PAGE_ACCESSED) != 0;
374}
375
376#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
377#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
378({ \
379 int __r; \
380 __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
381 __r; \
382})
383
384static inline int __pte_write(pte_t pte)
385{
386 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
387}
388
389#ifdef CONFIG_NUMA_BALANCING
390#define pte_savedwrite pte_savedwrite
391static inline bool pte_savedwrite(pte_t pte)
392{
393
394
395
396
397
398
399
400 return !(pte_raw(pte) & cpu_to_be64(_PAGE_RWX | _PAGE_PRIVILEGED));
401}
402#else
403#define pte_savedwrite pte_savedwrite
404static inline bool pte_savedwrite(pte_t pte)
405{
406 return false;
407}
408#endif
409
410static inline int pte_write(pte_t pte)
411{
412 return __pte_write(pte) || pte_savedwrite(pte);
413}
414
415static inline int pte_read(pte_t pte)
416{
417 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_READ));
418}
419
420#define __HAVE_ARCH_PTEP_SET_WRPROTECT
421static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
422 pte_t *ptep)
423{
424 if (__pte_write(*ptep))
425 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
426 else if (unlikely(pte_savedwrite(*ptep)))
427 pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 0);
428}
429
430#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT
431static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
432 unsigned long addr, pte_t *ptep)
433{
434
435
436
437
438 if (__pte_write(*ptep))
439 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
440 else if (unlikely(pte_savedwrite(*ptep)))
441 pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 1);
442}
443
444#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
445static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
446 unsigned long addr, pte_t *ptep)
447{
448 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
449 return __pte(old);
450}
451
452#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
453static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
454 unsigned long addr,
455 pte_t *ptep, int full)
456{
457 if (full && radix_enabled()) {
458
459
460
461
462 return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
463 }
464 return ptep_get_and_clear(mm, addr, ptep);
465}
466
467
468static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
469 pte_t * ptep)
470{
471 pte_update(mm, addr, ptep, ~0UL, 0, 0);
472}
473
474static inline int pte_dirty(pte_t pte)
475{
476 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));
477}
478
479static inline int pte_young(pte_t pte)
480{
481 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED));
482}
483
484static inline int pte_special(pte_t pte)
485{
486 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL));
487}
488
489static inline bool pte_exec(pte_t pte)
490{
491 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_EXEC));
492}
493
494
495#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
496static inline bool pte_soft_dirty(pte_t pte)
497{
498 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY));
499}
500
501static inline pte_t pte_mksoft_dirty(pte_t pte)
502{
503 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SOFT_DIRTY));
504}
505
506static inline pte_t pte_clear_soft_dirty(pte_t pte)
507{
508 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SOFT_DIRTY));
509}
510#endif
511
512#ifdef CONFIG_NUMA_BALANCING
513static inline int pte_protnone(pte_t pte)
514{
515 return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) ==
516 cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE);
517}
518
519#define pte_mk_savedwrite pte_mk_savedwrite
520static inline pte_t pte_mk_savedwrite(pte_t pte)
521{
522
523
524
525
526
527 VM_BUG_ON((pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_RWX | _PAGE_PRIVILEGED)) !=
528 cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED));
529 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED));
530}
531
532#define pte_clear_savedwrite pte_clear_savedwrite
533static inline pte_t pte_clear_savedwrite(pte_t pte)
534{
535
536
537
538 VM_BUG_ON(!pte_protnone(pte));
539 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED));
540}
541#else
542#define pte_clear_savedwrite pte_clear_savedwrite
543static inline pte_t pte_clear_savedwrite(pte_t pte)
544{
545 VM_WARN_ON(1);
546 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE));
547}
548#endif
549
550static inline int pte_present(pte_t pte)
551{
552
553
554
555
556
557
558 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID));
559}
560
561static inline bool pte_hw_valid(pte_t pte)
562{
563 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));
564}
565
566#ifdef CONFIG_PPC_MEM_KEYS
567extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute);
568#else
569static inline bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
570{
571 return true;
572}
573#endif
574
575static inline bool pte_user(pte_t pte)
576{
577 return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED));
578}
579
580#define pte_access_permitted pte_access_permitted
581static inline bool pte_access_permitted(pte_t pte, bool write)
582{
583
584
585
586
587 if (!pte_present(pte) || !pte_user(pte) || !pte_read(pte))
588 return false;
589
590 if (write && !pte_write(pte))
591 return false;
592
593 return arch_pte_access_permitted(pte_val(pte), write, 0);
594}
595
596
597
598
599
600
601
602
603static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
604{
605 return __pte((((pte_basic_t)(pfn) << PAGE_SHIFT) & PTE_RPN_MASK) |
606 pgprot_val(pgprot));
607}
608
609static inline unsigned long pte_pfn(pte_t pte)
610{
611 return (pte_val(pte) & PTE_RPN_MASK) >> PAGE_SHIFT;
612}
613
614
615static inline pte_t pte_wrprotect(pte_t pte)
616{
617 if (unlikely(pte_savedwrite(pte)))
618 return pte_clear_savedwrite(pte);
619 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE));
620}
621
622static inline pte_t pte_exprotect(pte_t pte)
623{
624 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_EXEC));
625}
626
627static inline pte_t pte_mkclean(pte_t pte)
628{
629 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_DIRTY));
630}
631
632static inline pte_t pte_mkold(pte_t pte)
633{
634 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_ACCESSED));
635}
636
637static inline pte_t pte_mkexec(pte_t pte)
638{
639 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC));
640}
641
642static inline pte_t pte_mkpte(pte_t pte)
643{
644 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PTE));
645}
646
647static inline pte_t pte_mkwrite(pte_t pte)
648{
649
650
651
652 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_RW));
653}
654
655static inline pte_t pte_mkdirty(pte_t pte)
656{
657 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_DIRTY | _PAGE_SOFT_DIRTY));
658}
659
660static inline pte_t pte_mkyoung(pte_t pte)
661{
662 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_ACCESSED));
663}
664
665static inline pte_t pte_mkspecial(pte_t pte)
666{
667 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL));
668}
669
670static inline pte_t pte_mkhuge(pte_t pte)
671{
672 return pte;
673}
674
675static inline pte_t pte_mkdevmap(pte_t pte)
676{
677 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL | _PAGE_DEVMAP));
678}
679
680static inline pte_t pte_mkprivileged(pte_t pte)
681{
682 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED));
683}
684
685static inline pte_t pte_mkuser(pte_t pte)
686{
687 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED));
688}
689
690
691
692
693
694
695
696static inline int pte_devmap(pte_t pte)
697{
698 u64 mask = cpu_to_be64(_PAGE_DEVMAP | _PAGE_PTE);
699
700 return (pte_raw(pte) & mask) == mask;
701}
702
703static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
704{
705
706 return __pte_raw((pte_raw(pte) & cpu_to_be64(_PAGE_CHG_MASK)) |
707 cpu_to_be64(pgprot_val(newprot)));
708}
709
710
711#define MAX_SWAPFILES_CHECK() do { \
712 BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \
713
714
715
716 \
717 BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \
718 BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \
719 } while (0)
720
721#define SWP_TYPE_BITS 5
722#define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \
723 & ((1UL << SWP_TYPE_BITS) - 1))
724#define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PAGE_SHIFT)
725#define __swp_entry(type, offset) ((swp_entry_t) { \
726 ((type) << _PAGE_BIT_SWAP_TYPE) \
727 | (((offset) << PAGE_SHIFT) & PTE_RPN_MASK)})
728
729
730
731
732
733
734#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) & ~_PAGE_PTE })
735#define __swp_entry_to_pte(x) __pte((x).val | _PAGE_PTE)
736#define __pmd_to_swp_entry(pmd) (__pte_to_swp_entry(pmd_pte(pmd)))
737#define __swp_entry_to_pmd(x) (pte_pmd(__swp_entry_to_pte(x)))
738
739#ifdef CONFIG_MEM_SOFT_DIRTY
740#define _PAGE_SWP_SOFT_DIRTY (1UL << (SWP_TYPE_BITS + _PAGE_BIT_SWAP_TYPE))
741#else
742#define _PAGE_SWP_SOFT_DIRTY 0UL
743#endif
744
745#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
746static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
747{
748 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
749}
750
751static inline bool pte_swp_soft_dirty(pte_t pte)
752{
753 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
754}
755
756static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
757{
758 return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SWP_SOFT_DIRTY));
759}
760#endif
761
762static inline bool check_pte_access(unsigned long access, unsigned long ptev)
763{
764
765
766
767 if (access & ~ptev)
768 return false;
769
770
771
772 if ((access & _PAGE_PRIVILEGED) != (ptev & _PAGE_PRIVILEGED))
773 return false;
774
775 return true;
776}
777
778
779
780
781static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
782 pte_t *ptep, pte_t entry,
783 unsigned long address,
784 int psize)
785{
786 if (radix_enabled())
787 return radix__ptep_set_access_flags(vma, ptep, entry,
788 address, psize);
789 return hash__ptep_set_access_flags(ptep, entry);
790}
791
792#define __HAVE_ARCH_PTE_SAME
793static inline int pte_same(pte_t pte_a, pte_t pte_b)
794{
795 if (radix_enabled())
796 return radix__pte_same(pte_a, pte_b);
797 return hash__pte_same(pte_a, pte_b);
798}
799
800static inline int pte_none(pte_t pte)
801{
802 if (radix_enabled())
803 return radix__pte_none(pte);
804 return hash__pte_none(pte);
805}
806
807static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
808 pte_t *ptep, pte_t pte, int percpu)
809{
810 if (radix_enabled())
811 return radix__set_pte_at(mm, addr, ptep, pte, percpu);
812 return hash__set_pte_at(mm, addr, ptep, pte, percpu);
813}
814
815#define _PAGE_CACHE_CTL (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
816
817#define pgprot_noncached pgprot_noncached
818static inline pgprot_t pgprot_noncached(pgprot_t prot)
819{
820 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
821 _PAGE_NON_IDEMPOTENT);
822}
823
824#define pgprot_noncached_wc pgprot_noncached_wc
825static inline pgprot_t pgprot_noncached_wc(pgprot_t prot)
826{
827 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
828 _PAGE_TOLERANT);
829}
830
831#define pgprot_cached pgprot_cached
832static inline pgprot_t pgprot_cached(pgprot_t prot)
833{
834 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL));
835}
836
837#define pgprot_writecombine pgprot_writecombine
838static inline pgprot_t pgprot_writecombine(pgprot_t prot)
839{
840 return pgprot_noncached_wc(prot);
841}
842
843
844
845static inline bool pte_ci(pte_t pte)
846{
847 __be64 pte_v = pte_raw(pte);
848
849 if (((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_TOLERANT)) ||
850 ((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_NON_IDEMPOTENT)))
851 return true;
852 return false;
853}
854
855static inline void pmd_clear(pmd_t *pmdp)
856{
857 *pmdp = __pmd(0);
858}
859
860static inline int pmd_none(pmd_t pmd)
861{
862 return !pmd_raw(pmd);
863}
864
865static inline int pmd_present(pmd_t pmd)
866{
867
868
869
870
871
872
873 if (pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID))
874 return true;
875
876 return false;
877}
878
879static inline int pmd_is_serializing(pmd_t pmd)
880{
881
882
883
884
885
886
887
888
889 if ((pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) ==
890 cpu_to_be64(_PAGE_INVALID))
891 return true;
892
893 return false;
894}
895
896static inline int pmd_bad(pmd_t pmd)
897{
898 if (radix_enabled())
899 return radix__pmd_bad(pmd);
900 return hash__pmd_bad(pmd);
901}
902
903static inline void pud_clear(pud_t *pudp)
904{
905 *pudp = __pud(0);
906}
907
908static inline int pud_none(pud_t pud)
909{
910 return !pud_raw(pud);
911}
912
913static inline int pud_present(pud_t pud)
914{
915 return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PRESENT));
916}
917
918extern struct page *pud_page(pud_t pud);
919extern struct page *pmd_page(pmd_t pmd);
920static inline pte_t pud_pte(pud_t pud)
921{
922 return __pte_raw(pud_raw(pud));
923}
924
925static inline pud_t pte_pud(pte_t pte)
926{
927 return __pud_raw(pte_raw(pte));
928}
929#define pud_write(pud) pte_write(pud_pte(pud))
930
931static inline int pud_bad(pud_t pud)
932{
933 if (radix_enabled())
934 return radix__pud_bad(pud);
935 return hash__pud_bad(pud);
936}
937
938#define pud_access_permitted pud_access_permitted
939static inline bool pud_access_permitted(pud_t pud, bool write)
940{
941 return pte_access_permitted(pud_pte(pud), write);
942}
943
944#define pgd_write(pgd) pte_write(pgd_pte(pgd))
945
946static inline void pgd_clear(pgd_t *pgdp)
947{
948 *pgdp = __pgd(0);
949}
950
951static inline int pgd_none(pgd_t pgd)
952{
953 return !pgd_raw(pgd);
954}
955
956static inline int pgd_present(pgd_t pgd)
957{
958 return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PRESENT));
959}
960
961static inline pte_t pgd_pte(pgd_t pgd)
962{
963 return __pte_raw(pgd_raw(pgd));
964}
965
966static inline pgd_t pte_pgd(pte_t pte)
967{
968 return __pgd_raw(pte_raw(pte));
969}
970
971static inline int pgd_bad(pgd_t pgd)
972{
973 if (radix_enabled())
974 return radix__pgd_bad(pgd);
975 return hash__pgd_bad(pgd);
976}
977
978#define pgd_access_permitted pgd_access_permitted
979static inline bool pgd_access_permitted(pgd_t pgd, bool write)
980{
981 return pte_access_permitted(pgd_pte(pgd), write);
982}
983
984extern struct page *pgd_page(pgd_t pgd);
985
986
987#define __pgtable_ptr_val(ptr) __pa(ptr)
988
989#define pmd_page_vaddr(pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)
990#define pud_page_vaddr(pud) __va(pud_val(pud) & ~PUD_MASKED_BITS)
991#define pgd_page_vaddr(pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS)
992
993#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
994#define pud_index(address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))
995#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
996#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))
997
998
999
1000
1001
1002
1003#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
1004
1005#define pud_offset(pgdp, addr) \
1006 (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr))
1007#define pmd_offset(pudp,addr) \
1008 (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))
1009#define pte_offset_kernel(dir,addr) \
1010 (((pte_t *) pmd_page_vaddr(*(dir))) + pte_index(addr))
1011
1012#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
1013
1014static inline void pte_unmap(pte_t *pte) { }
1015
1016
1017
1018#define pgd_offset_k(address) pgd_offset(&init_mm, address)
1019
1020#define pte_ERROR(e) \
1021 pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
1022#define pmd_ERROR(e) \
1023 pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
1024#define pud_ERROR(e) \
1025 pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
1026#define pgd_ERROR(e) \
1027 pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
1028
1029static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
1030{
1031 if (radix_enabled()) {
1032#if defined(CONFIG_PPC_RADIX_MMU) && defined(DEBUG_VM)
1033 unsigned long page_size = 1 << mmu_psize_defs[mmu_io_psize].shift;
1034 WARN((page_size != PAGE_SIZE), "I/O page size != PAGE_SIZE");
1035#endif
1036 return radix__map_kernel_page(ea, pa, prot, PAGE_SIZE);
1037 }
1038 return hash__map_kernel_page(ea, pa, prot);
1039}
1040
1041static inline int __meminit vmemmap_create_mapping(unsigned long start,
1042 unsigned long page_size,
1043 unsigned long phys)
1044{
1045 if (radix_enabled())
1046 return radix__vmemmap_create_mapping(start, page_size, phys);
1047 return hash__vmemmap_create_mapping(start, page_size, phys);
1048}
1049
1050#ifdef CONFIG_MEMORY_HOTPLUG
1051static inline void vmemmap_remove_mapping(unsigned long start,
1052 unsigned long page_size)
1053{
1054 if (radix_enabled())
1055 return radix__vmemmap_remove_mapping(start, page_size);
1056 return hash__vmemmap_remove_mapping(start, page_size);
1057}
1058#endif
1059
1060static inline pte_t pmd_pte(pmd_t pmd)
1061{
1062 return __pte_raw(pmd_raw(pmd));
1063}
1064
1065static inline pmd_t pte_pmd(pte_t pte)
1066{
1067 return __pmd_raw(pte_raw(pte));
1068}
1069
1070static inline pte_t *pmdp_ptep(pmd_t *pmd)
1071{
1072 return (pte_t *)pmd;
1073}
1074#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
1075#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
1076#define pmd_young(pmd) pte_young(pmd_pte(pmd))
1077#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
1078#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
1079#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
1080#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
1081#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
1082#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
1083#define pmd_mk_savedwrite(pmd) pte_pmd(pte_mk_savedwrite(pmd_pte(pmd)))
1084#define pmd_clear_savedwrite(pmd) pte_pmd(pte_clear_savedwrite(pmd_pte(pmd)))
1085
1086#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1087#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))
1088#define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd)))
1089#define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)))
1090
1091#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1092#define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd)))
1093#define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd))
1094#define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd)))
1095#endif
1096#endif
1097
1098#ifdef CONFIG_NUMA_BALANCING
1099static inline int pmd_protnone(pmd_t pmd)
1100{
1101 return pte_protnone(pmd_pte(pmd));
1102}
1103#endif
1104
1105#define pmd_write(pmd) pte_write(pmd_pte(pmd))
1106#define __pmd_write(pmd) __pte_write(pmd_pte(pmd))
1107#define pmd_savedwrite(pmd) pte_savedwrite(pmd_pte(pmd))
1108
1109#define pmd_access_permitted pmd_access_permitted
1110static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1111{
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122 if (pmd_is_serializing(pmd))
1123 return false;
1124
1125 return pte_access_permitted(pmd_pte(pmd), write);
1126}
1127
1128#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1129extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
1130extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
1131extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
1132extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1133 pmd_t *pmdp, pmd_t pmd);
1134extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
1135 pmd_t *pmd);
1136extern int hash__has_transparent_hugepage(void);
1137static inline int has_transparent_hugepage(void)
1138{
1139 if (radix_enabled())
1140 return radix__has_transparent_hugepage();
1141 return hash__has_transparent_hugepage();
1142}
1143#define has_transparent_hugepage has_transparent_hugepage
1144
1145static inline unsigned long
1146pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp,
1147 unsigned long clr, unsigned long set)
1148{
1149 if (radix_enabled())
1150 return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set);
1151 return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set);
1152}
1153
1154
1155
1156
1157
1158static inline int pmd_large(pmd_t pmd)
1159{
1160 return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
1161}
1162
1163static inline pmd_t pmd_mknotpresent(pmd_t pmd)
1164{
1165 return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT);
1166}
1167
1168
1169
1170
1171static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
1172 unsigned long addr, pmd_t *pmdp)
1173{
1174 unsigned long old;
1175
1176 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
1177 return 0;
1178 old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
1179 return ((old & _PAGE_ACCESSED) != 0);
1180}
1181
1182#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1183static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
1184 pmd_t *pmdp)
1185{
1186 if (__pmd_write((*pmdp)))
1187 pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);
1188 else if (unlikely(pmd_savedwrite(*pmdp)))
1189 pmd_hugepage_update(mm, addr, pmdp, 0, _PAGE_PRIVILEGED);
1190}
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203static inline int pmd_trans_huge(pmd_t pmd)
1204{
1205 if (!pmd_present(pmd))
1206 return false;
1207
1208 if (radix_enabled())
1209 return radix__pmd_trans_huge(pmd);
1210 return hash__pmd_trans_huge(pmd);
1211}
1212
1213#define __HAVE_ARCH_PMD_SAME
1214static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
1215{
1216 if (radix_enabled())
1217 return radix__pmd_same(pmd_a, pmd_b);
1218 return hash__pmd_same(pmd_a, pmd_b);
1219}
1220
1221static inline pmd_t pmd_mkhuge(pmd_t pmd)
1222{
1223 if (radix_enabled())
1224 return radix__pmd_mkhuge(pmd);
1225 return hash__pmd_mkhuge(pmd);
1226}
1227
1228#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1229extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1230 unsigned long address, pmd_t *pmdp,
1231 pmd_t entry, int dirty);
1232
1233#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1234extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1235 unsigned long address, pmd_t *pmdp);
1236
1237#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1238static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1239 unsigned long addr, pmd_t *pmdp)
1240{
1241 if (radix_enabled())
1242 return radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
1243 return hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
1244}
1245
1246static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1247 unsigned long address, pmd_t *pmdp)
1248{
1249 if (radix_enabled())
1250 return radix__pmdp_collapse_flush(vma, address, pmdp);
1251 return hash__pmdp_collapse_flush(vma, address, pmdp);
1252}
1253#define pmdp_collapse_flush pmdp_collapse_flush
1254
1255#define __HAVE_ARCH_PGTABLE_DEPOSIT
1256static inline void pgtable_trans_huge_deposit(struct mm_struct *mm,
1257 pmd_t *pmdp, pgtable_t pgtable)
1258{
1259 if (radix_enabled())
1260 return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
1261 return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
1262}
1263
1264#define __HAVE_ARCH_PGTABLE_WITHDRAW
1265static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm,
1266 pmd_t *pmdp)
1267{
1268 if (radix_enabled())
1269 return radix__pgtable_trans_huge_withdraw(mm, pmdp);
1270 return hash__pgtable_trans_huge_withdraw(mm, pmdp);
1271}
1272
1273#define __HAVE_ARCH_PMDP_INVALIDATE
1274extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
1275 pmd_t *pmdp);
1276
1277#define pmd_move_must_withdraw pmd_move_must_withdraw
1278struct spinlock;
1279extern int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
1280 struct spinlock *old_pmd_ptl,
1281 struct vm_area_struct *vma);
1282
1283
1284
1285
1286#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit
1287static inline bool arch_needs_pgtable_deposit(void)
1288{
1289 if (radix_enabled())
1290 return false;
1291 return true;
1292}
1293extern void serialize_against_pte_lookup(struct mm_struct *mm);
1294
1295
1296static inline pmd_t pmd_mkdevmap(pmd_t pmd)
1297{
1298 return __pmd(pmd_val(pmd) | (_PAGE_PTE | _PAGE_DEVMAP));
1299}
1300
1301static inline int pmd_devmap(pmd_t pmd)
1302{
1303 return pte_devmap(pmd_pte(pmd));
1304}
1305
1306static inline int pud_devmap(pud_t pud)
1307{
1308 return 0;
1309}
1310
1311static inline int pgd_devmap(pgd_t pgd)
1312{
1313 return 0;
1314}
1315#endif
1316
1317static inline int pud_pfn(pud_t pud)
1318{
1319
1320
1321
1322
1323
1324 BUILD_BUG();
1325 return 0;
1326}
1327#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1328pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *);
1329void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long,
1330 pte_t *, pte_t, pte_t);
1331
1332
1333
1334
1335static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_val)
1336{
1337 if (!(old_val & _PAGE_READ))
1338 return false;
1339
1340 if ((!(old_val & _PAGE_WRITE)) && (new_val & _PAGE_WRITE))
1341 return true;
1342
1343 return false;
1344}
1345
1346#endif
1347#endif
1348