1
2
3
4
5
6
7
8
9
10
11#ifndef _ASM_S390_PGTABLE_H
12#define _ASM_S390_PGTABLE_H
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#ifndef __ASSEMBLY__
30#include <linux/sched.h>
31#include <linux/mm_types.h>
32#include <linux/page-flags.h>
33#include <asm/bug.h>
34#include <asm/page.h>
35
36extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
37extern void paging_init(void);
38extern void vmem_map_init(void);
39
40
41
42
43
44#define update_mmu_cache(vma, address, ptep) do { } while (0)
45#define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
46
47
48
49
50
51
52extern unsigned long empty_zero_page;
53extern unsigned long zero_page_mask;
54
55#define ZERO_PAGE(vaddr) \
56 (virt_to_page((void *)(empty_zero_page + \
57 (((unsigned long)(vaddr)) &zero_page_mask))))
58#define __HAVE_COLOR_ZERO_PAGE
59
60
61#endif
62
63
64
65
66
67
68#ifndef CONFIG_64BIT
69# define PMD_SHIFT 20
70# define PUD_SHIFT 20
71# define PGDIR_SHIFT 20
72#else
73# define PMD_SHIFT 20
74# define PUD_SHIFT 31
75# define PGDIR_SHIFT 42
76#endif
77
78#define PMD_SIZE (1UL << PMD_SHIFT)
79#define PMD_MASK (~(PMD_SIZE-1))
80#define PUD_SIZE (1UL << PUD_SHIFT)
81#define PUD_MASK (~(PUD_SIZE-1))
82#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
83#define PGDIR_MASK (~(PGDIR_SIZE-1))
84
85
86
87
88
89
90
91#define PTRS_PER_PTE 256
92#ifndef CONFIG_64BIT
93#define PTRS_PER_PMD 1
94#define PTRS_PER_PUD 1
95#else
96#define PTRS_PER_PMD 2048
97#define PTRS_PER_PUD 2048
98#endif
99#define PTRS_PER_PGD 2048
100
101#define FIRST_USER_ADDRESS 0
102
103#define pte_ERROR(e) \
104 printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
105#define pmd_ERROR(e) \
106 printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
107#define pud_ERROR(e) \
108 printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
109#define pgd_ERROR(e) \
110 printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))
111
112#ifndef __ASSEMBLY__
113
114
115
116
117
118
119
120
121extern unsigned long VMALLOC_START;
122extern unsigned long VMALLOC_END;
123extern struct page *vmemmap;
124
125#define VMEM_MAX_PHYS ((unsigned long) vmemmap)
126
127#ifdef CONFIG_64BIT
128extern unsigned long MODULES_VADDR;
129extern unsigned long MODULES_END;
130#define MODULES_VADDR MODULES_VADDR
131#define MODULES_END MODULES_END
132#define MODULES_LEN (1UL << 31)
133#endif
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219#define _PAGE_CO 0x100
220#define _PAGE_RO 0x200
221#define _PAGE_INVALID 0x400
222
223
224#define _PAGE_SWT 0x001
225#define _PAGE_SWX 0x002
226#define _PAGE_SWC 0x004
227#define _PAGE_SWR 0x008
228#define _PAGE_SWW 0x010
229#define _PAGE_SPECIAL 0x020
230#define __HAVE_ARCH_PTE_SPECIAL
231
232
233#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \
234 _PAGE_SWC | _PAGE_SWR)
235
236
237#define _PAGE_TYPE_EMPTY 0x400
238#define _PAGE_TYPE_NONE 0x401
239#define _PAGE_TYPE_SWAP 0x403
240#define _PAGE_TYPE_FILE 0x601
241#define _PAGE_TYPE_RO 0x200
242#define _PAGE_TYPE_RW 0x000
243
244
245
246
247
248#define _HPAGE_TYPE_EMPTY 0x020
249#define _HPAGE_TYPE_NONE 0x220
250#define _HPAGE_TYPE_RO 0x200
251#define _HPAGE_TYPE_RW 0x000
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279#ifndef CONFIG_64BIT
280
281
282#define _ASCE_SPACE_SWITCH 0x80000000UL
283#define _ASCE_ORIGIN_MASK 0x7ffff000UL
284#define _ASCE_PRIVATE_SPACE 0x100
285#define _ASCE_ALT_EVENT 0x80
286#define _ASCE_TABLE_LENGTH 0x7f
287
288
289#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL
290#define _SEGMENT_ENTRY_RO 0x200
291#define _SEGMENT_ENTRY_INV 0x20
292#define _SEGMENT_ENTRY_COMMON 0x10
293#define _SEGMENT_ENTRY_PTL 0x0f
294
295#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL)
296#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
297
298
299#define PGSTE_ACC_BITS 0xf0000000UL
300#define PGSTE_FP_BIT 0x08000000UL
301#define PGSTE_PCL_BIT 0x00800000UL
302#define PGSTE_HR_BIT 0x00400000UL
303#define PGSTE_HC_BIT 0x00200000UL
304#define PGSTE_GR_BIT 0x00040000UL
305#define PGSTE_GC_BIT 0x00020000UL
306#define PGSTE_UR_BIT 0x00008000UL
307#define PGSTE_UC_BIT 0x00004000UL
308#define PGSTE_IN_BIT 0x00002000UL
309
310#else
311
312
313#define _ASCE_ORIGIN ~0xfffUL
314#define _ASCE_PRIVATE_SPACE 0x100
315#define _ASCE_ALT_EVENT 0x80
316#define _ASCE_SPACE_SWITCH 0x40
317#define _ASCE_REAL_SPACE 0x20
318#define _ASCE_TYPE_MASK 0x0c
319#define _ASCE_TYPE_REGION1 0x0c
320#define _ASCE_TYPE_REGION2 0x08
321#define _ASCE_TYPE_REGION3 0x04
322#define _ASCE_TYPE_SEGMENT 0x00
323#define _ASCE_TABLE_LENGTH 0x03
324
325
326#define _REGION_ENTRY_ORIGIN ~0xfffUL
327#define _REGION_ENTRY_RO 0x200
328#define _REGION_ENTRY_INV 0x20
329#define _REGION_ENTRY_TYPE_MASK 0x0c
330#define _REGION_ENTRY_TYPE_R1 0x0c
331#define _REGION_ENTRY_TYPE_R2 0x08
332#define _REGION_ENTRY_TYPE_R3 0x04
333#define _REGION_ENTRY_LENGTH 0x03
334
335#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
336#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV)
337#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
338#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV)
339#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
340#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV)
341
342#define _REGION3_ENTRY_LARGE 0x400
343#define _REGION3_ENTRY_RO 0x200
344#define _REGION3_ENTRY_CO 0x100
345
346
347#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL
348#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL
349#define _SEGMENT_ENTRY_RO 0x200
350#define _SEGMENT_ENTRY_INV 0x20
351
352#define _SEGMENT_ENTRY (0)
353#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
354
355#define _SEGMENT_ENTRY_LARGE 0x400
356#define _SEGMENT_ENTRY_CO 0x100
357#define _SEGMENT_ENTRY_SPLIT_BIT 0
358#define _SEGMENT_ENTRY_SPLIT (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
359
360
361#define _SEGMENT_CHG_MASK (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
362 | _SEGMENT_ENTRY_SPLIT | _SEGMENT_ENTRY_CO)
363
364
365#define PGSTE_ACC_BITS 0xf000000000000000UL
366#define PGSTE_FP_BIT 0x0800000000000000UL
367#define PGSTE_PCL_BIT 0x0080000000000000UL
368#define PGSTE_HR_BIT 0x0040000000000000UL
369#define PGSTE_HC_BIT 0x0020000000000000UL
370#define PGSTE_GR_BIT 0x0004000000000000UL
371#define PGSTE_GC_BIT 0x0002000000000000UL
372#define PGSTE_UR_BIT 0x0000800000000000UL
373#define PGSTE_UC_BIT 0x0000400000000000UL
374#define PGSTE_IN_BIT 0x0000200000000000UL
375
376#endif
377
378
379
380
381
382
383#define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
384 _ASCE_ALT_EVENT)
385
386
387
388
389#define PAGE_NONE __pgprot(_PAGE_TYPE_NONE)
390#define PAGE_RO __pgprot(_PAGE_TYPE_RO)
391#define PAGE_RW __pgprot(_PAGE_TYPE_RO | _PAGE_SWW)
392#define PAGE_RWC __pgprot(_PAGE_TYPE_RW | _PAGE_SWW | _PAGE_SWC)
393
394#define PAGE_KERNEL PAGE_RWC
395#define PAGE_SHARED PAGE_KERNEL
396#define PAGE_COPY PAGE_RO
397
398
399
400
401
402
403
404#define __P000 PAGE_NONE
405#define __P001 PAGE_RO
406#define __P010 PAGE_RO
407#define __P011 PAGE_RO
408#define __P100 PAGE_RO
409#define __P101 PAGE_RO
410#define __P110 PAGE_RO
411#define __P111 PAGE_RO
412
413#define __S000 PAGE_NONE
414#define __S001 PAGE_RO
415#define __S010 PAGE_RW
416#define __S011 PAGE_RW
417#define __S100 PAGE_RO
418#define __S101 PAGE_RO
419#define __S110 PAGE_RW
420#define __S111 PAGE_RW
421
422
423
424
425#define SEGMENT_NONE __pgprot(_HPAGE_TYPE_NONE)
426#define SEGMENT_RO __pgprot(_HPAGE_TYPE_RO)
427#define SEGMENT_RW __pgprot(_HPAGE_TYPE_RW)
428
429static inline int mm_exclusive(struct mm_struct *mm)
430{
431 return likely(mm == current->active_mm &&
432 atomic_read(&mm->context.attach_count) <= 1);
433}
434
435static inline int mm_has_pgste(struct mm_struct *mm)
436{
437#ifdef CONFIG_PGSTE
438 if (unlikely(mm->context.has_pgste))
439 return 1;
440#endif
441 return 0;
442}
443
444
445
446#ifndef CONFIG_64BIT
447
448static inline int pgd_present(pgd_t pgd) { return 1; }
449static inline int pgd_none(pgd_t pgd) { return 0; }
450static inline int pgd_bad(pgd_t pgd) { return 0; }
451
452static inline int pud_present(pud_t pud) { return 1; }
453static inline int pud_none(pud_t pud) { return 0; }
454static inline int pud_large(pud_t pud) { return 0; }
455static inline int pud_bad(pud_t pud) { return 0; }
456
457#else
458
459static inline int pgd_present(pgd_t pgd)
460{
461 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
462 return 1;
463 return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
464}
465
466static inline int pgd_none(pgd_t pgd)
467{
468 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
469 return 0;
470 return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL;
471}
472
473static inline int pgd_bad(pgd_t pgd)
474{
475
476
477
478
479
480 unsigned long mask =
481 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
482 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
483 return (pgd_val(pgd) & mask) != 0;
484}
485
486static inline int pud_present(pud_t pud)
487{
488 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
489 return 1;
490 return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
491}
492
493static inline int pud_none(pud_t pud)
494{
495 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
496 return 0;
497 return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL;
498}
499
500static inline int pud_large(pud_t pud)
501{
502 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
503 return 0;
504 return !!(pud_val(pud) & _REGION3_ENTRY_LARGE);
505}
506
507static inline int pud_bad(pud_t pud)
508{
509
510
511
512
513
514 unsigned long mask =
515 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
516 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
517 return (pud_val(pud) & mask) != 0;
518}
519
520#endif
521
522static inline int pmd_present(pmd_t pmd)
523{
524 unsigned long mask = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO;
525 return (pmd_val(pmd) & mask) == _HPAGE_TYPE_NONE ||
526 !(pmd_val(pmd) & _SEGMENT_ENTRY_INV);
527}
528
529static inline int pmd_none(pmd_t pmd)
530{
531 return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) &&
532 !(pmd_val(pmd) & _SEGMENT_ENTRY_RO);
533}
534
535static inline int pmd_large(pmd_t pmd)
536{
537#ifdef CONFIG_64BIT
538 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
539#else
540 return 0;
541#endif
542}
543
544static inline int pmd_bad(pmd_t pmd)
545{
546 unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV;
547 return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
548}
549
550#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
551extern void pmdp_splitting_flush(struct vm_area_struct *vma,
552 unsigned long addr, pmd_t *pmdp);
553
554#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
555extern int pmdp_set_access_flags(struct vm_area_struct *vma,
556 unsigned long address, pmd_t *pmdp,
557 pmd_t entry, int dirty);
558
559#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
560extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
561 unsigned long address, pmd_t *pmdp);
562
563#define __HAVE_ARCH_PMD_WRITE
564static inline int pmd_write(pmd_t pmd)
565{
566 return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0;
567}
568
569static inline int pmd_young(pmd_t pmd)
570{
571 return 0;
572}
573
574static inline int pte_none(pte_t pte)
575{
576 return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
577}
578
579static inline int pte_present(pte_t pte)
580{
581 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT | _PAGE_SWX;
582 return (pte_val(pte) & mask) == _PAGE_TYPE_NONE ||
583 (!(pte_val(pte) & _PAGE_INVALID) &&
584 !(pte_val(pte) & _PAGE_SWT));
585}
586
587static inline int pte_file(pte_t pte)
588{
589 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT;
590 return (pte_val(pte) & mask) == _PAGE_TYPE_FILE;
591}
592
593static inline int pte_special(pte_t pte)
594{
595 return (pte_val(pte) & _PAGE_SPECIAL);
596}
597
598#define __HAVE_ARCH_PTE_SAME
599static inline int pte_same(pte_t a, pte_t b)
600{
601 return pte_val(a) == pte_val(b);
602}
603
604static inline pgste_t pgste_get_lock(pte_t *ptep)
605{
606 unsigned long new = 0;
607#ifdef CONFIG_PGSTE
608 unsigned long old;
609
610 preempt_disable();
611 asm(
612 " lg %0,%2\n"
613 "0: lgr %1,%0\n"
614 " nihh %0,0xff7f\n"
615 " oihh %1,0x0080\n"
616 " csg %0,%1,%2\n"
617 " jl 0b\n"
618 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
619 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
620#endif
621 return __pgste(new);
622}
623
624static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
625{
626#ifdef CONFIG_PGSTE
627 asm(
628 " nihh %1,0xff7f\n"
629 " stg %1,%0\n"
630 : "=Q" (ptep[PTRS_PER_PTE])
631 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
632 : "cc", "memory");
633 preempt_enable();
634#endif
635}
636
637static inline void pgste_set(pte_t *ptep, pgste_t pgste)
638{
639#ifdef CONFIG_PGSTE
640 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
641#endif
642}
643
644static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
645{
646#ifdef CONFIG_PGSTE
647 unsigned long address, bits;
648 unsigned char skey;
649
650 if (pte_val(*ptep) & _PAGE_INVALID)
651 return pgste;
652 address = pte_val(*ptep) & PAGE_MASK;
653 skey = page_get_storage_key(address);
654 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
655
656 if (bits & _PAGE_CHANGED)
657 page_set_storage_key(address, skey ^ bits, 0);
658 else if (bits)
659 page_reset_referenced(address);
660
661 pgste_val(pgste) |= bits << 48;
662
663 bits |= (pgste_val(pgste) & (PGSTE_HR_BIT | PGSTE_HC_BIT)) >> 52;
664
665 pgste_val(pgste) |= bits << 45;
666
667 pgste_val(pgste) &= ~(PGSTE_HR_BIT | PGSTE_HC_BIT);
668 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
669
670 pgste_val(pgste) |=
671 (unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
672
673 pte_val(*ptep) |= (bits & _PAGE_REFERENCED) << 1;
674#endif
675 return pgste;
676
677}
678
679static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste)
680{
681#ifdef CONFIG_PGSTE
682 int young;
683
684 if (pte_val(*ptep) & _PAGE_INVALID)
685 return pgste;
686
687 young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
688 if (young)
689 pgste_val(pgste) |= PGSTE_GR_BIT;
690
691 if (pgste_val(pgste) & PGSTE_HR_BIT) {
692 pgste_val(pgste) &= ~PGSTE_HR_BIT;
693 young = 1;
694 }
695
696 if (young) {
697 pgste_val(pgste) |= PGSTE_UR_BIT;
698 pte_val(*ptep) |= _PAGE_SWR;
699 }
700#endif
701 return pgste;
702}
703
704static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry)
705{
706#ifdef CONFIG_PGSTE
707 unsigned long address;
708 unsigned long nkey;
709
710 if (pte_val(entry) & _PAGE_INVALID)
711 return;
712 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
713 address = pte_val(entry) & PAGE_MASK;
714
715
716
717
718
719 nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
720 page_set_storage_key(address, nkey, 0);
721#endif
722}
723
724static inline void pgste_set_pte(pte_t *ptep, pte_t entry)
725{
726 if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_SWW)) {
727
728
729
730
731 pte_val(entry) |= _PAGE_SWC;
732 pte_val(entry) &= ~_PAGE_RO;
733 }
734 *ptep = entry;
735}
736
737
738
739
740
741
742
743
744struct gmap {
745 struct list_head list;
746 struct mm_struct *mm;
747 unsigned long *table;
748 unsigned long asce;
749 void *private;
750 struct list_head crst_list;
751};
752
753
754
755
756
757
758
759struct gmap_rmap {
760 struct list_head list;
761 struct gmap *gmap;
762 unsigned long *entry;
763 unsigned long vmaddr;
764};
765
766
767
768
769
770
771struct gmap_pgtable {
772 unsigned long vmaddr;
773 struct list_head mapper;
774};
775
776
777
778
779
780struct gmap_notifier {
781 struct list_head list;
782 void (*notifier_call)(struct gmap *gmap, unsigned long address);
783};
784
785struct gmap *gmap_alloc(struct mm_struct *mm);
786void gmap_free(struct gmap *gmap);
787void gmap_enable(struct gmap *gmap);
788void gmap_disable(struct gmap *gmap);
789int gmap_map_segment(struct gmap *gmap, unsigned long from,
790 unsigned long to, unsigned long len);
791int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len);
792unsigned long __gmap_translate(unsigned long address, struct gmap *);
793unsigned long gmap_translate(unsigned long address, struct gmap *);
794unsigned long __gmap_fault(unsigned long address, struct gmap *);
795unsigned long gmap_fault(unsigned long address, struct gmap *);
796void gmap_discard(unsigned long from, unsigned long to, struct gmap *);
797
798void gmap_register_ipte_notifier(struct gmap_notifier *);
799void gmap_unregister_ipte_notifier(struct gmap_notifier *);
800int gmap_ipte_notify(struct gmap *, unsigned long start, unsigned long len);
801void gmap_do_ipte_notify(struct mm_struct *, unsigned long addr, pte_t *);
802
803static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
804 unsigned long addr,
805 pte_t *ptep, pgste_t pgste)
806{
807#ifdef CONFIG_PGSTE
808 if (pgste_val(pgste) & PGSTE_IN_BIT) {
809 pgste_val(pgste) &= ~PGSTE_IN_BIT;
810 gmap_do_ipte_notify(mm, addr, ptep);
811 }
812#endif
813 return pgste;
814}
815
816
817
818
819
820
821static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
822 pte_t *ptep, pte_t entry)
823{
824 pgste_t pgste;
825
826 if (mm_has_pgste(mm)) {
827 pgste = pgste_get_lock(ptep);
828 pgste_set_key(ptep, pgste, entry);
829 pgste_set_pte(ptep, entry);
830 pgste_set_unlock(ptep, pgste);
831 } else {
832 if (!(pte_val(entry) & _PAGE_INVALID) && MACHINE_HAS_EDAT1)
833 pte_val(entry) |= _PAGE_CO;
834 *ptep = entry;
835 }
836}
837
838
839
840
841
842static inline int pte_write(pte_t pte)
843{
844 return (pte_val(pte) & _PAGE_SWW) != 0;
845}
846
847static inline int pte_dirty(pte_t pte)
848{
849 return (pte_val(pte) & _PAGE_SWC) != 0;
850}
851
852static inline int pte_young(pte_t pte)
853{
854#ifdef CONFIG_PGSTE
855 if (pte_val(pte) & _PAGE_SWR)
856 return 1;
857#endif
858 return 0;
859}
860
861
862
863
864
865static inline void pgd_clear(pgd_t *pgd)
866{
867#ifdef CONFIG_64BIT
868 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
869 pgd_val(*pgd) = _REGION2_ENTRY_EMPTY;
870#endif
871}
872
873static inline void pud_clear(pud_t *pud)
874{
875#ifdef CONFIG_64BIT
876 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
877 pud_val(*pud) = _REGION3_ENTRY_EMPTY;
878#endif
879}
880
881static inline void pmd_clear(pmd_t *pmdp)
882{
883 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
884}
885
886static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
887{
888 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
889}
890
891
892
893
894
895static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
896{
897 pte_val(pte) &= _PAGE_CHG_MASK;
898 pte_val(pte) |= pgprot_val(newprot);
899 if ((pte_val(pte) & _PAGE_SWC) && (pte_val(pte) & _PAGE_SWW))
900 pte_val(pte) &= ~_PAGE_RO;
901 return pte;
902}
903
904static inline pte_t pte_wrprotect(pte_t pte)
905{
906 pte_val(pte) &= ~_PAGE_SWW;
907
908 if (!(pte_val(pte) & _PAGE_INVALID))
909 pte_val(pte) |= _PAGE_RO;
910 return pte;
911}
912
913static inline pte_t pte_mkwrite(pte_t pte)
914{
915 pte_val(pte) |= _PAGE_SWW;
916 if (pte_val(pte) & _PAGE_SWC)
917 pte_val(pte) &= ~_PAGE_RO;
918 return pte;
919}
920
921static inline pte_t pte_mkclean(pte_t pte)
922{
923 pte_val(pte) &= ~_PAGE_SWC;
924
925 if (!(pte_val(pte) & _PAGE_INVALID))
926 pte_val(pte) |= _PAGE_RO;
927 return pte;
928}
929
930static inline pte_t pte_mkdirty(pte_t pte)
931{
932 pte_val(pte) |= _PAGE_SWC;
933 if (pte_val(pte) & _PAGE_SWW)
934 pte_val(pte) &= ~_PAGE_RO;
935 return pte;
936}
937
938static inline pte_t pte_mkold(pte_t pte)
939{
940#ifdef CONFIG_PGSTE
941 pte_val(pte) &= ~_PAGE_SWR;
942#endif
943 return pte;
944}
945
946static inline pte_t pte_mkyoung(pte_t pte)
947{
948 return pte;
949}
950
951static inline pte_t pte_mkspecial(pte_t pte)
952{
953 pte_val(pte) |= _PAGE_SPECIAL;
954 return pte;
955}
956
957#ifdef CONFIG_HUGETLB_PAGE
958static inline pte_t pte_mkhuge(pte_t pte)
959{
960 pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO);
961 return pte;
962}
963#endif
964
965
966
967
968static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,
969 pte_t *ptep)
970{
971 pgste_t pgste;
972 int dirty = 0;
973
974 if (mm_has_pgste(mm)) {
975 pgste = pgste_get_lock(ptep);
976 pgste = pgste_update_all(ptep, pgste);
977 dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
978 pgste_val(pgste) &= ~PGSTE_UC_BIT;
979 pgste_set_unlock(ptep, pgste);
980 return dirty;
981 }
982 return dirty;
983}
984
985
986
987
988static inline int ptep_test_and_clear_user_young(struct mm_struct *mm,
989 pte_t *ptep)
990{
991 pgste_t pgste;
992 int young = 0;
993
994 if (mm_has_pgste(mm)) {
995 pgste = pgste_get_lock(ptep);
996 pgste = pgste_update_young(ptep, pgste);
997 young = !!(pgste_val(pgste) & PGSTE_UR_BIT);
998 pgste_val(pgste) &= ~PGSTE_UR_BIT;
999 pgste_set_unlock(ptep, pgste);
1000 }
1001 return young;
1002}
1003
1004#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1005static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1006 unsigned long addr, pte_t *ptep)
1007{
1008 pgste_t pgste;
1009 pte_t pte;
1010
1011 if (mm_has_pgste(vma->vm_mm)) {
1012 pgste = pgste_get_lock(ptep);
1013 pgste = pgste_update_young(ptep, pgste);
1014 pte = *ptep;
1015 *ptep = pte_mkold(pte);
1016 pgste_set_unlock(ptep, pgste);
1017 return pte_young(pte);
1018 }
1019 return 0;
1020}
1021
1022#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1023static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1024 unsigned long address, pte_t *ptep)
1025{
1026
1027
1028
1029
1030 return ptep_test_and_clear_young(vma, address, ptep);
1031}
1032
1033static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
1034{
1035 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1036#ifndef CONFIG_64BIT
1037
1038 pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
1039#else
1040
1041 pte_t *pto = ptep;
1042#endif
1043 asm volatile(
1044 " ipte %2,%3"
1045 : "=m" (*ptep) : "m" (*ptep),
1046 "a" (pto), "a" (address));
1047 }
1048}
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1064static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1065 unsigned long address, pte_t *ptep)
1066{
1067 pgste_t pgste;
1068 pte_t pte;
1069
1070 mm->context.flush_mm = 1;
1071 if (mm_has_pgste(mm)) {
1072 pgste = pgste_get_lock(ptep);
1073 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1074 }
1075
1076 pte = *ptep;
1077 if (!mm_exclusive(mm))
1078 __ptep_ipte(address, ptep);
1079 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1080
1081 if (mm_has_pgste(mm)) {
1082 pgste = pgste_update_all(&pte, pgste);
1083 pgste_set_unlock(ptep, pgste);
1084 }
1085 return pte;
1086}
1087
1088#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1089static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
1090 unsigned long address,
1091 pte_t *ptep)
1092{
1093 pgste_t pgste;
1094 pte_t pte;
1095
1096 mm->context.flush_mm = 1;
1097 if (mm_has_pgste(mm)) {
1098 pgste = pgste_get_lock(ptep);
1099 pgste_ipte_notify(mm, address, ptep, pgste);
1100 }
1101
1102 pte = *ptep;
1103 if (!mm_exclusive(mm))
1104 __ptep_ipte(address, ptep);
1105
1106 if (mm_has_pgste(mm)) {
1107 pgste = pgste_update_all(&pte, pgste);
1108 pgste_set(ptep, pgste);
1109 }
1110 return pte;
1111}
1112
1113static inline void ptep_modify_prot_commit(struct mm_struct *mm,
1114 unsigned long address,
1115 pte_t *ptep, pte_t pte)
1116{
1117 pgste_t pgste;
1118
1119 if (mm_has_pgste(mm)) {
1120 pgste = *(pgste_t *)(ptep + PTRS_PER_PTE);
1121 pgste_set_key(ptep, pgste, pte);
1122 pgste_set_pte(ptep, pte);
1123 pgste_set_unlock(ptep, pgste);
1124 } else
1125 *ptep = pte;
1126}
1127
1128#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
1129static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1130 unsigned long address, pte_t *ptep)
1131{
1132 pgste_t pgste;
1133 pte_t pte;
1134
1135 if (mm_has_pgste(vma->vm_mm)) {
1136 pgste = pgste_get_lock(ptep);
1137 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
1138 }
1139
1140 pte = *ptep;
1141 __ptep_ipte(address, ptep);
1142 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1143
1144 if (mm_has_pgste(vma->vm_mm)) {
1145 pgste = pgste_update_all(&pte, pgste);
1146 pgste_set_unlock(ptep, pgste);
1147 }
1148 return pte;
1149}
1150
1151
1152
1153
1154
1155
1156
1157
1158#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1159static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1160 unsigned long address,
1161 pte_t *ptep, int full)
1162{
1163 pgste_t pgste;
1164 pte_t pte;
1165
1166 if (mm_has_pgste(mm)) {
1167 pgste = pgste_get_lock(ptep);
1168 if (!full)
1169 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1170 }
1171
1172 pte = *ptep;
1173 if (!full)
1174 __ptep_ipte(address, ptep);
1175 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1176
1177 if (mm_has_pgste(mm)) {
1178 pgste = pgste_update_all(&pte, pgste);
1179 pgste_set_unlock(ptep, pgste);
1180 }
1181 return pte;
1182}
1183
1184#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1185static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,
1186 unsigned long address, pte_t *ptep)
1187{
1188 pgste_t pgste;
1189 pte_t pte = *ptep;
1190
1191 if (pte_write(pte)) {
1192 mm->context.flush_mm = 1;
1193 if (mm_has_pgste(mm)) {
1194 pgste = pgste_get_lock(ptep);
1195 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1196 }
1197
1198 if (!mm_exclusive(mm))
1199 __ptep_ipte(address, ptep);
1200 pte = pte_wrprotect(pte);
1201
1202 if (mm_has_pgste(mm)) {
1203 pgste_set_pte(ptep, pte);
1204 pgste_set_unlock(ptep, pgste);
1205 } else
1206 *ptep = pte;
1207 }
1208 return pte;
1209}
1210
1211#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1212static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1213 unsigned long address, pte_t *ptep,
1214 pte_t entry, int dirty)
1215{
1216 pgste_t pgste;
1217
1218 if (pte_same(*ptep, entry))
1219 return 0;
1220 if (mm_has_pgste(vma->vm_mm)) {
1221 pgste = pgste_get_lock(ptep);
1222 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
1223 }
1224
1225 __ptep_ipte(address, ptep);
1226
1227 if (mm_has_pgste(vma->vm_mm)) {
1228 pgste_set_pte(ptep, entry);
1229 pgste_set_unlock(ptep, pgste);
1230 } else
1231 *ptep = entry;
1232 return 1;
1233}
1234
1235
1236
1237
1238
1239static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1240{
1241 pte_t __pte;
1242 pte_val(__pte) = physpage + pgprot_val(pgprot);
1243 return __pte;
1244}
1245
1246static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1247{
1248 unsigned long physpage = page_to_phys(page);
1249 pte_t __pte = mk_pte_phys(physpage, pgprot);
1250
1251 if ((pte_val(__pte) & _PAGE_SWW) && PageDirty(page)) {
1252 pte_val(__pte) |= _PAGE_SWC;
1253 pte_val(__pte) &= ~_PAGE_RO;
1254 }
1255 return __pte;
1256}
1257
1258#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1259#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1260#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1261#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
1262
1263#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
1264#define pgd_offset_k(address) pgd_offset(&init_mm, address)
1265
1266#ifndef CONFIG_64BIT
1267
1268#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1269#define pud_deref(pmd) ({ BUG(); 0UL; })
1270#define pgd_deref(pmd) ({ BUG(); 0UL; })
1271
1272#define pud_offset(pgd, address) ((pud_t *) pgd)
1273#define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address))
1274
1275#else
1276
1277#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1278#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
1279#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
1280
1281static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
1282{
1283 pud_t *pud = (pud_t *) pgd;
1284 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
1285 pud = (pud_t *) pgd_deref(*pgd);
1286 return pud + pud_index(address);
1287}
1288
1289static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
1290{
1291 pmd_t *pmd = (pmd_t *) pud;
1292 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
1293 pmd = (pmd_t *) pud_deref(*pud);
1294 return pmd + pmd_index(address);
1295}
1296
1297#endif
1298
1299#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
1300#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1301#define pte_page(x) pfn_to_page(pte_pfn(x))
1302
1303#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
1304
1305
1306#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
1307#define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
1308#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
1309#define pte_unmap(pte) do { } while (0)
1310
1311static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
1312{
1313 unsigned long sto = (unsigned long) pmdp -
1314 pmd_index(address) * sizeof(pmd_t);
1315
1316 if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
1317 asm volatile(
1318 " .insn rrf,0xb98e0000,%2,%3,0,0"
1319 : "=m" (*pmdp)
1320 : "m" (*pmdp), "a" (sto),
1321 "a" ((address & HPAGE_MASK))
1322 : "cc"
1323 );
1324 }
1325}
1326
1327#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
1328static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1329{
1330
1331
1332
1333
1334 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1335 return pgprot_val(SEGMENT_NONE);
1336 if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1337 return pgprot_val(SEGMENT_RO);
1338 return pgprot_val(SEGMENT_RW);
1339}
1340
1341static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1342{
1343 pmd_val(pmd) &= _SEGMENT_CHG_MASK;
1344 pmd_val(pmd) |= massage_pgprot_pmd(newprot);
1345 return pmd;
1346}
1347
1348static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1349{
1350 pmd_t __pmd;
1351 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
1352 return __pmd;
1353}
1354
1355static inline pmd_t pmd_mkwrite(pmd_t pmd)
1356{
1357
1358 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_INV))
1359 pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO;
1360 return pmd;
1361}
1362#endif
1363
1364#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1365
1366#define __HAVE_ARCH_PGTABLE_DEPOSIT
1367extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1368 pgtable_t pgtable);
1369
1370#define __HAVE_ARCH_PGTABLE_WITHDRAW
1371extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1372
1373static inline int pmd_trans_splitting(pmd_t pmd)
1374{
1375 return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
1376}
1377
1378static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1379 pmd_t *pmdp, pmd_t entry)
1380{
1381 if (!(pmd_val(entry) & _SEGMENT_ENTRY_INV) && MACHINE_HAS_EDAT1)
1382 pmd_val(entry) |= _SEGMENT_ENTRY_CO;
1383 *pmdp = entry;
1384}
1385
1386static inline pmd_t pmd_mkhuge(pmd_t pmd)
1387{
1388 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
1389 return pmd;
1390}
1391
1392static inline pmd_t pmd_wrprotect(pmd_t pmd)
1393{
1394 pmd_val(pmd) |= _SEGMENT_ENTRY_RO;
1395 return pmd;
1396}
1397
1398static inline pmd_t pmd_mkdirty(pmd_t pmd)
1399{
1400
1401 return pmd;
1402}
1403
1404static inline pmd_t pmd_mkold(pmd_t pmd)
1405{
1406
1407 return pmd;
1408}
1409
1410static inline pmd_t pmd_mkyoung(pmd_t pmd)
1411{
1412
1413 return pmd;
1414}
1415
1416#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1417static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1418 unsigned long address, pmd_t *pmdp)
1419{
1420 unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK;
1421 long tmp, rc;
1422 int counter;
1423
1424 rc = 0;
1425 if (MACHINE_HAS_RRBM) {
1426 counter = PTRS_PER_PTE >> 6;
1427 asm volatile(
1428 "0: .insn rre,0xb9ae0000,%0,%3\n"
1429 " ogr %1,%0\n"
1430 " la %3,0(%4,%3)\n"
1431 " brct %2,0b\n"
1432 : "=&d" (tmp), "+&d" (rc), "+d" (counter),
1433 "+a" (pmd_addr)
1434 : "a" (64 * 4096UL) : "cc");
1435 rc = !!rc;
1436 } else {
1437 counter = PTRS_PER_PTE;
1438 asm volatile(
1439 "0: rrbe 0,%2\n"
1440 " la %2,0(%3,%2)\n"
1441 " brc 12,1f\n"
1442 " lhi %0,1\n"
1443 "1: brct %1,0b\n"
1444 : "+d" (rc), "+d" (counter), "+a" (pmd_addr)
1445 : "a" (4096UL) : "cc");
1446 }
1447 return rc;
1448}
1449
1450#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
1451static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
1452 unsigned long address, pmd_t *pmdp)
1453{
1454 pmd_t pmd = *pmdp;
1455
1456 __pmd_idte(address, pmdp);
1457 pmd_clear(pmdp);
1458 return pmd;
1459}
1460
1461#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
1462static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
1463 unsigned long address, pmd_t *pmdp)
1464{
1465 return pmdp_get_and_clear(vma->vm_mm, address, pmdp);
1466}
1467
1468#define __HAVE_ARCH_PMDP_INVALIDATE
1469static inline void pmdp_invalidate(struct vm_area_struct *vma,
1470 unsigned long address, pmd_t *pmdp)
1471{
1472 __pmd_idte(address, pmdp);
1473}
1474
1475#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1476static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1477 unsigned long address, pmd_t *pmdp)
1478{
1479 pmd_t pmd = *pmdp;
1480
1481 if (pmd_write(pmd)) {
1482 __pmd_idte(address, pmdp);
1483 set_pmd_at(mm, address, pmdp, pmd_wrprotect(pmd));
1484 }
1485}
1486
1487#define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
1488#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
1489
1490static inline int pmd_trans_huge(pmd_t pmd)
1491{
1492 return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
1493}
1494
1495static inline int has_transparent_hugepage(void)
1496{
1497 return MACHINE_HAS_HPAGE ? 1 : 0;
1498}
1499
1500static inline unsigned long pmd_pfn(pmd_t pmd)
1501{
1502 return pmd_val(pmd) >> PAGE_SHIFT;
1503}
1504#endif
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541#ifndef CONFIG_64BIT
1542#define __SWP_OFFSET_MASK (~0UL >> 12)
1543#else
1544#define __SWP_OFFSET_MASK (~0UL >> 11)
1545#endif
1546static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1547{
1548 pte_t pte;
1549 offset &= __SWP_OFFSET_MASK;
1550 pte_val(pte) = _PAGE_TYPE_SWAP | ((type & 0x1f) << 2) |
1551 ((offset & 1UL) << 7) | ((offset & ~1UL) << 11);
1552 return pte;
1553}
1554
1555#define __swp_type(entry) (((entry).val >> 2) & 0x1f)
1556#define __swp_offset(entry) (((entry).val >> 11) | (((entry).val >> 7) & 1))
1557#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
1558
1559#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1560#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
1561
1562#ifndef CONFIG_64BIT
1563# define PTE_FILE_MAX_BITS 26
1564#else
1565# define PTE_FILE_MAX_BITS 59
1566#endif
1567
1568#define pte_to_pgoff(__pte) \
1569 ((((__pte).pte >> 12) << 7) + (((__pte).pte >> 1) & 0x7f))
1570
1571#define pgoff_to_pte(__off) \
1572 ((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \
1573 | _PAGE_TYPE_FILE })
1574
1575#endif
1576
1577#define kern_addr_valid(addr) (1)
1578
1579extern int vmem_add_mapping(unsigned long start, unsigned long size);
1580extern int vmem_remove_mapping(unsigned long start, unsigned long size);
1581extern int s390_enable_sie(void);
1582
1583
1584
1585
1586static inline void pgtable_cache_init(void) { }
1587static inline void check_pgt_cache(void) { }
1588
1589#include <asm-generic/pgtable.h>
1590
1591#endif
1592