1
2
3
4
5
6
7
8
9
10
11#ifndef _ASM_S390_PGTABLE_H
12#define _ASM_S390_PGTABLE_H
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#ifndef __ASSEMBLY__
30#include <linux/sched.h>
31#include <linux/mm_types.h>
32#include <linux/page-flags.h>
33#include <asm/bug.h>
34#include <asm/page.h>
35
36extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
37extern void paging_init(void);
38extern void vmem_map_init(void);
39
40
41
42
43
44#define update_mmu_cache(vma, address, ptep) do { } while (0)
45#define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
46
47
48
49
50
51
52extern unsigned long empty_zero_page;
53extern unsigned long zero_page_mask;
54
55#define ZERO_PAGE(vaddr) \
56 (virt_to_page((void *)(empty_zero_page + \
57 (((unsigned long)(vaddr)) &zero_page_mask))))
58#define __HAVE_COLOR_ZERO_PAGE
59
60
61#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \
62 remap_pfn_range(vma, vaddr, pfn, size, prot)
63
64#endif
65
66
67
68
69
70
71#ifndef CONFIG_64BIT
72# define PMD_SHIFT 20
73# define PUD_SHIFT 20
74# define PGDIR_SHIFT 20
75#else
76# define PMD_SHIFT 20
77# define PUD_SHIFT 31
78# define PGDIR_SHIFT 42
79#endif
80
81#define PMD_SIZE (1UL << PMD_SHIFT)
82#define PMD_MASK (~(PMD_SIZE-1))
83#define PUD_SIZE (1UL << PUD_SHIFT)
84#define PUD_MASK (~(PUD_SIZE-1))
85#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
86#define PGDIR_MASK (~(PGDIR_SIZE-1))
87
88
89
90
91
92
93
94#define PTRS_PER_PTE 256
95#ifndef CONFIG_64BIT
96#define PTRS_PER_PMD 1
97#define PTRS_PER_PUD 1
98#else
99#define PTRS_PER_PMD 2048
100#define PTRS_PER_PUD 2048
101#endif
102#define PTRS_PER_PGD 2048
103
104#define FIRST_USER_ADDRESS 0
105
106#define pte_ERROR(e) \
107 printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
108#define pmd_ERROR(e) \
109 printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
110#define pud_ERROR(e) \
111 printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
112#define pgd_ERROR(e) \
113 printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))
114
115#ifndef __ASSEMBLY__
116
117
118
119
120
121
122
123
124extern unsigned long VMALLOC_START;
125extern unsigned long VMALLOC_END;
126extern struct page *vmemmap;
127
128#define VMEM_MAX_PHYS ((unsigned long) vmemmap)
129
130#ifdef CONFIG_64BIT
131extern unsigned long MODULES_VADDR;
132extern unsigned long MODULES_END;
133#define MODULES_VADDR MODULES_VADDR
134#define MODULES_END MODULES_END
135#define MODULES_LEN (1UL << 31)
136#endif
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222#define _PAGE_CO 0x100
223#define _PAGE_RO 0x200
224#define _PAGE_INVALID 0x400
225
226
227#define _PAGE_SWT 0x001
228#define _PAGE_SWX 0x002
229#define _PAGE_SWC 0x004
230#define _PAGE_SWR 0x008
231#define _PAGE_SWW 0x010
232#define _PAGE_SPECIAL 0x020
233#define __HAVE_ARCH_PTE_SPECIAL
234
235
236#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \
237 _PAGE_SWC | _PAGE_SWR)
238
239
240#define _PAGE_TYPE_EMPTY 0x400
241#define _PAGE_TYPE_NONE 0x401
242#define _PAGE_TYPE_SWAP 0x403
243#define _PAGE_TYPE_FILE 0x601
244#define _PAGE_TYPE_RO 0x200
245#define _PAGE_TYPE_RW 0x000
246
247
248
249
250
251#define _HPAGE_TYPE_EMPTY 0x020
252#define _HPAGE_TYPE_NONE 0x220
253#define _HPAGE_TYPE_RO 0x200
254#define _HPAGE_TYPE_RW 0x000
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282#ifndef CONFIG_64BIT
283
284
285#define _ASCE_SPACE_SWITCH 0x80000000UL
286#define _ASCE_ORIGIN_MASK 0x7ffff000UL
287#define _ASCE_PRIVATE_SPACE 0x100
288#define _ASCE_ALT_EVENT 0x80
289#define _ASCE_TABLE_LENGTH 0x7f
290
291
292#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL
293#define _SEGMENT_ENTRY_RO 0x200
294#define _SEGMENT_ENTRY_INV 0x20
295#define _SEGMENT_ENTRY_COMMON 0x10
296#define _SEGMENT_ENTRY_PTL 0x0f
297
298#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL)
299#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
300
301
302#define RCP_ACC_BITS 0xf0000000UL
303#define RCP_FP_BIT 0x08000000UL
304#define RCP_PCL_BIT 0x00800000UL
305#define RCP_HR_BIT 0x00400000UL
306#define RCP_HC_BIT 0x00200000UL
307#define RCP_GR_BIT 0x00040000UL
308#define RCP_GC_BIT 0x00020000UL
309#define RCP_IN_BIT 0x00002000UL
310
311
312#define KVM_UR_BIT 0x00008000UL
313#define KVM_UC_BIT 0x00004000UL
314
315#else
316
317
318#define _ASCE_ORIGIN ~0xfffUL
319#define _ASCE_PRIVATE_SPACE 0x100
320#define _ASCE_ALT_EVENT 0x80
321#define _ASCE_SPACE_SWITCH 0x40
322#define _ASCE_REAL_SPACE 0x20
323#define _ASCE_TYPE_MASK 0x0c
324#define _ASCE_TYPE_REGION1 0x0c
325#define _ASCE_TYPE_REGION2 0x08
326#define _ASCE_TYPE_REGION3 0x04
327#define _ASCE_TYPE_SEGMENT 0x00
328#define _ASCE_TABLE_LENGTH 0x03
329
330
331#define _REGION_ENTRY_ORIGIN ~0xfffUL
332#define _REGION_ENTRY_RO 0x200
333#define _REGION_ENTRY_INV 0x20
334#define _REGION_ENTRY_TYPE_MASK 0x0c
335#define _REGION_ENTRY_TYPE_R1 0x0c
336#define _REGION_ENTRY_TYPE_R2 0x08
337#define _REGION_ENTRY_TYPE_R3 0x04
338#define _REGION_ENTRY_LENGTH 0x03
339
340#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
341#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV)
342#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
343#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV)
344#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
345#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV)
346
347#define _REGION3_ENTRY_LARGE 0x400
348#define _REGION3_ENTRY_RO 0x200
349#define _REGION3_ENTRY_CO 0x100
350
351
352#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL
353#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL
354#define _SEGMENT_ENTRY_RO 0x200
355#define _SEGMENT_ENTRY_INV 0x20
356
357#define _SEGMENT_ENTRY (0)
358#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
359
360#define _SEGMENT_ENTRY_LARGE 0x400
361#define _SEGMENT_ENTRY_CO 0x100
362#define _SEGMENT_ENTRY_SPLIT_BIT 0
363#define _SEGMENT_ENTRY_SPLIT (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
364
365
366#define _SEGMENT_CHG_MASK (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
367 | _SEGMENT_ENTRY_SPLIT | _SEGMENT_ENTRY_CO)
368
369
370#define RCP_ACC_BITS 0xf000000000000000UL
371#define RCP_FP_BIT 0x0800000000000000UL
372#define RCP_PCL_BIT 0x0080000000000000UL
373#define RCP_HR_BIT 0x0040000000000000UL
374#define RCP_HC_BIT 0x0020000000000000UL
375#define RCP_GR_BIT 0x0004000000000000UL
376#define RCP_GC_BIT 0x0002000000000000UL
377#define RCP_IN_BIT 0x0000200000000000UL
378
379
380#define KVM_UR_BIT 0x0000800000000000UL
381#define KVM_UC_BIT 0x0000400000000000UL
382
383#endif
384
385
386
387
388
389
390#define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
391 _ASCE_ALT_EVENT)
392
393
394
395
396#define PAGE_NONE __pgprot(_PAGE_TYPE_NONE)
397#define PAGE_RO __pgprot(_PAGE_TYPE_RO)
398#define PAGE_RW __pgprot(_PAGE_TYPE_RO | _PAGE_SWW)
399#define PAGE_RWC __pgprot(_PAGE_TYPE_RW | _PAGE_SWW | _PAGE_SWC)
400
401#define PAGE_KERNEL PAGE_RWC
402#define PAGE_SHARED PAGE_KERNEL
403#define PAGE_COPY PAGE_RO
404
405
406
407
408
409
410
411#define __P000 PAGE_NONE
412#define __P001 PAGE_RO
413#define __P010 PAGE_RO
414#define __P011 PAGE_RO
415#define __P100 PAGE_RO
416#define __P101 PAGE_RO
417#define __P110 PAGE_RO
418#define __P111 PAGE_RO
419
420#define __S000 PAGE_NONE
421#define __S001 PAGE_RO
422#define __S010 PAGE_RW
423#define __S011 PAGE_RW
424#define __S100 PAGE_RO
425#define __S101 PAGE_RO
426#define __S110 PAGE_RW
427#define __S111 PAGE_RW
428
429
430
431
432#define SEGMENT_NONE __pgprot(_HPAGE_TYPE_NONE)
433#define SEGMENT_RO __pgprot(_HPAGE_TYPE_RO)
434#define SEGMENT_RW __pgprot(_HPAGE_TYPE_RW)
435
436static inline int mm_exclusive(struct mm_struct *mm)
437{
438 return likely(mm == current->active_mm &&
439 atomic_read(&mm->context.attach_count) <= 1);
440}
441
442static inline int mm_has_pgste(struct mm_struct *mm)
443{
444#ifdef CONFIG_PGSTE
445 if (unlikely(mm->context.has_pgste))
446 return 1;
447#endif
448 return 0;
449}
450
451
452
453#ifndef CONFIG_64BIT
454
455static inline int pgd_present(pgd_t pgd) { return 1; }
456static inline int pgd_none(pgd_t pgd) { return 0; }
457static inline int pgd_bad(pgd_t pgd) { return 0; }
458
459static inline int pud_present(pud_t pud) { return 1; }
460static inline int pud_none(pud_t pud) { return 0; }
461static inline int pud_large(pud_t pud) { return 0; }
462static inline int pud_bad(pud_t pud) { return 0; }
463
464#else
465
466static inline int pgd_present(pgd_t pgd)
467{
468 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
469 return 1;
470 return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
471}
472
473static inline int pgd_none(pgd_t pgd)
474{
475 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
476 return 0;
477 return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL;
478}
479
480static inline int pgd_bad(pgd_t pgd)
481{
482
483
484
485
486
487 unsigned long mask =
488 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
489 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
490 return (pgd_val(pgd) & mask) != 0;
491}
492
493static inline int pud_present(pud_t pud)
494{
495 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
496 return 1;
497 return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
498}
499
500static inline int pud_none(pud_t pud)
501{
502 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
503 return 0;
504 return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL;
505}
506
507static inline int pud_large(pud_t pud)
508{
509 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
510 return 0;
511 return !!(pud_val(pud) & _REGION3_ENTRY_LARGE);
512}
513
514static inline int pud_bad(pud_t pud)
515{
516
517
518
519
520
521 unsigned long mask =
522 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
523 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
524 return (pud_val(pud) & mask) != 0;
525}
526
527#endif
528
529static inline int pmd_present(pmd_t pmd)
530{
531 unsigned long mask = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO;
532 return (pmd_val(pmd) & mask) == _HPAGE_TYPE_NONE ||
533 !(pmd_val(pmd) & _SEGMENT_ENTRY_INV);
534}
535
536static inline int pmd_none(pmd_t pmd)
537{
538 return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) &&
539 !(pmd_val(pmd) & _SEGMENT_ENTRY_RO);
540}
541
542static inline int pmd_large(pmd_t pmd)
543{
544#ifdef CONFIG_64BIT
545 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
546#else
547 return 0;
548#endif
549}
550
551static inline int pmd_bad(pmd_t pmd)
552{
553 unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV;
554 return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
555}
556
557#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
558extern void pmdp_splitting_flush(struct vm_area_struct *vma,
559 unsigned long addr, pmd_t *pmdp);
560
561#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
562extern int pmdp_set_access_flags(struct vm_area_struct *vma,
563 unsigned long address, pmd_t *pmdp,
564 pmd_t entry, int dirty);
565
566#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
567extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
568 unsigned long address, pmd_t *pmdp);
569
570#define __HAVE_ARCH_PMD_WRITE
571static inline int pmd_write(pmd_t pmd)
572{
573 return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0;
574}
575
576static inline int pmd_young(pmd_t pmd)
577{
578 return 0;
579}
580
581static inline int pte_none(pte_t pte)
582{
583 return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
584}
585
586static inline int pte_present(pte_t pte)
587{
588 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT | _PAGE_SWX;
589 return (pte_val(pte) & mask) == _PAGE_TYPE_NONE ||
590 (!(pte_val(pte) & _PAGE_INVALID) &&
591 !(pte_val(pte) & _PAGE_SWT));
592}
593
594static inline int pte_file(pte_t pte)
595{
596 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT;
597 return (pte_val(pte) & mask) == _PAGE_TYPE_FILE;
598}
599
600static inline int pte_special(pte_t pte)
601{
602 return (pte_val(pte) & _PAGE_SPECIAL);
603}
604
605#define __HAVE_ARCH_PTE_SAME
606static inline int pte_same(pte_t a, pte_t b)
607{
608 return pte_val(a) == pte_val(b);
609}
610
611static inline pgste_t pgste_get_lock(pte_t *ptep)
612{
613 unsigned long new = 0;
614#ifdef CONFIG_PGSTE
615 unsigned long old;
616
617 preempt_disable();
618 asm(
619 " lg %0,%2\n"
620 "0: lgr %1,%0\n"
621 " nihh %0,0xff7f\n"
622 " oihh %1,0x0080\n"
623 " csg %0,%1,%2\n"
624 " jl 0b\n"
625 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
626 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
627#endif
628 return __pgste(new);
629}
630
631static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
632{
633#ifdef CONFIG_PGSTE
634 asm(
635 " nihh %1,0xff7f\n"
636 " stg %1,%0\n"
637 : "=Q" (ptep[PTRS_PER_PTE])
638 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
639 : "cc", "memory");
640 preempt_enable();
641#endif
642}
643
644static inline void pgste_set(pte_t *ptep, pgste_t pgste)
645{
646#ifdef CONFIG_PGSTE
647 *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
648#endif
649}
650
651static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
652{
653#ifdef CONFIG_PGSTE
654 unsigned long address, bits;
655 unsigned char skey;
656
657 if (pte_val(*ptep) & _PAGE_INVALID)
658 return pgste;
659 address = pte_val(*ptep) & PAGE_MASK;
660 skey = page_get_storage_key(address);
661 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
662
663 if (bits & _PAGE_CHANGED)
664 page_set_storage_key(address, skey ^ bits, 0);
665 else if (bits)
666 page_reset_referenced(address);
667
668 pgste_val(pgste) |= bits << 48;
669
670 bits |= (pgste_val(pgste) & (RCP_HR_BIT | RCP_HC_BIT)) >> 52;
671
672 pgste_val(pgste) |= bits << 45;
673
674 pgste_val(pgste) &= ~(RCP_HR_BIT | RCP_HC_BIT);
675 pgste_val(pgste) &= ~(RCP_ACC_BITS | RCP_FP_BIT);
676
677 pgste_val(pgste) |=
678 (unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
679
680 pte_val(*ptep) |= (bits & _PAGE_REFERENCED) << 1;
681#endif
682 return pgste;
683
684}
685
686static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste)
687{
688#ifdef CONFIG_PGSTE
689 int young;
690
691 if (pte_val(*ptep) & _PAGE_INVALID)
692 return pgste;
693
694 young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
695 if (young)
696 pgste_val(pgste) |= RCP_GR_BIT;
697
698 if (pgste_val(pgste) & RCP_HR_BIT) {
699 pgste_val(pgste) &= ~RCP_HR_BIT;
700 young = 1;
701 }
702
703 if (young) {
704 pgste_val(pgste) |= KVM_UR_BIT;
705 pte_val(*ptep) |= _PAGE_SWR;
706 }
707#endif
708 return pgste;
709}
710
711static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry)
712{
713#ifdef CONFIG_PGSTE
714 unsigned long address;
715 unsigned long nkey;
716
717 if (pte_val(entry) & _PAGE_INVALID)
718 return;
719 VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
720 address = pte_val(entry) & PAGE_MASK;
721
722
723
724
725
726 nkey = (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56;
727 page_set_storage_key(address, nkey, 0);
728#endif
729}
730
731static inline void pgste_set_pte(pte_t *ptep, pte_t entry)
732{
733 if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_SWW)) {
734
735
736
737
738 pte_val(entry) |= _PAGE_SWC;
739 pte_val(entry) &= ~_PAGE_RO;
740 }
741 *ptep = entry;
742}
743
744
745
746
747
748
749
750
751struct gmap {
752 struct list_head list;
753 struct mm_struct *mm;
754 unsigned long *table;
755 unsigned long asce;
756 struct list_head crst_list;
757};
758
759
760
761
762
763
764
765struct gmap_rmap {
766 struct list_head list;
767 struct gmap *gmap;
768 unsigned long *entry;
769 unsigned long vmaddr;
770};
771
772
773
774
775
776
777struct gmap_pgtable {
778 unsigned long vmaddr;
779 struct list_head mapper;
780};
781
782
783
784
785
786struct gmap_notifier {
787 struct list_head list;
788 void (*notifier_call)(struct gmap *gmap, unsigned long address);
789};
790
791struct gmap *gmap_alloc(struct mm_struct *mm);
792void gmap_free(struct gmap *gmap);
793void gmap_enable(struct gmap *gmap);
794void gmap_disable(struct gmap *gmap);
795int gmap_map_segment(struct gmap *gmap, unsigned long from,
796 unsigned long to, unsigned long len);
797int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len);
798unsigned long __gmap_translate(unsigned long address, struct gmap *);
799unsigned long gmap_translate(unsigned long address, struct gmap *);
800unsigned long __gmap_fault(unsigned long address, struct gmap *);
801unsigned long gmap_fault(unsigned long address, struct gmap *);
802void gmap_discard(unsigned long from, unsigned long to, struct gmap *);
803
804void gmap_register_ipte_notifier(struct gmap_notifier *);
805void gmap_unregister_ipte_notifier(struct gmap_notifier *);
806int gmap_ipte_notify(struct gmap *, unsigned long start, unsigned long len);
807void gmap_do_ipte_notify(struct mm_struct *, unsigned long addr, pte_t *);
808
809static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
810 unsigned long addr,
811 pte_t *ptep, pgste_t pgste)
812{
813#ifdef CONFIG_PGSTE
814 if (pgste_val(pgste) & RCP_IN_BIT) {
815 pgste_val(pgste) &= ~RCP_IN_BIT;
816 gmap_do_ipte_notify(mm, addr, ptep);
817 }
818#endif
819 return pgste;
820}
821
822
823
824
825
826
827static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
828 pte_t *ptep, pte_t entry)
829{
830 pgste_t pgste;
831
832 if (mm_has_pgste(mm)) {
833 pgste = pgste_get_lock(ptep);
834 pgste_set_key(ptep, pgste, entry);
835 pgste_set_pte(ptep, entry);
836 pgste_set_unlock(ptep, pgste);
837 } else {
838 if (!(pte_val(entry) & _PAGE_INVALID) && MACHINE_HAS_EDAT1)
839 pte_val(entry) |= _PAGE_CO;
840 *ptep = entry;
841 }
842}
843
844
845
846
847
848static inline int pte_write(pte_t pte)
849{
850 return (pte_val(pte) & _PAGE_SWW) != 0;
851}
852
853static inline int pte_dirty(pte_t pte)
854{
855 return (pte_val(pte) & _PAGE_SWC) != 0;
856}
857
858static inline int pte_young(pte_t pte)
859{
860#ifdef CONFIG_PGSTE
861 if (pte_val(pte) & _PAGE_SWR)
862 return 1;
863#endif
864 return 0;
865}
866
867
868
869
870
871static inline void pgd_clear(pgd_t *pgd)
872{
873#ifdef CONFIG_64BIT
874 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
875 pgd_val(*pgd) = _REGION2_ENTRY_EMPTY;
876#endif
877}
878
879static inline void pud_clear(pud_t *pud)
880{
881#ifdef CONFIG_64BIT
882 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
883 pud_val(*pud) = _REGION3_ENTRY_EMPTY;
884#endif
885}
886
887static inline void pmd_clear(pmd_t *pmdp)
888{
889 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
890}
891
892static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
893{
894 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
895}
896
897
898
899
900
901static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
902{
903 pte_val(pte) &= _PAGE_CHG_MASK;
904 pte_val(pte) |= pgprot_val(newprot);
905 if ((pte_val(pte) & _PAGE_SWC) && (pte_val(pte) & _PAGE_SWW))
906 pte_val(pte) &= ~_PAGE_RO;
907 return pte;
908}
909
910static inline pte_t pte_wrprotect(pte_t pte)
911{
912 pte_val(pte) &= ~_PAGE_SWW;
913
914 if (!(pte_val(pte) & _PAGE_INVALID))
915 pte_val(pte) |= _PAGE_RO;
916 return pte;
917}
918
919static inline pte_t pte_mkwrite(pte_t pte)
920{
921 pte_val(pte) |= _PAGE_SWW;
922 if (pte_val(pte) & _PAGE_SWC)
923 pte_val(pte) &= ~_PAGE_RO;
924 return pte;
925}
926
927static inline pte_t pte_mkclean(pte_t pte)
928{
929 pte_val(pte) &= ~_PAGE_SWC;
930
931 if (!(pte_val(pte) & _PAGE_INVALID))
932 pte_val(pte) |= _PAGE_RO;
933 return pte;
934}
935
936static inline pte_t pte_mkdirty(pte_t pte)
937{
938 pte_val(pte) |= _PAGE_SWC;
939 if (pte_val(pte) & _PAGE_SWW)
940 pte_val(pte) &= ~_PAGE_RO;
941 return pte;
942}
943
944static inline pte_t pte_mkold(pte_t pte)
945{
946#ifdef CONFIG_PGSTE
947 pte_val(pte) &= ~_PAGE_SWR;
948#endif
949 return pte;
950}
951
952static inline pte_t pte_mkyoung(pte_t pte)
953{
954 return pte;
955}
956
957static inline pte_t pte_mkspecial(pte_t pte)
958{
959 pte_val(pte) |= _PAGE_SPECIAL;
960 return pte;
961}
962
963#ifdef CONFIG_HUGETLB_PAGE
964static inline pte_t pte_mkhuge(pte_t pte)
965{
966 pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO);
967 return pte;
968}
969#endif
970
971
972
973
974static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,
975 pte_t *ptep)
976{
977 pgste_t pgste;
978 int dirty = 0;
979
980 if (mm_has_pgste(mm)) {
981 pgste = pgste_get_lock(ptep);
982 pgste = pgste_update_all(ptep, pgste);
983 dirty = !!(pgste_val(pgste) & KVM_UC_BIT);
984 pgste_val(pgste) &= ~KVM_UC_BIT;
985 pgste_set_unlock(ptep, pgste);
986 return dirty;
987 }
988 return dirty;
989}
990
991
992
993
994static inline int ptep_test_and_clear_user_young(struct mm_struct *mm,
995 pte_t *ptep)
996{
997 pgste_t pgste;
998 int young = 0;
999
1000 if (mm_has_pgste(mm)) {
1001 pgste = pgste_get_lock(ptep);
1002 pgste = pgste_update_young(ptep, pgste);
1003 young = !!(pgste_val(pgste) & KVM_UR_BIT);
1004 pgste_val(pgste) &= ~KVM_UR_BIT;
1005 pgste_set_unlock(ptep, pgste);
1006 }
1007 return young;
1008}
1009
1010#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1011static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1012 unsigned long addr, pte_t *ptep)
1013{
1014 pgste_t pgste;
1015 pte_t pte;
1016
1017 if (mm_has_pgste(vma->vm_mm)) {
1018 pgste = pgste_get_lock(ptep);
1019 pgste = pgste_update_young(ptep, pgste);
1020 pte = *ptep;
1021 *ptep = pte_mkold(pte);
1022 pgste_set_unlock(ptep, pgste);
1023 return pte_young(pte);
1024 }
1025 return 0;
1026}
1027
1028#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1029static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1030 unsigned long address, pte_t *ptep)
1031{
1032
1033
1034
1035
1036 return ptep_test_and_clear_young(vma, address, ptep);
1037}
1038
1039static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
1040{
1041 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1042#ifndef CONFIG_64BIT
1043
1044 pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
1045#else
1046
1047 pte_t *pto = ptep;
1048#endif
1049 asm volatile(
1050 " ipte %2,%3"
1051 : "=m" (*ptep) : "m" (*ptep),
1052 "a" (pto), "a" (address));
1053 }
1054}
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1070static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1071 unsigned long address, pte_t *ptep)
1072{
1073 pgste_t pgste;
1074 pte_t pte;
1075
1076 mm->context.flush_mm = 1;
1077 if (mm_has_pgste(mm)) {
1078 pgste = pgste_get_lock(ptep);
1079 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1080 }
1081
1082 pte = *ptep;
1083 if (!mm_exclusive(mm))
1084 __ptep_ipte(address, ptep);
1085 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1086
1087 if (mm_has_pgste(mm)) {
1088 pgste = pgste_update_all(&pte, pgste);
1089 pgste_set_unlock(ptep, pgste);
1090 }
1091 return pte;
1092}
1093
1094#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1095static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
1096 unsigned long address,
1097 pte_t *ptep)
1098{
1099 pgste_t pgste;
1100 pte_t pte;
1101
1102 mm->context.flush_mm = 1;
1103 if (mm_has_pgste(mm)) {
1104 pgste = pgste_get_lock(ptep);
1105 pgste_ipte_notify(mm, address, ptep, pgste);
1106 }
1107
1108 pte = *ptep;
1109 if (!mm_exclusive(mm))
1110 __ptep_ipte(address, ptep);
1111
1112 if (mm_has_pgste(mm)) {
1113 pgste = pgste_update_all(&pte, pgste);
1114 pgste_set(ptep, pgste);
1115 }
1116 return pte;
1117}
1118
1119static inline void ptep_modify_prot_commit(struct mm_struct *mm,
1120 unsigned long address,
1121 pte_t *ptep, pte_t pte)
1122{
1123 pgste_t pgste;
1124
1125 if (mm_has_pgste(mm)) {
1126 pgste = *(pgste_t *)(ptep + PTRS_PER_PTE);
1127 pgste_set_key(ptep, pgste, pte);
1128 pgste_set_pte(ptep, pte);
1129 pgste_set_unlock(ptep, pgste);
1130 } else
1131 *ptep = pte;
1132}
1133
1134#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
1135static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1136 unsigned long address, pte_t *ptep)
1137{
1138 pgste_t pgste;
1139 pte_t pte;
1140
1141 if (mm_has_pgste(vma->vm_mm)) {
1142 pgste = pgste_get_lock(ptep);
1143 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
1144 }
1145
1146 pte = *ptep;
1147 __ptep_ipte(address, ptep);
1148 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1149
1150 if (mm_has_pgste(vma->vm_mm)) {
1151 pgste = pgste_update_all(&pte, pgste);
1152 pgste_set_unlock(ptep, pgste);
1153 }
1154 return pte;
1155}
1156
1157
1158
1159
1160
1161
1162
1163
1164#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1165static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1166 unsigned long address,
1167 pte_t *ptep, int full)
1168{
1169 pgste_t pgste;
1170 pte_t pte;
1171
1172 if (mm_has_pgste(mm)) {
1173 pgste = pgste_get_lock(ptep);
1174 if (!full)
1175 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1176 }
1177
1178 pte = *ptep;
1179 if (!full)
1180 __ptep_ipte(address, ptep);
1181 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1182
1183 if (mm_has_pgste(mm)) {
1184 pgste = pgste_update_all(&pte, pgste);
1185 pgste_set_unlock(ptep, pgste);
1186 }
1187 return pte;
1188}
1189
1190#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1191static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,
1192 unsigned long address, pte_t *ptep)
1193{
1194 pgste_t pgste;
1195 pte_t pte = *ptep;
1196
1197 if (pte_write(pte)) {
1198 mm->context.flush_mm = 1;
1199 if (mm_has_pgste(mm)) {
1200 pgste = pgste_get_lock(ptep);
1201 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1202 }
1203
1204 if (!mm_exclusive(mm))
1205 __ptep_ipte(address, ptep);
1206 pte = pte_wrprotect(pte);
1207
1208 if (mm_has_pgste(mm)) {
1209 pgste_set_pte(ptep, pte);
1210 pgste_set_unlock(ptep, pgste);
1211 } else
1212 *ptep = pte;
1213 }
1214 return pte;
1215}
1216
1217#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1218static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1219 unsigned long address, pte_t *ptep,
1220 pte_t entry, int dirty)
1221{
1222 pgste_t pgste;
1223
1224 if (pte_same(*ptep, entry))
1225 return 0;
1226 if (mm_has_pgste(vma->vm_mm)) {
1227 pgste = pgste_get_lock(ptep);
1228 pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
1229 }
1230
1231 __ptep_ipte(address, ptep);
1232
1233 if (mm_has_pgste(vma->vm_mm)) {
1234 pgste_set_pte(ptep, entry);
1235 pgste_set_unlock(ptep, pgste);
1236 } else
1237 *ptep = entry;
1238 return 1;
1239}
1240
1241
1242
1243
1244
1245static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1246{
1247 pte_t __pte;
1248 pte_val(__pte) = physpage + pgprot_val(pgprot);
1249 return __pte;
1250}
1251
1252static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1253{
1254 unsigned long physpage = page_to_phys(page);
1255 pte_t __pte = mk_pte_phys(physpage, pgprot);
1256
1257 if ((pte_val(__pte) & _PAGE_SWW) && PageDirty(page)) {
1258 pte_val(__pte) |= _PAGE_SWC;
1259 pte_val(__pte) &= ~_PAGE_RO;
1260 }
1261 return __pte;
1262}
1263
1264#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1265#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1266#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1267#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
1268
1269#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
1270#define pgd_offset_k(address) pgd_offset(&init_mm, address)
1271
1272#ifndef CONFIG_64BIT
1273
1274#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1275#define pud_deref(pmd) ({ BUG(); 0UL; })
1276#define pgd_deref(pmd) ({ BUG(); 0UL; })
1277
1278#define pud_offset(pgd, address) ((pud_t *) pgd)
1279#define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address))
1280
1281#else
1282
1283#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1284#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
1285#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
1286
1287static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
1288{
1289 pud_t *pud = (pud_t *) pgd;
1290 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
1291 pud = (pud_t *) pgd_deref(*pgd);
1292 return pud + pud_index(address);
1293}
1294
1295static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
1296{
1297 pmd_t *pmd = (pmd_t *) pud;
1298 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
1299 pmd = (pmd_t *) pud_deref(*pud);
1300 return pmd + pmd_index(address);
1301}
1302
1303#endif
1304
1305#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
1306#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1307#define pte_page(x) pfn_to_page(pte_pfn(x))
1308
1309#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
1310
1311
1312#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
1313#define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
1314#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
1315#define pte_unmap(pte) do { } while (0)
1316
1317static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
1318{
1319 unsigned long sto = (unsigned long) pmdp -
1320 pmd_index(address) * sizeof(pmd_t);
1321
1322 if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
1323 asm volatile(
1324 " .insn rrf,0xb98e0000,%2,%3,0,0"
1325 : "=m" (*pmdp)
1326 : "m" (*pmdp), "a" (sto),
1327 "a" ((address & HPAGE_MASK))
1328 : "cc"
1329 );
1330 }
1331}
1332
1333#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
1334static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1335{
1336
1337
1338
1339
1340 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1341 return pgprot_val(SEGMENT_NONE);
1342 if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1343 return pgprot_val(SEGMENT_RO);
1344 return pgprot_val(SEGMENT_RW);
1345}
1346
1347static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1348{
1349 pmd_val(pmd) &= _SEGMENT_CHG_MASK;
1350 pmd_val(pmd) |= massage_pgprot_pmd(newprot);
1351 return pmd;
1352}
1353
1354static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1355{
1356 pmd_t __pmd;
1357 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
1358 return __pmd;
1359}
1360
1361static inline pmd_t pmd_mkwrite(pmd_t pmd)
1362{
1363
1364 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_INV))
1365 pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO;
1366 return pmd;
1367}
1368#endif
1369
1370#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1371
1372#define __HAVE_ARCH_PGTABLE_DEPOSIT
1373extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable);
1374
1375#define __HAVE_ARCH_PGTABLE_WITHDRAW
1376extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm);
1377
1378static inline int pmd_trans_splitting(pmd_t pmd)
1379{
1380 return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
1381}
1382
1383static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1384 pmd_t *pmdp, pmd_t entry)
1385{
1386 if (!(pmd_val(entry) & _SEGMENT_ENTRY_INV) && MACHINE_HAS_EDAT1)
1387 pmd_val(entry) |= _SEGMENT_ENTRY_CO;
1388 *pmdp = entry;
1389}
1390
1391static inline pmd_t pmd_mkhuge(pmd_t pmd)
1392{
1393 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
1394 return pmd;
1395}
1396
1397static inline pmd_t pmd_wrprotect(pmd_t pmd)
1398{
1399 pmd_val(pmd) |= _SEGMENT_ENTRY_RO;
1400 return pmd;
1401}
1402
1403static inline pmd_t pmd_mkdirty(pmd_t pmd)
1404{
1405
1406 return pmd;
1407}
1408
1409static inline pmd_t pmd_mkold(pmd_t pmd)
1410{
1411
1412 return pmd;
1413}
1414
1415static inline pmd_t pmd_mkyoung(pmd_t pmd)
1416{
1417
1418 return pmd;
1419}
1420
1421#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1422static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1423 unsigned long address, pmd_t *pmdp)
1424{
1425 unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK;
1426 long tmp, rc;
1427 int counter;
1428
1429 rc = 0;
1430 if (MACHINE_HAS_RRBM) {
1431 counter = PTRS_PER_PTE >> 6;
1432 asm volatile(
1433 "0: .insn rre,0xb9ae0000,%0,%3\n"
1434 " ogr %1,%0\n"
1435 " la %3,0(%4,%3)\n"
1436 " brct %2,0b\n"
1437 : "=&d" (tmp), "+&d" (rc), "+d" (counter),
1438 "+a" (pmd_addr)
1439 : "a" (64 * 4096UL) : "cc");
1440 rc = !!rc;
1441 } else {
1442 counter = PTRS_PER_PTE;
1443 asm volatile(
1444 "0: rrbe 0,%2\n"
1445 " la %2,0(%3,%2)\n"
1446 " brc 12,1f\n"
1447 " lhi %0,1\n"
1448 "1: brct %1,0b\n"
1449 : "+d" (rc), "+d" (counter), "+a" (pmd_addr)
1450 : "a" (4096UL) : "cc");
1451 }
1452 return rc;
1453}
1454
1455#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
1456static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
1457 unsigned long address, pmd_t *pmdp)
1458{
1459 pmd_t pmd = *pmdp;
1460
1461 __pmd_idte(address, pmdp);
1462 pmd_clear(pmdp);
1463 return pmd;
1464}
1465
1466#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
1467static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
1468 unsigned long address, pmd_t *pmdp)
1469{
1470 return pmdp_get_and_clear(vma->vm_mm, address, pmdp);
1471}
1472
1473#define __HAVE_ARCH_PMDP_INVALIDATE
1474static inline void pmdp_invalidate(struct vm_area_struct *vma,
1475 unsigned long address, pmd_t *pmdp)
1476{
1477 __pmd_idte(address, pmdp);
1478}
1479
1480#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1481static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1482 unsigned long address, pmd_t *pmdp)
1483{
1484 pmd_t pmd = *pmdp;
1485
1486 if (pmd_write(pmd)) {
1487 __pmd_idte(address, pmdp);
1488 set_pmd_at(mm, address, pmdp, pmd_wrprotect(pmd));
1489 }
1490}
1491
1492#define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
1493#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
1494
1495static inline int pmd_trans_huge(pmd_t pmd)
1496{
1497 return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
1498}
1499
1500static inline int has_transparent_hugepage(void)
1501{
1502 return MACHINE_HAS_HPAGE ? 1 : 0;
1503}
1504
1505static inline unsigned long pmd_pfn(pmd_t pmd)
1506{
1507 return pmd_val(pmd) >> PAGE_SHIFT;
1508}
1509#endif
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546#ifndef CONFIG_64BIT
1547#define __SWP_OFFSET_MASK (~0UL >> 12)
1548#else
1549#define __SWP_OFFSET_MASK (~0UL >> 11)
1550#endif
1551static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1552{
1553 pte_t pte;
1554 offset &= __SWP_OFFSET_MASK;
1555 pte_val(pte) = _PAGE_TYPE_SWAP | ((type & 0x1f) << 2) |
1556 ((offset & 1UL) << 7) | ((offset & ~1UL) << 11);
1557 return pte;
1558}
1559
1560#define __swp_type(entry) (((entry).val >> 2) & 0x1f)
1561#define __swp_offset(entry) (((entry).val >> 11) | (((entry).val >> 7) & 1))
1562#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
1563
1564#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1565#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
1566
1567#ifndef CONFIG_64BIT
1568# define PTE_FILE_MAX_BITS 26
1569#else
1570# define PTE_FILE_MAX_BITS 59
1571#endif
1572
1573#define pte_to_pgoff(__pte) \
1574 ((((__pte).pte >> 12) << 7) + (((__pte).pte >> 1) & 0x7f))
1575
1576#define pgoff_to_pte(__off) \
1577 ((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \
1578 | _PAGE_TYPE_FILE })
1579
1580#endif
1581
1582#define kern_addr_valid(addr) (1)
1583
1584extern int vmem_add_mapping(unsigned long start, unsigned long size);
1585extern int vmem_remove_mapping(unsigned long start, unsigned long size);
1586extern int s390_enable_sie(void);
1587
1588
1589
1590
1591static inline void pgtable_cache_init(void) { }
1592static inline void check_pgt_cache(void) { }
1593
1594#include <asm-generic/pgtable.h>
1595
1596#endif
1597