LXR linux/arch/s390/mm/hugetlbpage.c

   1/*
   2 *  IBM System z Huge TLB Page Support for Kernel.
   3 *
   4 *    Copyright IBM Corp. 2007
   5 *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
   6 */
   7
   8#include <linux/mm.h>
   9#include <linux/hugetlb.h>
  10
  11static inline pmd_t __pte_to_pmd(pte_t pte)
  12{
  13        pmd_t pmd;
  14
  15        /*
  16         * Convert encoding               pte bits         pmd bits
  17         *                              lIR.uswrdy.p    dy..R...I...wr
  18         * empty                        010.000000.0 -> 00..0...1...00
  19         * prot-none, clean, old        111.000000.1 -> 00..1...1...00
  20         * prot-none, clean, young      111.000001.1 -> 01..1...1...00
  21         * prot-none, dirty, old        111.000010.1 -> 10..1...1...00
  22         * prot-none, dirty, young      111.000011.1 -> 11..1...1...00
  23         * read-only, clean, old        111.000100.1 -> 00..1...1...01
  24         * read-only, clean, young      101.000101.1 -> 01..1...0...01
  25         * read-only, dirty, old        111.000110.1 -> 10..1...1...01
  26         * read-only, dirty, young      101.000111.1 -> 11..1...0...01
  27         * read-write, clean, old       111.001100.1 -> 00..1...1...11
  28         * read-write, clean, young     101.001101.1 -> 01..1...0...11
  29         * read-write, dirty, old       110.001110.1 -> 10..0...1...11
  30         * read-write, dirty, young     100.001111.1 -> 11..0...0...11
  31         * HW-bits: R read-only, I invalid
  32         * SW-bits: p present, y young, d dirty, r read, w write, s special,
  33         *          u unused, l large
  34         */
  35        if (pte_present(pte)) {
  36                pmd_val(pmd) = pte_val(pte) & PAGE_MASK;
  37                pmd_val(pmd) |= (pte_val(pte) & _PAGE_READ) >> 4;
  38                pmd_val(pmd) |= (pte_val(pte) & _PAGE_WRITE) >> 4;
  39                pmd_val(pmd) |= (pte_val(pte) & _PAGE_INVALID) >> 5;
  40                pmd_val(pmd) |= (pte_val(pte) & _PAGE_PROTECT);
  41                pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10;
  42                pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10;
  43        } else
  44                pmd_val(pmd) = _SEGMENT_ENTRY_INVALID;
  45        return pmd;
  46}
  47
  48static inline pte_t __pmd_to_pte(pmd_t pmd)
  49{
  50        pte_t pte;
  51
  52        /*
  53         * Convert encoding                pmd bits         pte bits
  54         *                              dy..R...I...wr    lIR.uswrdy.p
  55         * empty                        00..0...1...00 -> 010.000000.0
  56         * prot-none, clean, old        00..1...1...00 -> 111.000000.1
  57         * prot-none, clean, young      01..1...1...00 -> 111.000001.1
  58         * prot-none, dirty, old        10..1...1...00 -> 111.000010.1
  59         * prot-none, dirty, young      11..1...1...00 -> 111.000011.1
  60         * read-only, clean, old        00..1...1...01 -> 111.000100.1
  61         * read-only, clean, young      01..1...0...01 -> 101.000101.1
  62         * read-only, dirty, old        10..1...1...01 -> 111.000110.1
  63         * read-only, dirty, young      11..1...0...01 -> 101.000111.1
  64         * read-write, clean, old       00..1...1...11 -> 111.001100.1
  65         * read-write, clean, young     01..1...0...11 -> 101.001101.1
  66         * read-write, dirty, old       10..0...1...11 -> 110.001110.1
  67         * read-write, dirty, young     11..0...0...11 -> 100.001111.1
  68         * HW-bits: R read-only, I invalid
  69         * SW-bits: p present, y young, d dirty, r read, w write, s special,
  70         *          u unused, l large
  71         */
  72        if (pmd_present(pmd)) {
  73                pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE;
  74                pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
  75                pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_READ) << 4;
  76                pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4;
  77                pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5;
  78                pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT);
  79                pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) >> 10;
  80                pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) >> 10;
  81        } else
  82                pte_val(pte) = _PAGE_INVALID;
  83        return pte;
  84}
  85
  86void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  87                     pte_t *ptep, pte_t pte)
  88{
  89        pmd_t pmd;
  90
  91        pmd = __pte_to_pmd(pte);
  92        if (!MACHINE_HAS_HPAGE) {
  93                /* Emulated huge ptes loose the dirty and young bit */
  94                pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
  95                pmd_val(pmd) |= pte_page(pte)[1].index;
  96        } else
  97                pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
  98        *(pmd_t *) ptep = pmd;
  99}
 100
 101pte_t huge_ptep_get(pte_t *ptep)
 102{
 103        unsigned long origin;
 104        pmd_t pmd;
 105
 106        pmd = *(pmd_t *) ptep;
 107        if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) {
 108                origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN;
 109                pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
 110                pmd_val(pmd) |= *(unsigned long *) origin;
 111                /* Emulated huge ptes are young and dirty by definition */
 112                pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG | _SEGMENT_ENTRY_DIRTY;
 113        }
 114        return __pmd_to_pte(pmd);
 115}
 116
 117pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 118                              unsigned long addr, pte_t *ptep)
 119{
 120        pmd_t *pmdp = (pmd_t *) ptep;
 121        pte_t pte = huge_ptep_get(ptep);
 122
 123        pmdp_flush_direct(mm, addr, pmdp);
 124        pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
 125        return pte;
 126}
 127
 128int arch_prepare_hugepage(struct page *page)
 129{
 130        unsigned long addr = page_to_phys(page);
 131        pte_t pte;
 132        pte_t *ptep;
 133        int i;
 134
 135        if (MACHINE_HAS_HPAGE)
 136                return 0;
 137
 138        ptep = (pte_t *) pte_alloc_one(&init_mm, addr);
 139        if (!ptep)
 140                return -ENOMEM;
 141
 142        pte_val(pte) = addr;
 143        for (i = 0; i < PTRS_PER_PTE; i++) {
 144                set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte);
 145                pte_val(pte) += PAGE_SIZE;
 146        }
 147        page[1].index = (unsigned long) ptep;
 148        return 0;
 149}
 150
 151void arch_release_hugepage(struct page *page)
 152{
 153        pte_t *ptep;
 154
 155        if (MACHINE_HAS_HPAGE)
 156                return;
 157
 158        ptep = (pte_t *) page[1].index;
 159        if (!ptep)
 160                return;
 161        clear_table((unsigned long *) ptep, _PAGE_INVALID,
 162                    PTRS_PER_PTE * sizeof(pte_t));
 163        page_table_free(&init_mm, (unsigned long *) ptep);
 164        page[1].index = 0;
 165}
 166
 167pte_t *huge_pte_alloc(struct mm_struct *mm,
 168                        unsigned long addr, unsigned long sz)
 169{
 170        pgd_t *pgdp;
 171        pud_t *pudp;
 172        pmd_t *pmdp = NULL;
 173
 174        pgdp = pgd_offset(mm, addr);
 175        pudp = pud_alloc(mm, pgdp, addr);
 176        if (pudp)
 177                pmdp = pmd_alloc(mm, pudp, addr);
 178        return (pte_t *) pmdp;
 179}
 180
 181pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 182{
 183        pgd_t *pgdp;
 184        pud_t *pudp;
 185        pmd_t *pmdp = NULL;
 186
 187        pgdp = pgd_offset(mm, addr);
 188        if (pgd_present(*pgdp)) {
 189                pudp = pud_offset(pgdp, addr);
 190                if (pud_present(*pudp))
 191                        pmdp = pmd_offset(pudp, addr);
 192        }
 193        return (pte_t *) pmdp;
 194}
 195
 196int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 197{
 198        return 0;
 199}
 200
 201int pmd_huge(pmd_t pmd)
 202{
 203        if (!MACHINE_HAS_HPAGE)
 204                return 0;
 205
 206        return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
 207}
 208
 209int pud_huge(pud_t pud)
 210{
 211        return 0;
 212}
 213