LXR linux/arch/s390/mm/hugetlbpage.c

   1/*
   2 *  IBM System z Huge TLB Page Support for Kernel.
   3 *
   4 *    Copyright IBM Corp. 2007,2016
   5 *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
   6 */
   7
   8#include <linux/mm.h>
   9#include <linux/hugetlb.h>
  10
  11static inline unsigned long __pte_to_rste(pte_t pte)
  12{
  13        int none, young, prot;
  14        unsigned long rste;
  15
  16        /*
  17         * Convert encoding               pte bits      pmd/pud bits
  18         *                              .IR...wrdytp    ..R...I...y.
  19         * empty                        .10...000000 -> ..0...1...0.
  20         * prot-none, clean, old        .11...000001 -> ..0...1...1.
  21         * prot-none, clean, young      .11...000101 -> ..1...1...1.
  22         * prot-none, dirty, old        .10...001001 -> ..0...1...1.
  23         * prot-none, dirty, young      .10...001101 -> ..1...1...1.
  24         * read-only, clean, old        .11...010001 -> ..1...1...0.
  25         * read-only, clean, young      .01...010101 -> ..1...0...1.
  26         * read-only, dirty, old        .11...011001 -> ..1...1...0.
  27         * read-only, dirty, young      .01...011101 -> ..1...0...1.
  28         * read-write, clean, old       .11...110001 -> ..0...1...0.
  29         * read-write, clean, young     .01...110101 -> ..0...0...1.
  30         * read-write, dirty, old       .10...111001 -> ..0...1...0.
  31         * read-write, dirty, young     .00...111101 -> ..0...0...1.
  32         * Huge ptes are dirty by definition, a clean pte is made dirty
  33         * by the conversion.
  34         */
  35        if (pte_present(pte)) {
  36                rste = pte_val(pte) & PAGE_MASK;
  37                if (pte_val(pte) & _PAGE_INVALID)
  38                        rste |= _SEGMENT_ENTRY_INVALID;
  39                none = (pte_val(pte) & _PAGE_PRESENT) &&
  40                        !(pte_val(pte) & _PAGE_READ) &&
  41                        !(pte_val(pte) & _PAGE_WRITE);
  42                prot = (pte_val(pte) & _PAGE_PROTECT) &&
  43                        !(pte_val(pte) & _PAGE_WRITE);
  44                young = pte_val(pte) & _PAGE_YOUNG;
  45                if (none || young)
  46                        rste |= _SEGMENT_ENTRY_YOUNG;
  47                if (prot || (none && young))
  48                        rste |= _SEGMENT_ENTRY_PROTECT;
  49        } else
  50                rste = _SEGMENT_ENTRY_INVALID;
  51        return rste;
  52}
  53
  54static inline pte_t __rste_to_pte(unsigned long rste)
  55{
  56        int present;
  57        pte_t pte;
  58
  59        if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
  60                present = pud_present(__pud(rste));
  61        else
  62                present = pmd_present(__pmd(rste));
  63
  64        /*
  65         * Convert encoding     pmd/pud bits      pte bits
  66         *                      ..R...I...y.    .IR...wrdytp
  67         * empty                ..0...1...0. -> .10...000000
  68         * prot-none, old       ..0...1...1. -> .10...001001
  69         * prot-none, young     ..1...1...1. -> .10...001101
  70         * read-only, old       ..1...1...0. -> .11...011001
  71         * read-only, young     ..1...0...1. -> .01...011101
  72         * read-write, old      ..0...1...0. -> .10...111001
  73         * read-write, young    ..0...0...1. -> .00...111101
  74         * Huge ptes are dirty by definition
  75         */
  76        if (present) {
  77                pte_val(pte) = _PAGE_PRESENT | _PAGE_LARGE | _PAGE_DIRTY |
  78                               (rste & PAGE_MASK);
  79                if (rste & _SEGMENT_ENTRY_INVALID)
  80                        pte_val(pte) |= _PAGE_INVALID;
  81                if (pmd_prot_none(__pmd(rste))) {
  82                        if (rste & _SEGMENT_ENTRY_PROTECT)
  83                                pte_val(pte) |= _PAGE_YOUNG;
  84                } else {
  85                        pte_val(pte) |= _PAGE_READ;
  86                        if (rste & _SEGMENT_ENTRY_PROTECT)
  87                                pte_val(pte) |= _PAGE_PROTECT;
  88                        else
  89                                pte_val(pte) |= _PAGE_WRITE;
  90                        if (rste & _SEGMENT_ENTRY_YOUNG)
  91                                pte_val(pte) |= _PAGE_YOUNG;
  92                }
  93        } else
  94                pte_val(pte) = _PAGE_INVALID;
  95        return pte;
  96}
  97
  98void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  99                     pte_t *ptep, pte_t pte)
 100{
 101        unsigned long rste = __pte_to_rste(pte);
 102
 103        /* Set correct table type for 2G hugepages */
 104        if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 105                rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE;
 106        else {
 107                if (!MACHINE_HAS_HPAGE) {
 108                        rste &= ~_SEGMENT_ENTRY_ORIGIN;
 109                        rste |= pte_page(pte)[1].index;
 110                } else
 111                        rste |= _SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO;
 112        }
 113        pte_val(*ptep) = rste;
 114}
 115
 116pte_t huge_ptep_get(pte_t *ptep)
 117{
 118        unsigned long origin, rste;
 119
 120        rste = pte_val(*ptep);
 121        if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
 122                if (!MACHINE_HAS_HPAGE && pmd_present(__pmd(rste))) {
 123                        origin = rste & _SEGMENT_ENTRY_ORIGIN;
 124                        rste &= ~_SEGMENT_ENTRY_ORIGIN;
 125                        rste |= *(unsigned long *) origin;
 126                }
 127        return __rste_to_pte(rste);
 128}
 129
 130pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 131                              unsigned long addr, pte_t *ptep)
 132{
 133        pte_t pte = huge_ptep_get(ptep);
 134        pmd_t *pmdp = (pmd_t *) ptep;
 135        pud_t *pudp = (pud_t *) ptep;
 136
 137        if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
 138                __pudp_idte(addr, pudp);
 139                pud_val(*pudp) = _REGION3_ENTRY_EMPTY;
 140        } else {
 141                if (MACHINE_HAS_IDTE)
 142                        __pmd_idte(addr, pmdp);
 143                else
 144                        __pmd_csp(pmdp);
 145                pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
 146        }
 147        return pte;
 148}
 149
 150int arch_prepare_hugepage(struct page *page)
 151{
 152        unsigned long addr = page_to_phys(page);
 153        pte_t pte;
 154        pte_t *ptep;
 155        int i;
 156
 157        if (MACHINE_HAS_HPAGE)
 158                return 0;
 159
 160        ptep = (pte_t *) pte_alloc_one(&init_mm, addr);
 161        if (!ptep)
 162                return -ENOMEM;
 163
 164        pte_val(pte) = addr;
 165        for (i = 0; i < PTRS_PER_PTE; i++) {
 166                set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte);
 167                pte_val(pte) += PAGE_SIZE;
 168        }
 169        page[1].index = (unsigned long) ptep;
 170        return 0;
 171}
 172
 173void arch_release_hugepage(struct page *page)
 174{
 175        pte_t *ptep;
 176
 177        if (MACHINE_HAS_HPAGE)
 178                return;
 179
 180        ptep = (pte_t *) page[1].index;
 181        if (!ptep)
 182                return;
 183        clear_table((unsigned long *) ptep, _PAGE_INVALID,
 184                    PTRS_PER_PTE * sizeof(pte_t));
 185        page_table_free(&init_mm, (unsigned long *) ptep);
 186        page[1].index = 0;
 187}
 188
 189pte_t *huge_pte_alloc(struct mm_struct *mm,
 190                        unsigned long addr, unsigned long sz)
 191{
 192        pgd_t *pgdp;
 193        pud_t *pudp;
 194        pmd_t *pmdp = NULL;
 195
 196        pgdp = pgd_offset(mm, addr);
 197        pudp = pud_alloc(mm, pgdp, addr);
 198        if (pudp) {
 199                if (sz == PUD_SIZE)
 200                        return (pte_t *) pudp;
 201                else if (sz == PMD_SIZE)
 202                        pmdp = pmd_alloc(mm, pudp, addr);
 203        }
 204        return (pte_t *) pmdp;
 205}
 206
 207pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 208{
 209        pgd_t *pgdp;
 210        pud_t *pudp;
 211        pmd_t *pmdp = NULL;
 212
 213        pgdp = pgd_offset(mm, addr);
 214        if (pgd_present(*pgdp)) {
 215                pudp = pud_offset(pgdp, addr);
 216                if (pud_present(*pudp)) {
 217                        if (pud_large(*pudp))
 218                                return (pte_t *) pudp;
 219                        pmdp = pmd_offset(pudp, addr);
 220                }
 221        }
 222        return (pte_t *) pmdp;
 223}
 224
 225int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 226{
 227        return 0;
 228}
 229
 230int pmd_huge(pmd_t pmd)
 231{
 232        if (!MACHINE_HAS_HPAGE)
 233                return 0;
 234
 235        return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
 236}
 237
 238int pud_huge(pud_t pud)
 239{
 240        return pud_large(pud);
 241}
 242
 243struct page *
 244follow_huge_pud(struct mm_struct *mm, unsigned long address,
 245                pud_t *pud, int flags)
 246{
 247        if (flags & FOLL_GET)
 248                return NULL;
 249
 250        return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
 251}
 252
 253static __init int setup_hugepagesz(char *opt)
 254{
 255        unsigned long size;
 256        char *string = opt;
 257
 258        size = memparse(opt, &opt);
 259        if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
 260                hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
 261        } else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
 262                hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
 263        } else {
 264                pr_err("hugepagesz= specifies an unsupported page size %s\n",
 265                        string);
 266                return 0;
 267        }
 268        return 1;
 269}
 270__setup("hugepagesz=", setup_hugepagesz);
 271