LXR linux/arch/x86/mm/ident

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Helper routines for building identity mapping page tables. This is
   4 * included by both the compressed kernel and the regular kernel.
   5 */
   6
   7static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
   8                           unsigned long addr, unsigned long end)
   9{
  10        addr &= PMD_MASK;
  11        for (; addr < end; addr += PMD_SIZE) {
  12                pmd_t *pmd = pmd_page + pmd_index(addr);
  13
  14                if (pmd_present(*pmd))
  15                        continue;
  16
  17                set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
  18        }
  19}
  20
  21static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
  22                          unsigned long addr, unsigned long end)
  23{
  24        unsigned long next;
  25
  26        for (; addr < end; addr = next) {
  27                pud_t *pud = pud_page + pud_index(addr);
  28                pmd_t *pmd;
  29
  30                next = (addr & PUD_MASK) + PUD_SIZE;
  31                if (next > end)
  32                        next = end;
  33
  34                if (info->direct_gbpages) {
  35                        pud_t pudval;
  36
  37                        if (pud_present(*pud))
  38                                continue;
  39
  40                        addr &= PUD_MASK;
  41                        pudval = __pud((addr - info->offset) | info->page_flag);
  42                        set_pud(pud, pudval);
  43                        continue;
  44                }
  45
  46                if (pud_present(*pud)) {
  47                        pmd = pmd_offset(pud, 0);
  48                        ident_pmd_init(info, pmd, addr, next);
  49                        continue;
  50                }
  51                pmd = (pmd_t *)info->alloc_pgt_page(info->context);
  52                if (!pmd)
  53                        return -ENOMEM;
  54                ident_pmd_init(info, pmd, addr, next);
  55                set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag));
  56        }
  57
  58        return 0;
  59}
  60
  61static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
  62                          unsigned long addr, unsigned long end)
  63{
  64        unsigned long next;
  65
  66        for (; addr < end; addr = next) {
  67                p4d_t *p4d = p4d_page + p4d_index(addr);
  68                pud_t *pud;
  69
  70                next = (addr & P4D_MASK) + P4D_SIZE;
  71                if (next > end)
  72                        next = end;
  73
  74                if (p4d_present(*p4d)) {
  75                        pud = pud_offset(p4d, 0);
  76                        ident_pud_init(info, pud, addr, next);
  77                        continue;
  78                }
  79                pud = (pud_t *)info->alloc_pgt_page(info->context);
  80                if (!pud)
  81                        return -ENOMEM;
  82                ident_pud_init(info, pud, addr, next);
  83                set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag));
  84        }
  85
  86        return 0;
  87}
  88
  89int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
  90                              unsigned long pstart, unsigned long pend)
  91{
  92        unsigned long addr = pstart + info->offset;
  93        unsigned long end = pend + info->offset;
  94        unsigned long next;
  95        int result;
  96
  97        /* Set the default pagetable flags if not supplied */
  98        if (!info->kernpg_flag)
  99                info->kernpg_flag = _KERNPG_TABLE;
 100
 101        /* Filter out unsupported __PAGE_KERNEL_* bits: */
 102        info->kernpg_flag &= __default_kernel_pte_mask;
 103
 104        for (; addr < end; addr = next) {
 105                pgd_t *pgd = pgd_page + pgd_index(addr);
 106                p4d_t *p4d;
 107
 108                next = (addr & PGDIR_MASK) + PGDIR_SIZE;
 109                if (next > end)
 110                        next = end;
 111
 112                if (pgd_present(*pgd)) {
 113                        p4d = p4d_offset(pgd, 0);
 114                        result = ident_p4d_init(info, p4d, addr, next);
 115                        if (result)
 116                                return result;
 117                        continue;
 118                }
 119
 120                p4d = (p4d_t *)info->alloc_pgt_page(info->context);
 121                if (!p4d)
 122                        return -ENOMEM;
 123                result = ident_p4d_init(info, p4d, addr, next);
 124                if (result)
 125                        return result;
 126                if (pgtable_l5_enabled()) {
 127                        set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
 128                } else {
 129                        /*
 130                         * With p4d folded, pgd is equal to p4d.
 131                         * The pgd entry has to point to the pud page table in this case.
 132                         */
 133                        pud_t *pud = pud_offset(p4d, 0);
 134                        set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag));
 135                }
 136        }
 137
 138        return 0;
 139}
 140