LXR linux/arch/x86/boot/compressed/pgtable

   1#include <asm/processor.h>
   2#include "pgtable.h"
   3#include "../string.h"
   4
   5/*
   6 * __force_order is used by special_insns.h asm code to force instruction
   7 * serialization.
   8 *
   9 * It is not referenced from the code, but GCC < 5 with -fPIE would fail
  10 * due to an undefined symbol. Define it to make these ancient GCCs work.
  11 */
  12unsigned long __force_order;
  13
  14#define BIOS_START_MIN          0x20000U        /* 128K, less than this is insane */
  15#define BIOS_START_MAX          0x9f000U        /* 640K, absolute maximum */
  16
  17struct paging_config {
  18        unsigned long trampoline_start;
  19        unsigned long l5_required;
  20};
  21
  22/* Buffer to preserve trampoline memory */
  23static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
  24
  25/*
  26 * Trampoline address will be printed by extract_kernel() for debugging
  27 * purposes.
  28 *
  29 * Avoid putting the pointer into .bss as it will be cleared between
  30 * paging_prepare() and extract_kernel().
  31 */
  32unsigned long *trampoline_32bit __section(.data);
  33
  34struct paging_config paging_prepare(void)
  35{
  36        struct paging_config paging_config = {};
  37        unsigned long bios_start, ebda_start;
  38
  39        /*
  40         * Check if LA57 is desired and supported.
  41         *
  42         * There are two parts to the check:
  43         *   - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y
  44         *   - if the machine supports 5-level paging:
  45         *     + CPUID leaf 7 is supported
  46         *     + the leaf has the feature bit set
  47         *
  48         * That's substitute for boot_cpu_has() in early boot code.
  49         */
  50        if (IS_ENABLED(CONFIG_X86_5LEVEL) &&
  51                        native_cpuid_eax(0) >= 7 &&
  52                        (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
  53                paging_config.l5_required = 1;
  54        }
  55
  56        /*
  57         * Find a suitable spot for the trampoline.
  58         * This code is based on reserve_bios_regions().
  59         */
  60
  61        ebda_start = *(unsigned short *)0x40e << 4;
  62        bios_start = *(unsigned short *)0x413 << 10;
  63
  64        if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX)
  65                bios_start = BIOS_START_MAX;
  66
  67        if (ebda_start > BIOS_START_MIN && ebda_start < bios_start)
  68                bios_start = ebda_start;
  69
  70        /* Place the trampoline just below the end of low memory, aligned to 4k */
  71        paging_config.trampoline_start = bios_start - TRAMPOLINE_32BIT_SIZE;
  72        paging_config.trampoline_start = round_down(paging_config.trampoline_start, PAGE_SIZE);
  73
  74        trampoline_32bit = (unsigned long *)paging_config.trampoline_start;
  75
  76        /* Preserve trampoline memory */
  77        memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
  78
  79        /* Clear trampoline memory first */
  80        memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
  81
  82        /* Copy trampoline code in place */
  83        memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
  84                        &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
  85
  86        /*
  87         * The code below prepares page table in trampoline memory.
  88         *
  89         * The new page table will be used by trampoline code for switching
  90         * from 4- to 5-level paging or vice versa.
  91         *
  92         * If switching is not required, the page table is unused: trampoline
  93         * code wouldn't touch CR3.
  94         */
  95
  96        /*
  97         * We are not going to use the page table in trampoline memory if we
  98         * are already in the desired paging mode.
  99         */
 100        if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57))
 101                goto out;
 102
 103        if (paging_config.l5_required) {
 104                /*
 105                 * For 4- to 5-level paging transition, set up current CR3 as
 106                 * the first and the only entry in a new top-level page table.
 107                 */
 108                trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC;
 109        } else {
 110                unsigned long src;
 111
 112                /*
 113                 * For 5- to 4-level paging transition, copy page table pointed
 114                 * by first entry in the current top-level page table as our
 115                 * new top-level page table.
 116                 *
 117                 * We cannot just point to the page table from trampoline as it
 118                 * may be above 4G.
 119                 */
 120                src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
 121                memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
 122                       (void *)src, PAGE_SIZE);
 123        }
 124
 125out:
 126        return paging_config;
 127}
 128
 129void cleanup_trampoline(void *pgtable)
 130{
 131        void *trampoline_pgtable;
 132
 133        trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET;
 134
 135        /*
 136         * Move the top level page table out of trampoline memory,
 137         * if it's there.
 138         */
 139        if ((void *)__native_read_cr3() == trampoline_pgtable) {
 140                memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
 141                native_write_cr3((unsigned long)pgtable);
 142        }
 143
 144        /* Restore trampoline memory */
 145        memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
 146}
 147