LXR linux/arch/x86/kernel/head

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 *  linux/arch/x86/kernel/head_64.S -- start in 32bit and switch to 64bit
   4 *
   5 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
   6 *  Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
   7 *  Copyright (C) 2000 Karsten Keil <kkeil@suse.de>
   8 *  Copyright (C) 2001,2002 Andi Kleen <ak@suse.de>
   9 *  Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com>
  10 */
  11
  12
  13#include <linux/linkage.h>
  14#include <linux/threads.h>
  15#include <linux/init.h>
  16#include <linux/pgtable.h>
  17#include <asm/segment.h>
  18#include <asm/page.h>
  19#include <asm/msr.h>
  20#include <asm/cache.h>
  21#include <asm/processor-flags.h>
  22#include <asm/percpu.h>
  23#include <asm/nops.h>
  24#include "../entry/calling.h"
  25#include <asm/export.h>
  26#include <asm/nospec-branch.h>
  27#include <asm/fixmap.h>
  28
  29/*
  30 * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
  31 * because we need identity-mapped pages.
  32 */
  33#define l4_index(x)     (((x) >> 39) & 511)
  34#define pud_index(x)    (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
  35
  36L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4)
  37L4_START_KERNEL = l4_index(__START_KERNEL_map)
  38
  39L3_START_KERNEL = pud_index(__START_KERNEL_map)
  40
  41        .text
  42        __HEAD
  43        .code64
  44SYM_CODE_START_NOALIGN(startup_64)
  45        UNWIND_HINT_EMPTY
  46        /*
  47         * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
  48         * and someone has loaded an identity mapped page table
  49         * for us.  These identity mapped page tables map all of the
  50         * kernel pages and possibly all of memory.
  51         *
  52         * %rsi holds a physical pointer to real_mode_data.
  53         *
  54         * We come here either directly from a 64bit bootloader, or from
  55         * arch/x86/boot/compressed/head_64.S.
  56         *
  57         * We only come here initially at boot nothing else comes here.
  58         *
  59         * Since we may be loaded at an address different from what we were
  60         * compiled to run at we first fixup the physical addresses in our page
  61         * tables and then reload them.
  62         */
  63
  64        /* Set up the stack for verify_cpu(), similar to initial_stack below */
  65        leaq    (__end_init_task - FRAME_SIZE)(%rip), %rsp
  66
  67        leaq    _text(%rip), %rdi
  68        pushq   %rsi
  69        call    startup_64_setup_env
  70        popq    %rsi
  71
  72        /* Now switch to __KERNEL_CS so IRET works reliably */
  73        pushq   $__KERNEL_CS
  74        leaq    .Lon_kernel_cs(%rip), %rax
  75        pushq   %rax
  76        lretq
  77
  78.Lon_kernel_cs:
  79        UNWIND_HINT_EMPTY
  80
  81        /* Sanitize CPU configuration */
  82        call verify_cpu
  83
  84        /*
  85         * Perform pagetable fixups. Additionally, if SME is active, encrypt
  86         * the kernel and retrieve the modifier (SME encryption mask if SME
  87         * is active) to be added to the initial pgdir entry that will be
  88         * programmed into CR3.
  89         */
  90        leaq    _text(%rip), %rdi
  91        pushq   %rsi
  92        call    __startup_64
  93        popq    %rsi
  94
  95        /* Form the CR3 value being sure to include the CR3 modifier */
  96        addq    $(early_top_pgt - __START_KERNEL_map), %rax
  97        jmp 1f
  98SYM_CODE_END(startup_64)
  99
 100SYM_CODE_START(secondary_startup_64)
 101        UNWIND_HINT_EMPTY
 102        /*
 103         * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
 104         * and someone has loaded a mapped page table.
 105         *
 106         * %rsi holds a physical pointer to real_mode_data.
 107         *
 108         * We come here either from startup_64 (using physical addresses)
 109         * or from trampoline.S (using virtual addresses).
 110         *
 111         * Using virtual addresses from trampoline.S removes the need
 112         * to have any identity mapped pages in the kernel page table
 113         * after the boot processor executes this code.
 114         */
 115
 116        /* Sanitize CPU configuration */
 117        call verify_cpu
 118
 119        /*
 120         * The secondary_startup_64_no_verify entry point is only used by
 121         * SEV-ES guests. In those guests the call to verify_cpu() would cause
 122         * #VC exceptions which can not be handled at this stage of secondary
 123         * CPU bringup.
 124         *
 125         * All non SEV-ES systems, especially Intel systems, need to execute
 126         * verify_cpu() above to make sure NX is enabled.
 127         */
 128SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 129        UNWIND_HINT_EMPTY
 130
 131        /*
 132         * Retrieve the modifier (SME encryption mask if SME is active) to be
 133         * added to the initial pgdir entry that will be programmed into CR3.
 134         */
 135        pushq   %rsi
 136        call    __startup_secondary_64
 137        popq    %rsi
 138
 139        /* Form the CR3 value being sure to include the CR3 modifier */
 140        addq    $(init_top_pgt - __START_KERNEL_map), %rax
 1411:
 142
 143        /* Enable PAE mode, PGE and LA57 */
 144        movl    $(X86_CR4_PAE | X86_CR4_PGE), %ecx
 145#ifdef CONFIG_X86_5LEVEL
 146        testl   $1, __pgtable_l5_enabled(%rip)
 147        jz      1f
 148        orl     $X86_CR4_LA57, %ecx
 1491:
 150#endif
 151        movq    %rcx, %cr4
 152
 153        /* Setup early boot stage 4-/5-level pagetables. */
 154        addq    phys_base(%rip), %rax
 155
 156        /*
 157         * For SEV guests: Verify that the C-bit is correct. A malicious
 158         * hypervisor could lie about the C-bit position to perform a ROP
 159         * attack on the guest by writing to the unencrypted stack and wait for
 160         * the next RET instruction.
 161         * %rsi carries pointer to realmode data and is callee-clobbered. Save
 162         * and restore it.
 163         */
 164        pushq   %rsi
 165        movq    %rax, %rdi
 166        call    sev_verify_cbit
 167        popq    %rsi
 168
 169        /* Switch to new page-table */
 170        movq    %rax, %cr3
 171
 172        /* Ensure I am executing from virtual addresses */
 173        movq    $1f, %rax
 174        ANNOTATE_RETPOLINE_SAFE
 175        jmp     *%rax
 1761:
 177        UNWIND_HINT_EMPTY
 178
 179        /*
 180         * We must switch to a new descriptor in kernel space for the GDT
 181         * because soon the kernel won't have access anymore to the userspace
 182         * addresses where we're currently running on. We have to do that here
 183         * because in 32bit we couldn't load a 64bit linear address.
 184         */
 185        lgdt    early_gdt_descr(%rip)
 186
 187        /* set up data segments */
 188        xorl %eax,%eax
 189        movl %eax,%ds
 190        movl %eax,%ss
 191        movl %eax,%es
 192
 193        /*
 194         * We don't really need to load %fs or %gs, but load them anyway
 195         * to kill any stale realmode selectors.  This allows execution
 196         * under VT hardware.
 197         */
 198        movl %eax,%fs
 199        movl %eax,%gs
 200
 201        /* Set up %gs.
 202         *
 203         * The base of %gs always points to fixed_percpu_data. If the
 204         * stack protector canary is enabled, it is located at %gs:40.
 205         * Note that, on SMP, the boot cpu uses init data section until
 206         * the per cpu areas are set up.
 207         */
 208        movl    $MSR_GS_BASE,%ecx
 209        movl    initial_gs(%rip),%eax
 210        movl    initial_gs+4(%rip),%edx
 211        wrmsr
 212
 213        /*
 214         * Setup a boot time stack - Any secondary CPU will have lost its stack
 215         * by now because the cr3-switch above unmaps the real-mode stack
 216         */
 217        movq initial_stack(%rip), %rsp
 218
 219        /* Setup and Load IDT */
 220        pushq   %rsi
 221        call    early_setup_idt
 222        popq    %rsi
 223
 224        /* Check if nx is implemented */
 225        movl    $0x80000001, %eax
 226        cpuid
 227        movl    %edx,%edi
 228
 229        /* Setup EFER (Extended Feature Enable Register) */
 230        movl    $MSR_EFER, %ecx
 231        rdmsr
 232        btsl    $_EFER_SCE, %eax        /* Enable System Call */
 233        btl     $20,%edi                /* No Execute supported? */
 234        jnc     1f
 235        btsl    $_EFER_NX, %eax
 236        btsq    $_PAGE_BIT_NX,early_pmd_flags(%rip)
 2371:      wrmsr                           /* Make changes effective */
 238
 239        /* Setup cr0 */
 240        movl    $CR0_STATE, %eax
 241        /* Make changes effective */
 242        movq    %rax, %cr0
 243
 244        /* zero EFLAGS after setting rsp */
 245        pushq $0
 246        popfq
 247
 248        /* rsi is pointer to real mode structure with interesting info.
 249           pass it to C */
 250        movq    %rsi, %rdi
 251
 252.Ljump_to_C_code:
 253        /*
 254         * Jump to run C code and to be on a real kernel address.
 255         * Since we are running on identity-mapped space we have to jump
 256         * to the full 64bit address, this is only possible as indirect
 257         * jump.  In addition we need to ensure %cs is set so we make this
 258         * a far return.
 259         *
 260         * Note: do not change to far jump indirect with 64bit offset.
 261         *
 262         * AMD does not support far jump indirect with 64bit offset.
 263         * AMD64 Architecture Programmer's Manual, Volume 3: states only
 264         *      JMP FAR mem16:16 FF /5 Far jump indirect,
 265         *              with the target specified by a far pointer in memory.
 266         *      JMP FAR mem16:32 FF /5 Far jump indirect,
 267         *              with the target specified by a far pointer in memory.
 268         *
 269         * Intel64 does support 64bit offset.
 270         * Software Developer Manual Vol 2: states:
 271         *      FF /5 JMP m16:16 Jump far, absolute indirect,
 272         *              address given in m16:16
 273         *      FF /5 JMP m16:32 Jump far, absolute indirect,
 274         *              address given in m16:32.
 275         *      REX.W + FF /5 JMP m16:64 Jump far, absolute indirect,
 276         *              address given in m16:64.
 277         */
 278        pushq   $.Lafter_lret   # put return address on stack for unwinder
 279        xorl    %ebp, %ebp      # clear frame pointer
 280        movq    initial_code(%rip), %rax
 281        pushq   $__KERNEL_CS    # set correct cs
 282        pushq   %rax            # target address in negative space
 283        lretq
 284.Lafter_lret:
 285SYM_CODE_END(secondary_startup_64)
 286
 287#include "verify_cpu.S"
 288#include "sev_verify_cbit.S"
 289
 290#ifdef CONFIG_HOTPLUG_CPU
 291/*
 292 * Boot CPU0 entry point. It's called from play_dead(). Everything has been set
 293 * up already except stack. We just set up stack here. Then call
 294 * start_secondary() via .Ljump_to_C_code.
 295 */
 296SYM_CODE_START(start_cpu0)
 297        UNWIND_HINT_EMPTY
 298        movq    initial_stack(%rip), %rsp
 299        jmp     .Ljump_to_C_code
 300SYM_CODE_END(start_cpu0)
 301#endif
 302
 303#ifdef CONFIG_AMD_MEM_ENCRYPT
 304/*
 305 * VC Exception handler used during early boot when running on kernel
 306 * addresses, but before the switch to the idt_table can be made.
 307 * The early_idt_handler_array can't be used here because it calls into a lot
 308 * of __init code and this handler is also used during CPU offlining/onlining.
 309 * Therefore this handler ends up in the .text section so that it stays around
 310 * when .init.text is freed.
 311 */
 312SYM_CODE_START_NOALIGN(vc_boot_ghcb)
 313        UNWIND_HINT_IRET_REGS offset=8
 314
 315        /* Build pt_regs */
 316        PUSH_AND_CLEAR_REGS
 317
 318        /* Call C handler */
 319        movq    %rsp, %rdi
 320        movq    ORIG_RAX(%rsp), %rsi
 321        movq    initial_vc_handler(%rip), %rax
 322        ANNOTATE_RETPOLINE_SAFE
 323        call    *%rax
 324
 325        /* Unwind pt_regs */
 326        POP_REGS
 327
 328        /* Remove Error Code */
 329        addq    $8, %rsp
 330
 331        /* Pure iret required here - don't use INTERRUPT_RETURN */
 332        iretq
 333SYM_CODE_END(vc_boot_ghcb)
 334#endif
 335
 336        /* Both SMP bootup and ACPI suspend change these variables */
 337        __REFDATA
 338        .balign 8
 339SYM_DATA(initial_code,  .quad x86_64_start_kernel)
 340SYM_DATA(initial_gs,    .quad INIT_PER_CPU_VAR(fixed_percpu_data))
 341#ifdef CONFIG_AMD_MEM_ENCRYPT
 342SYM_DATA(initial_vc_handler,    .quad handle_vc_boot_ghcb)
 343#endif
 344
 345/*
 346 * The FRAME_SIZE gap is a convention which helps the in-kernel unwinder
 347 * reliably detect the end of the stack.
 348 */
 349SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - FRAME_SIZE)
 350        __FINITDATA
 351
 352        __INIT
 353SYM_CODE_START(early_idt_handler_array)
 354        i = 0
 355        .rept NUM_EXCEPTION_VECTORS
 356        .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
 357                UNWIND_HINT_IRET_REGS
 358                pushq $0        # Dummy error code, to make stack frame uniform
 359        .else
 360                UNWIND_HINT_IRET_REGS offset=8
 361        .endif
 362        pushq $i                # 72(%rsp) Vector number
 363        jmp early_idt_handler_common
 364        UNWIND_HINT_IRET_REGS
 365        i = i + 1
 366        .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
 367        .endr
 368        UNWIND_HINT_IRET_REGS offset=16
 369SYM_CODE_END(early_idt_handler_array)
 370
 371SYM_CODE_START_LOCAL(early_idt_handler_common)
 372        /*
 373         * The stack is the hardware frame, an error code or zero, and the
 374         * vector number.
 375         */
 376        cld
 377
 378        incl early_recursion_flag(%rip)
 379
 380        /* The vector number is currently in the pt_regs->di slot. */
 381        pushq %rsi                              /* pt_regs->si */
 382        movq 8(%rsp), %rsi                      /* RSI = vector number */
 383        movq %rdi, 8(%rsp)                      /* pt_regs->di = RDI */
 384        pushq %rdx                              /* pt_regs->dx */
 385        pushq %rcx                              /* pt_regs->cx */
 386        pushq %rax                              /* pt_regs->ax */
 387        pushq %r8                               /* pt_regs->r8 */
 388        pushq %r9                               /* pt_regs->r9 */
 389        pushq %r10                              /* pt_regs->r10 */
 390        pushq %r11                              /* pt_regs->r11 */
 391        pushq %rbx                              /* pt_regs->bx */
 392        pushq %rbp                              /* pt_regs->bp */
 393        pushq %r12                              /* pt_regs->r12 */
 394        pushq %r13                              /* pt_regs->r13 */
 395        pushq %r14                              /* pt_regs->r14 */
 396        pushq %r15                              /* pt_regs->r15 */
 397        UNWIND_HINT_REGS
 398
 399        movq %rsp,%rdi          /* RDI = pt_regs; RSI is already trapnr */
 400        call do_early_exception
 401
 402        decl early_recursion_flag(%rip)
 403        jmp restore_regs_and_return_to_kernel
 404SYM_CODE_END(early_idt_handler_common)
 405
 406#ifdef CONFIG_AMD_MEM_ENCRYPT
 407/*
 408 * VC Exception handler used during very early boot. The
 409 * early_idt_handler_array can't be used because it returns via the
 410 * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
 411 *
 412 * This handler will end up in the .init.text section and not be
 413 * available to boot secondary CPUs.
 414 */
 415SYM_CODE_START_NOALIGN(vc_no_ghcb)
 416        UNWIND_HINT_IRET_REGS offset=8
 417
 418        /* Build pt_regs */
 419        PUSH_AND_CLEAR_REGS
 420
 421        /* Call C handler */
 422        movq    %rsp, %rdi
 423        movq    ORIG_RAX(%rsp), %rsi
 424        call    do_vc_no_ghcb
 425
 426        /* Unwind pt_regs */
 427        POP_REGS
 428
 429        /* Remove Error Code */
 430        addq    $8, %rsp
 431
 432        /* Pure iret required here - don't use INTERRUPT_RETURN */
 433        iretq
 434SYM_CODE_END(vc_no_ghcb)
 435#endif
 436
 437#define SYM_DATA_START_PAGE_ALIGNED(name)                       \
 438        SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE)
 439
 440#ifdef CONFIG_PAGE_TABLE_ISOLATION
 441/*
 442 * Each PGD needs to be 8k long and 8k aligned.  We do not
 443 * ever go out to userspace with these, so we do not
 444 * strictly *need* the second page, but this allows us to
 445 * have a single set_pgd() implementation that does not
 446 * need to worry about whether it has 4k or 8k to work
 447 * with.
 448 *
 449 * This ensures PGDs are 8k long:
 450 */
 451#define PTI_USER_PGD_FILL       512
 452/* This ensures they are 8k-aligned: */
 453#define SYM_DATA_START_PTI_ALIGNED(name) \
 454        SYM_START(name, SYM_L_GLOBAL, .balign 2 * PAGE_SIZE)
 455#else
 456#define SYM_DATA_START_PTI_ALIGNED(name) \
 457        SYM_DATA_START_PAGE_ALIGNED(name)
 458#define PTI_USER_PGD_FILL       0
 459#endif
 460
 461/* Automate the creation of 1 to 1 mapping pmd entries */
 462#define PMDS(START, PERM, COUNT)                        \
 463        i = 0 ;                                         \
 464        .rept (COUNT) ;                                 \
 465        .quad   (START) + (i << PMD_SHIFT) + (PERM) ;   \
 466        i = i + 1 ;                                     \
 467        .endr
 468
 469        __INITDATA
 470        .balign 4
 471
 472SYM_DATA_START_PTI_ALIGNED(early_top_pgt)
 473        .fill   512,8,0
 474        .fill   PTI_USER_PGD_FILL,8,0
 475SYM_DATA_END(early_top_pgt)
 476
 477SYM_DATA_START_PAGE_ALIGNED(early_dynamic_pgts)
 478        .fill   512*EARLY_DYNAMIC_PAGE_TABLES,8,0
 479SYM_DATA_END(early_dynamic_pgts)
 480
 481SYM_DATA(early_recursion_flag, .long 0)
 482
 483        .data
 484
 485#if defined(CONFIG_XEN_PV) || defined(CONFIG_PVH)
 486SYM_DATA_START_PTI_ALIGNED(init_top_pgt)
 487        .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
 488        .org    init_top_pgt + L4_PAGE_OFFSET*8, 0
 489        .quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
 490        .org    init_top_pgt + L4_START_KERNEL*8, 0
 491        /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
 492        .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
 493        .fill   PTI_USER_PGD_FILL,8,0
 494SYM_DATA_END(init_top_pgt)
 495
 496SYM_DATA_START_PAGE_ALIGNED(level3_ident_pgt)
 497        .quad   level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
 498        .fill   511, 8, 0
 499SYM_DATA_END(level3_ident_pgt)
 500SYM_DATA_START_PAGE_ALIGNED(level2_ident_pgt)
 501        /*
 502         * Since I easily can, map the first 1G.
 503         * Don't set NX because code runs from these pages.
 504         *
 505         * Note: This sets _PAGE_GLOBAL despite whether
 506         * the CPU supports it or it is enabled.  But,
 507         * the CPU should ignore the bit.
 508         */
 509        PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
 510SYM_DATA_END(level2_ident_pgt)
 511#else
 512SYM_DATA_START_PTI_ALIGNED(init_top_pgt)
 513        .fill   512,8,0
 514        .fill   PTI_USER_PGD_FILL,8,0
 515SYM_DATA_END(init_top_pgt)
 516#endif
 517
 518#ifdef CONFIG_X86_5LEVEL
 519SYM_DATA_START_PAGE_ALIGNED(level4_kernel_pgt)
 520        .fill   511,8,0
 521        .quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
 522SYM_DATA_END(level4_kernel_pgt)
 523#endif
 524
 525SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt)
 526        .fill   L3_START_KERNEL,8,0
 527        /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
 528        .quad   level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
 529        .quad   level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
 530SYM_DATA_END(level3_kernel_pgt)
 531
 532SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt)
 533        /*
 534         * Kernel high mapping.
 535         *
 536         * The kernel code+data+bss must be located below KERNEL_IMAGE_SIZE in
 537         * virtual address space, which is 1 GiB if RANDOMIZE_BASE is enabled,
 538         * 512 MiB otherwise.
 539         *
 540         * (NOTE: after that starts the module area, see MODULES_VADDR.)
 541         *
 542         * This table is eventually used by the kernel during normal runtime.
 543         * Care must be taken to clear out undesired bits later, like _PAGE_RW
 544         * or _PAGE_GLOBAL in some cases.
 545         */
 546        PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE/PMD_SIZE)
 547SYM_DATA_END(level2_kernel_pgt)
 548
 549SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt)
 550        .fill   (512 - 4 - FIXMAP_PMD_NUM),8,0
 551        pgtno = 0
 552        .rept (FIXMAP_PMD_NUM)
 553        .quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
 554                + _PAGE_TABLE_NOENC;
 555        pgtno = pgtno + 1
 556        .endr
 557        /* 6 MB reserved space + a 2MB hole */
 558        .fill   4,8,0
 559SYM_DATA_END(level2_fixmap_pgt)
 560
 561SYM_DATA_START_PAGE_ALIGNED(level1_fixmap_pgt)
 562        .rept (FIXMAP_PMD_NUM)
 563        .fill   512,8,0
 564        .endr
 565SYM_DATA_END(level1_fixmap_pgt)
 566
 567#undef PMDS
 568
 569        .data
 570        .align 16
 571
 572SYM_DATA(early_gdt_descr,               .word GDT_ENTRIES*8-1)
 573SYM_DATA_LOCAL(early_gdt_descr_base,    .quad INIT_PER_CPU_VAR(gdt_page))
 574
 575        .align 16
 576/* This must match the first entry in level2_kernel_pgt */
 577SYM_DATA(phys_base, .quad 0x0)
 578EXPORT_SYMBOL(phys_base)
 579
 580#include "../../x86/xen/xen-head.S"
 581
 582        __PAGE_ALIGNED_BSS
 583SYM_DATA_START_PAGE_ALIGNED(empty_zero_page)
 584        .skip PAGE_SIZE
 585SYM_DATA_END(empty_zero_page)
 586EXPORT_SYMBOL(empty_zero_page)
 587
 588