LXR linux/arch/x86/kernel/head

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 *
   4 *  Copyright (C) 1991, 1992  Linus Torvalds
   5 *
   6 *  Enhanced CPU detection and feature setting code by Mike Jagdis
   7 *  and Martin Mares, November 1997.
   8 */
   9
  10.text
  11#include <linux/threads.h>
  12#include <linux/init.h>
  13#include <linux/linkage.h>
  14#include <asm/segment.h>
  15#include <asm/page_types.h>
  16#include <asm/pgtable_types.h>
  17#include <asm/cache.h>
  18#include <asm/thread_info.h>
  19#include <asm/asm-offsets.h>
  20#include <asm/setup.h>
  21#include <asm/processor-flags.h>
  22#include <asm/msr-index.h>
  23#include <asm/cpufeatures.h>
  24#include <asm/percpu.h>
  25#include <asm/nops.h>
  26#include <asm/bootparam.h>
  27#include <asm/export.h>
  28#include <asm/pgtable_32.h>
  29
  30/* Physical address */
  31#define pa(X) ((X) - __PAGE_OFFSET)
  32
  33/*
  34 * References to members of the new_cpu_data structure.
  35 */
  36
  37#define X86             new_cpu_data+CPUINFO_x86
  38#define X86_VENDOR      new_cpu_data+CPUINFO_x86_vendor
  39#define X86_MODEL       new_cpu_data+CPUINFO_x86_model
  40#define X86_STEPPING    new_cpu_data+CPUINFO_x86_stepping
  41#define X86_HARD_MATH   new_cpu_data+CPUINFO_hard_math
  42#define X86_CPUID       new_cpu_data+CPUINFO_cpuid_level
  43#define X86_CAPABILITY  new_cpu_data+CPUINFO_x86_capability
  44#define X86_VENDOR_ID   new_cpu_data+CPUINFO_x86_vendor_id
  45
  46
  47#define SIZEOF_PTREGS 17*4
  48
  49/*
  50 * Worst-case size of the kernel mapping we need to make:
  51 * a relocatable kernel can live anywhere in lowmem, so we need to be able
  52 * to map all of lowmem.
  53 */
  54KERNEL_PAGES = LOWMEM_PAGES
  55
  56INIT_MAP_SIZE = PAGE_TABLE_SIZE(KERNEL_PAGES) * PAGE_SIZE
  57RESERVE_BRK(pagetables, INIT_MAP_SIZE)
  58
  59/*
  60 * 32-bit kernel entrypoint; only used by the boot CPU.  On entry,
  61 * %esi points to the real-mode code as a 32-bit pointer.
  62 * CS and DS must be 4 GB flat segments, but we don't depend on
  63 * any particular GDT layout, because we load our own as soon as we
  64 * can.
  65 */
  66__HEAD
  67ENTRY(startup_32)
  68        movl pa(initial_stack),%ecx
  69        
  70        /* test KEEP_SEGMENTS flag to see if the bootloader is asking
  71                us to not reload segments */
  72        testb $KEEP_SEGMENTS, BP_loadflags(%esi)
  73        jnz 2f
  74
  75/*
  76 * Set segments to known values.
  77 */
  78        lgdt pa(boot_gdt_descr)
  79        movl $(__BOOT_DS),%eax
  80        movl %eax,%ds
  81        movl %eax,%es
  82        movl %eax,%fs
  83        movl %eax,%gs
  84        movl %eax,%ss
  852:
  86        leal -__PAGE_OFFSET(%ecx),%esp
  87
  88/*
  89 * Clear BSS first so that there are no surprises...
  90 */
  91        cld
  92        xorl %eax,%eax
  93        movl $pa(__bss_start),%edi
  94        movl $pa(__bss_stop),%ecx
  95        subl %edi,%ecx
  96        shrl $2,%ecx
  97        rep ; stosl
  98/*
  99 * Copy bootup parameters out of the way.
 100 * Note: %esi still has the pointer to the real-mode data.
 101 * With the kexec as boot loader, parameter segment might be loaded beyond
 102 * kernel image and might not even be addressable by early boot page tables.
 103 * (kexec on panic case). Hence copy out the parameters before initializing
 104 * page tables.
 105 */
 106        movl $pa(boot_params),%edi
 107        movl $(PARAM_SIZE/4),%ecx
 108        cld
 109        rep
 110        movsl
 111        movl pa(boot_params) + NEW_CL_POINTER,%esi
 112        andl %esi,%esi
 113        jz 1f                   # No command line
 114        movl $pa(boot_command_line),%edi
 115        movl $(COMMAND_LINE_SIZE/4),%ecx
 116        rep
 117        movsl
 1181:
 119
 120#ifdef CONFIG_OLPC
 121        /* save OFW's pgdir table for later use when calling into OFW */
 122        movl %cr3, %eax
 123        movl %eax, pa(olpc_ofw_pgd)
 124#endif
 125
 126#ifdef CONFIG_MICROCODE
 127        /* Early load ucode on BSP. */
 128        call load_ucode_bsp
 129#endif
 130
 131        /* Create early pagetables. */
 132        call  mk_early_pgtbl_32
 133
 134        /* Do early initialization of the fixmap area */
 135        movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
 136#ifdef  CONFIG_X86_PAE
 137#define KPMDS (((-__PAGE_OFFSET) >> 30) & 3) /* Number of kernel PMDs */
 138        movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8)
 139#else
 140        movl %eax,pa(initial_page_table+0xffc)
 141#endif
 142
 143#ifdef CONFIG_PARAVIRT
 144        /* This is can only trip for a broken bootloader... */
 145        cmpw $0x207, pa(boot_params + BP_version)
 146        jb .Ldefault_entry
 147
 148        /* Paravirt-compatible boot parameters.  Look to see what architecture
 149                we're booting under. */
 150        movl pa(boot_params + BP_hardware_subarch), %eax
 151        cmpl $num_subarch_entries, %eax
 152        jae .Lbad_subarch
 153
 154        movl pa(subarch_entries)(,%eax,4), %eax
 155        subl $__PAGE_OFFSET, %eax
 156        jmp *%eax
 157
 158.Lbad_subarch:
 159WEAK(xen_entry)
 160        /* Unknown implementation; there's really
 161           nothing we can do at this point. */
 162        ud2a
 163
 164        __INITDATA
 165
 166subarch_entries:
 167        .long .Ldefault_entry           /* normal x86/PC */
 168        .long xen_entry                 /* Xen hypervisor */
 169        .long .Ldefault_entry           /* Moorestown MID */
 170num_subarch_entries = (. - subarch_entries) / 4
 171.previous
 172#else
 173        jmp .Ldefault_entry
 174#endif /* CONFIG_PARAVIRT */
 175
 176#ifdef CONFIG_HOTPLUG_CPU
 177/*
 178 * Boot CPU0 entry point. It's called from play_dead(). Everything has been set
 179 * up already except stack. We just set up stack here. Then call
 180 * start_secondary().
 181 */
 182ENTRY(start_cpu0)
 183        movl initial_stack, %ecx
 184        movl %ecx, %esp
 185        call *(initial_code)
 1861:      jmp 1b
 187ENDPROC(start_cpu0)
 188#endif
 189
 190/*
 191 * Non-boot CPU entry point; entered from trampoline.S
 192 * We can't lgdt here, because lgdt itself uses a data segment, but
 193 * we know the trampoline has already loaded the boot_gdt for us.
 194 *
 195 * If cpu hotplug is not supported then this code can go in init section
 196 * which will be freed later
 197 */
 198ENTRY(startup_32_smp)
 199        cld
 200        movl $(__BOOT_DS),%eax
 201        movl %eax,%ds
 202        movl %eax,%es
 203        movl %eax,%fs
 204        movl %eax,%gs
 205        movl pa(initial_stack),%ecx
 206        movl %eax,%ss
 207        leal -__PAGE_OFFSET(%ecx),%esp
 208
 209#ifdef CONFIG_MICROCODE
 210        /* Early load ucode on AP. */
 211        call load_ucode_ap
 212#endif
 213
 214.Ldefault_entry:
 215        movl $(CR0_STATE & ~X86_CR0_PG),%eax
 216        movl %eax,%cr0
 217
 218/*
 219 * We want to start out with EFLAGS unambiguously cleared. Some BIOSes leave
 220 * bits like NT set. This would confuse the debugger if this code is traced. So
 221 * initialize them properly now before switching to protected mode. That means
 222 * DF in particular (even though we have cleared it earlier after copying the
 223 * command line) because GCC expects it.
 224 */
 225        pushl $0
 226        popfl
 227
 228/*
 229 * New page tables may be in 4Mbyte page mode and may be using the global pages.
 230 *
 231 * NOTE! If we are on a 486 we may have no cr4 at all! Specifically, cr4 exists
 232 * if and only if CPUID exists and has flags other than the FPU flag set.
 233 */
 234        movl $-1,pa(X86_CPUID)          # preset CPUID level
 235        movl $X86_EFLAGS_ID,%ecx
 236        pushl %ecx
 237        popfl                           # set EFLAGS=ID
 238        pushfl
 239        popl %eax                       # get EFLAGS
 240        testl $X86_EFLAGS_ID,%eax       # did EFLAGS.ID remained set?
 241        jz .Lenable_paging              # hw disallowed setting of ID bit
 242                                        # which means no CPUID and no CR4
 243
 244        xorl %eax,%eax
 245        cpuid
 246        movl %eax,pa(X86_CPUID)         # save largest std CPUID function
 247
 248        movl $1,%eax
 249        cpuid
 250        andl $~1,%edx                   # Ignore CPUID.FPU
 251        jz .Lenable_paging              # No flags or only CPUID.FPU = no CR4
 252
 253        movl pa(mmu_cr4_features),%eax
 254        movl %eax,%cr4
 255
 256        testb $X86_CR4_PAE, %al         # check if PAE is enabled
 257        jz .Lenable_paging
 258
 259        /* Check if extended functions are implemented */
 260        movl $0x80000000, %eax
 261        cpuid
 262        /* Value must be in the range 0x80000001 to 0x8000ffff */
 263        subl $0x80000001, %eax
 264        cmpl $(0x8000ffff-0x80000001), %eax
 265        ja .Lenable_paging
 266
 267        /* Clear bogus XD_DISABLE bits */
 268        call verify_cpu
 269
 270        mov $0x80000001, %eax
 271        cpuid
 272        /* Execute Disable bit supported? */
 273        btl $(X86_FEATURE_NX & 31), %edx
 274        jnc .Lenable_paging
 275
 276        /* Setup EFER (Extended Feature Enable Register) */
 277        movl $MSR_EFER, %ecx
 278        rdmsr
 279
 280        btsl $_EFER_NX, %eax
 281        /* Make changes effective */
 282        wrmsr
 283
 284.Lenable_paging:
 285
 286/*
 287 * Enable paging
 288 */
 289        movl $pa(initial_page_table), %eax
 290        movl %eax,%cr3          /* set the page table pointer.. */
 291        movl $CR0_STATE,%eax
 292        movl %eax,%cr0          /* ..and set paging (PG) bit */
 293        ljmp $__BOOT_CS,$1f     /* Clear prefetch and normalize %eip */
 2941:
 295        /* Shift the stack pointer to a virtual address */
 296        addl $__PAGE_OFFSET, %esp
 297
 298/*
 299 * start system 32-bit setup. We need to re-do some of the things done
 300 * in 16-bit mode for the "real" operations.
 301 */
 302        movl setup_once_ref,%eax
 303        andl %eax,%eax
 304        jz 1f                           # Did we do this already?
 305        call *%eax
 3061:
 307
 308/*
 309 * Check if it is 486
 310 */
 311        movb $4,X86                     # at least 486
 312        cmpl $-1,X86_CPUID
 313        je .Lis486
 314
 315        /* get vendor info */
 316        xorl %eax,%eax                  # call CPUID with 0 -> return vendor ID
 317        cpuid
 318        movl %eax,X86_CPUID             # save CPUID level
 319        movl %ebx,X86_VENDOR_ID         # lo 4 chars
 320        movl %edx,X86_VENDOR_ID+4       # next 4 chars
 321        movl %ecx,X86_VENDOR_ID+8       # last 4 chars
 322
 323        orl %eax,%eax                   # do we have processor info as well?
 324        je .Lis486
 325
 326        movl $1,%eax            # Use the CPUID instruction to get CPU type
 327        cpuid
 328        movb %al,%cl            # save reg for future use
 329        andb $0x0f,%ah          # mask processor family
 330        movb %ah,X86
 331        andb $0xf0,%al          # mask model
 332        shrb $4,%al
 333        movb %al,X86_MODEL
 334        andb $0x0f,%cl          # mask mask revision
 335        movb %cl,X86_STEPPING
 336        movl %edx,X86_CAPABILITY
 337
 338.Lis486:
 339        movl $0x50022,%ecx      # set AM, WP, NE and MP
 340        movl %cr0,%eax
 341        andl $0x80000011,%eax   # Save PG,PE,ET
 342        orl %ecx,%eax
 343        movl %eax,%cr0
 344
 345        lgdt early_gdt_descr
 346        ljmp $(__KERNEL_CS),$1f
 3471:      movl $(__KERNEL_DS),%eax        # reload all the segment registers
 348        movl %eax,%ss                   # after changing gdt.
 349
 350        movl $(__USER_DS),%eax          # DS/ES contains default USER segment
 351        movl %eax,%ds
 352        movl %eax,%es
 353
 354        movl $(__KERNEL_PERCPU), %eax
 355        movl %eax,%fs                   # set this cpu's percpu
 356
 357        movl $(__KERNEL_STACK_CANARY),%eax
 358        movl %eax,%gs
 359
 360        xorl %eax,%eax                  # Clear LDT
 361        lldt %ax
 362
 363        call *(initial_code)
 3641:      jmp 1b
 365ENDPROC(startup_32_smp)
 366
 367#include "verify_cpu.S"
 368
 369/*
 370 *  setup_once
 371 *
 372 *  The setup work we only want to run on the BSP.
 373 *
 374 *  Warning: %esi is live across this function.
 375 */
 376__INIT
 377setup_once:
 378#ifdef CONFIG_STACKPROTECTOR
 379        /*
 380         * Configure the stack canary. The linker can't handle this by
 381         * relocation.  Manually set base address in stack canary
 382         * segment descriptor.
 383         */
 384        movl $gdt_page,%eax
 385        movl $stack_canary,%ecx
 386        movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
 387        shrl $16, %ecx
 388        movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
 389        movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax)
 390#endif
 391
 392        andl $0,setup_once_ref  /* Once is enough, thanks */
 393        ret
 394
 395ENTRY(early_idt_handler_array)
 396        # 36(%esp) %eflags
 397        # 32(%esp) %cs
 398        # 28(%esp) %eip
 399        # 24(%rsp) error code
 400        i = 0
 401        .rept NUM_EXCEPTION_VECTORS
 402        .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
 403        pushl $0                # Dummy error code, to make stack frame uniform
 404        .endif
 405        pushl $i                # 20(%esp) Vector number
 406        jmp early_idt_handler_common
 407        i = i + 1
 408        .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
 409        .endr
 410ENDPROC(early_idt_handler_array)
 411        
 412early_idt_handler_common:
 413        /*
 414         * The stack is the hardware frame, an error code or zero, and the
 415         * vector number.
 416         */
 417        cld
 418
 419        incl %ss:early_recursion_flag
 420
 421        /* The vector number is in pt_regs->gs */
 422
 423        cld
 424        pushl   %fs             /* pt_regs->fs (__fsh varies by model) */
 425        pushl   %es             /* pt_regs->es (__esh varies by model) */
 426        pushl   %ds             /* pt_regs->ds (__dsh varies by model) */
 427        pushl   %eax            /* pt_regs->ax */
 428        pushl   %ebp            /* pt_regs->bp */
 429        pushl   %edi            /* pt_regs->di */
 430        pushl   %esi            /* pt_regs->si */
 431        pushl   %edx            /* pt_regs->dx */
 432        pushl   %ecx            /* pt_regs->cx */
 433        pushl   %ebx            /* pt_regs->bx */
 434
 435        /* Fix up DS and ES */
 436        movl    $(__KERNEL_DS), %ecx
 437        movl    %ecx, %ds
 438        movl    %ecx, %es
 439
 440        /* Load the vector number into EDX */
 441        movl    PT_GS(%esp), %edx
 442
 443        /* Load GS into pt_regs->gs (and maybe clobber __gsh) */
 444        movw    %gs, PT_GS(%esp)
 445
 446        movl    %esp, %eax      /* args are pt_regs (EAX), trapnr (EDX) */
 447        call    early_fixup_exception
 448
 449        popl    %ebx            /* pt_regs->bx */
 450        popl    %ecx            /* pt_regs->cx */
 451        popl    %edx            /* pt_regs->dx */
 452        popl    %esi            /* pt_regs->si */
 453        popl    %edi            /* pt_regs->di */
 454        popl    %ebp            /* pt_regs->bp */
 455        popl    %eax            /* pt_regs->ax */
 456        popl    %ds             /* pt_regs->ds (always ignores __dsh) */
 457        popl    %es             /* pt_regs->es (always ignores __esh) */
 458        popl    %fs             /* pt_regs->fs (always ignores __fsh) */
 459        popl    %gs             /* pt_regs->gs (always ignores __gsh) */
 460        decl    %ss:early_recursion_flag
 461        addl    $4, %esp        /* pop pt_regs->orig_ax */
 462        iret
 463ENDPROC(early_idt_handler_common)
 464
 465/* This is the default interrupt "handler" :-) */
 466ENTRY(early_ignore_irq)
 467        cld
 468#ifdef CONFIG_PRINTK
 469        pushl %eax
 470        pushl %ecx
 471        pushl %edx
 472        pushl %es
 473        pushl %ds
 474        movl $(__KERNEL_DS),%eax
 475        movl %eax,%ds
 476        movl %eax,%es
 477        cmpl $2,early_recursion_flag
 478        je hlt_loop
 479        incl early_recursion_flag
 480        pushl 16(%esp)
 481        pushl 24(%esp)
 482        pushl 32(%esp)
 483        pushl 40(%esp)
 484        pushl $int_msg
 485        call printk
 486
 487        call dump_stack
 488
 489        addl $(5*4),%esp
 490        popl %ds
 491        popl %es
 492        popl %edx
 493        popl %ecx
 494        popl %eax
 495#endif
 496        iret
 497
 498hlt_loop:
 499        hlt
 500        jmp hlt_loop
 501ENDPROC(early_ignore_irq)
 502
 503__INITDATA
 504        .align 4
 505GLOBAL(early_recursion_flag)
 506        .long 0
 507
 508__REFDATA
 509        .align 4
 510ENTRY(initial_code)
 511        .long i386_start_kernel
 512ENTRY(setup_once_ref)
 513        .long setup_once
 514
 515#ifdef CONFIG_PAGE_TABLE_ISOLATION
 516#define PGD_ALIGN       (2 * PAGE_SIZE)
 517#define PTI_USER_PGD_FILL       1024
 518#else
 519#define PGD_ALIGN       (PAGE_SIZE)
 520#define PTI_USER_PGD_FILL       0
 521#endif
 522/*
 523 * BSS section
 524 */
 525__PAGE_ALIGNED_BSS
 526        .align PGD_ALIGN
 527#ifdef CONFIG_X86_PAE
 528.globl initial_pg_pmd
 529initial_pg_pmd:
 530        .fill 1024*KPMDS,4,0
 531#else
 532.globl initial_page_table
 533initial_page_table:
 534        .fill 1024,4,0
 535#endif
 536        .align PGD_ALIGN
 537initial_pg_fixmap:
 538        .fill 1024,4,0
 539.globl swapper_pg_dir
 540        .align PGD_ALIGN
 541swapper_pg_dir:
 542        .fill 1024,4,0
 543        .fill PTI_USER_PGD_FILL,4,0
 544.globl empty_zero_page
 545empty_zero_page:
 546        .fill 4096,1,0
 547EXPORT_SYMBOL(empty_zero_page)
 548
 549/*
 550 * This starts the data section.
 551 */
 552#ifdef CONFIG_X86_PAE
 553__PAGE_ALIGNED_DATA
 554        /* Page-aligned for the benefit of paravirt? */
 555        .align PGD_ALIGN
 556ENTRY(initial_page_table)
 557        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0     /* low identity map */
 558# if KPMDS == 3
 559        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0
 560        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
 561        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0
 562# elif KPMDS == 2
 563        .long   0,0
 564        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0
 565        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
 566# elif KPMDS == 1
 567        .long   0,0
 568        .long   0,0
 569        .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0
 570# else
 571#  error "Kernel PMDs should be 1, 2 or 3"
 572# endif
 573        .align PAGE_SIZE                /* needs to be page-sized too */
 574#endif
 575
 576.data
 577.balign 4
 578ENTRY(initial_stack)
 579        /*
 580         * The SIZEOF_PTREGS gap is a convention which helps the in-kernel
 581         * unwinder reliably detect the end of the stack.
 582         */
 583        .long init_thread_union + THREAD_SIZE - SIZEOF_PTREGS - \
 584              TOP_OF_KERNEL_STACK_PADDING;
 585
 586__INITRODATA
 587int_msg:
 588        .asciz "Unknown interrupt or fault at: %p %p %p\n"
 589
 590#include "../../x86/xen/xen-head.S"
 591
 592/*
 593 * The IDT and GDT 'descriptors' are a strange 48-bit object
 594 * only used by the lidt and lgdt instructions. They are not
 595 * like usual segment descriptors - they consist of a 16-bit
 596 * segment size, and 32-bit linear address value:
 597 */
 598
 599        .data
 600.globl boot_gdt_descr
 601
 602        ALIGN
 603# early boot GDT descriptor (must use 1:1 address mapping)
 604        .word 0                         # 32 bit align gdt_desc.address
 605boot_gdt_descr:
 606        .word __BOOT_DS+7
 607        .long boot_gdt - __PAGE_OFFSET
 608
 609# boot GDT descriptor (later on used by CPU#0):
 610        .word 0                         # 32 bit align gdt_desc.address
 611ENTRY(early_gdt_descr)
 612        .word GDT_ENTRIES*8-1
 613        .long gdt_page                  /* Overwritten for secondary CPUs */
 614
 615/*
 616 * The boot_gdt must mirror the equivalent in setup.S and is
 617 * used only for booting.
 618 */
 619        .align L1_CACHE_BYTES
 620ENTRY(boot_gdt)
 621        .fill GDT_ENTRY_BOOT_CS,8,0
 622        .quad 0x00cf9a000000ffff        /* kernel 4GB code at 0x00000000 */
 623        .quad 0x00cf92000000ffff        /* kernel 4GB data at 0x00000000 */
 624