LXR linux/arch/riscv/kernel/head.S

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (C) 2012 Regents of the University of California
   4 */
   5
   6#include <asm/asm-offsets.h>
   7#include <asm/asm.h>
   8#include <linux/init.h>
   9#include <linux/linkage.h>
  10#include <asm/thread_info.h>
  11#include <asm/page.h>
  12#include <asm/pgtable.h>
  13#include <asm/csr.h>
  14#include <asm/hwcap.h>
  15#include <asm/image.h>
  16#include "efi-header.S"
  17
  18#ifdef CONFIG_XIP_KERNEL
  19.macro XIP_FIXUP_OFFSET reg
  20        REG_L t0, _xip_fixup
  21        add \reg, \reg, t0
  22.endm
  23_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET
  24#else
  25.macro XIP_FIXUP_OFFSET reg
  26.endm
  27#endif /* CONFIG_XIP_KERNEL */
  28
  29__HEAD
  30ENTRY(_start)
  31        /*
  32         * Image header expected by Linux boot-loaders. The image header data
  33         * structure is described in asm/image.h.
  34         * Do not modify it without modifying the structure and all bootloaders
  35         * that expects this header format!!
  36         */
  37#ifdef CONFIG_EFI
  38        /*
  39         * This instruction decodes to "MZ" ASCII required by UEFI.
  40         */
  41        c.li s4,-13
  42        j _start_kernel
  43#else
  44        /* jump to start kernel */
  45        j _start_kernel
  46        /* reserved */
  47        .word 0
  48#endif
  49        .balign 8
  50#ifdef CONFIG_RISCV_M_MODE
  51        /* Image load offset (0MB) from start of RAM for M-mode */
  52        .dword 0
  53#else
  54#if __riscv_xlen == 64
  55        /* Image load offset(2MB) from start of RAM */
  56        .dword 0x200000
  57#else
  58        /* Image load offset(4MB) from start of RAM */
  59        .dword 0x400000
  60#endif
  61#endif
  62        /* Effective size of kernel image */
  63        .dword _end - _start
  64        .dword __HEAD_FLAGS
  65        .word RISCV_HEADER_VERSION
  66        .word 0
  67        .dword 0
  68        .ascii RISCV_IMAGE_MAGIC
  69        .balign 4
  70        .ascii RISCV_IMAGE_MAGIC2
  71#ifdef CONFIG_EFI
  72        .word pe_head_start - _start
  73pe_head_start:
  74
  75        __EFI_PE_HEADER
  76#else
  77        .word 0
  78#endif
  79
  80.align 2
  81#ifdef CONFIG_MMU
  82relocate:
  83        /* Relocate return address */
  84        la a1, kernel_map
  85        XIP_FIXUP_OFFSET a1
  86        REG_L a1, KERNEL_MAP_VIRT_ADDR(a1)
  87        la a2, _start
  88        sub a1, a1, a2
  89        add ra, ra, a1
  90
  91        /* Point stvec to virtual address of intruction after satp write */
  92        la a2, 1f
  93        add a2, a2, a1
  94        csrw CSR_TVEC, a2
  95
  96        /* Compute satp for kernel page tables, but don't load it yet */
  97        srl a2, a0, PAGE_SHIFT
  98        li a1, SATP_MODE
  99        or a2, a2, a1
 100
 101        /*
 102         * Load trampoline page directory, which will cause us to trap to
 103         * stvec if VA != PA, or simply fall through if VA == PA.  We need a
 104         * full fence here because setup_vm() just wrote these PTEs and we need
 105         * to ensure the new translations are in use.
 106         */
 107        la a0, trampoline_pg_dir
 108        XIP_FIXUP_OFFSET a0
 109        srl a0, a0, PAGE_SHIFT
 110        or a0, a0, a1
 111        sfence.vma
 112        csrw CSR_SATP, a0
 113.align 2
 1141:
 115        /* Set trap vector to spin forever to help debug */
 116        la a0, .Lsecondary_park
 117        csrw CSR_TVEC, a0
 118
 119        /* Reload the global pointer */
 120.option push
 121.option norelax
 122        la gp, __global_pointer$
 123.option pop
 124
 125        /*
 126         * Switch to kernel page tables.  A full fence is necessary in order to
 127         * avoid using the trampoline translations, which are only correct for
 128         * the first superpage.  Fetching the fence is guarnteed to work
 129         * because that first superpage is translated the same way.
 130         */
 131        csrw CSR_SATP, a2
 132        sfence.vma
 133
 134        ret
 135#endif /* CONFIG_MMU */
 136#ifdef CONFIG_SMP
 137        .global secondary_start_sbi
 138secondary_start_sbi:
 139        /* Mask all interrupts */
 140        csrw CSR_IE, zero
 141        csrw CSR_IP, zero
 142
 143        /* Load the global pointer */
 144        .option push
 145        .option norelax
 146                la gp, __global_pointer$
 147        .option pop
 148
 149        /*
 150         * Disable FPU to detect illegal usage of
 151         * floating point in kernel space
 152         */
 153        li t0, SR_FS
 154        csrc CSR_STATUS, t0
 155
 156        /* Set trap vector to spin forever to help debug */
 157        la a3, .Lsecondary_park
 158        csrw CSR_TVEC, a3
 159
 160        slli a3, a0, LGREG
 161        la a4, __cpu_up_stack_pointer
 162        XIP_FIXUP_OFFSET a4
 163        la a5, __cpu_up_task_pointer
 164        XIP_FIXUP_OFFSET a5
 165        add a4, a3, a4
 166        add a5, a3, a5
 167        REG_L sp, (a4)
 168        REG_L tp, (a5)
 169
 170        .global secondary_start_common
 171secondary_start_common:
 172
 173#ifdef CONFIG_MMU
 174        /* Enable virtual memory and relocate to virtual address */
 175        la a0, swapper_pg_dir
 176        XIP_FIXUP_OFFSET a0
 177        call relocate
 178#endif
 179        call setup_trap_vector
 180        tail smp_callin
 181#endif /* CONFIG_SMP */
 182
 183.align 2
 184setup_trap_vector:
 185        /* Set trap vector to exception handler */
 186        la a0, handle_exception
 187        csrw CSR_TVEC, a0
 188
 189        /*
 190         * Set sup0 scratch register to 0, indicating to exception vector that
 191         * we are presently executing in kernel.
 192         */
 193        csrw CSR_SCRATCH, zero
 194        ret
 195
 196.align 2
 197.Lsecondary_park:
 198        /* We lack SMP support or have too many harts, so park this hart */
 199        wfi
 200        j .Lsecondary_park
 201
 202END(_start)
 203
 204ENTRY(_start_kernel)
 205        /* Mask all interrupts */
 206        csrw CSR_IE, zero
 207        csrw CSR_IP, zero
 208
 209#ifdef CONFIG_RISCV_M_MODE
 210        /* flush the instruction cache */
 211        fence.i
 212
 213        /* Reset all registers except ra, a0, a1 */
 214        call reset_regs
 215
 216        /*
 217         * Setup a PMP to permit access to all of memory.  Some machines may
 218         * not implement PMPs, so we set up a quick trap handler to just skip
 219         * touching the PMPs on any trap.
 220         */
 221        la a0, pmp_done
 222        csrw CSR_TVEC, a0
 223
 224        li a0, -1
 225        csrw CSR_PMPADDR0, a0
 226        li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X)
 227        csrw CSR_PMPCFG0, a0
 228.align 2
 229pmp_done:
 230
 231        /*
 232         * The hartid in a0 is expected later on, and we have no firmware
 233         * to hand it to us.
 234         */
 235        csrr a0, CSR_MHARTID
 236#endif /* CONFIG_RISCV_M_MODE */
 237
 238        /* Load the global pointer */
 239.option push
 240.option norelax
 241        la gp, __global_pointer$
 242.option pop
 243
 244        /*
 245         * Disable FPU to detect illegal usage of
 246         * floating point in kernel space
 247         */
 248        li t0, SR_FS
 249        csrc CSR_STATUS, t0
 250
 251#ifdef CONFIG_SMP
 252        li t0, CONFIG_NR_CPUS
 253        blt a0, t0, .Lgood_cores
 254        tail .Lsecondary_park
 255.Lgood_cores:
 256#endif
 257
 258#ifndef CONFIG_XIP_KERNEL
 259        /* Pick one hart to run the main boot sequence */
 260        la a3, hart_lottery
 261        li a2, 1
 262        amoadd.w a3, a2, (a3)
 263        bnez a3, .Lsecondary_start
 264
 265#else
 266        /* hart_lottery in flash contains a magic number */
 267        la a3, hart_lottery
 268        mv a2, a3
 269        XIP_FIXUP_OFFSET a2
 270        lw t1, (a3)
 271        amoswap.w t0, t1, (a2)
 272        /* first time here if hart_lottery in RAM is not set */
 273        beq t0, t1, .Lsecondary_start
 274
 275        la sp, _end + THREAD_SIZE
 276        XIP_FIXUP_OFFSET sp
 277        mv s0, a0
 278        call __copy_data
 279
 280        /* Restore a0 copy */
 281        mv a0, s0
 282#endif
 283
 284#ifndef CONFIG_XIP_KERNEL
 285        /* Clear BSS for flat non-ELF images */
 286        la a3, __bss_start
 287        la a4, __bss_stop
 288        ble a4, a3, clear_bss_done
 289clear_bss:
 290        REG_S zero, (a3)
 291        add a3, a3, RISCV_SZPTR
 292        blt a3, a4, clear_bss
 293clear_bss_done:
 294#endif
 295        /* Save hart ID and DTB physical address */
 296        mv s0, a0
 297        mv s1, a1
 298
 299        la a2, boot_cpu_hartid
 300        XIP_FIXUP_OFFSET a2
 301        REG_S a0, (a2)
 302
 303        /* Initialize page tables and relocate to virtual addresses */
 304        la sp, init_thread_union + THREAD_SIZE
 305        XIP_FIXUP_OFFSET sp
 306#ifdef CONFIG_BUILTIN_DTB
 307        la a0, __dtb_start
 308#else
 309        mv a0, s1
 310#endif /* CONFIG_BUILTIN_DTB */
 311        call setup_vm
 312#ifdef CONFIG_MMU
 313        la a0, early_pg_dir
 314        XIP_FIXUP_OFFSET a0
 315        call relocate
 316#endif /* CONFIG_MMU */
 317
 318        call setup_trap_vector
 319        /* Restore C environment */
 320        la tp, init_task
 321        sw zero, TASK_TI_CPU(tp)
 322        la sp, init_thread_union + THREAD_SIZE
 323
 324#ifdef CONFIG_KASAN
 325        call kasan_early_init
 326#endif
 327        /* Start the kernel */
 328        call soc_early_init
 329        tail start_kernel
 330
 331.Lsecondary_start:
 332#ifdef CONFIG_SMP
 333        /* Set trap vector to spin forever to help debug */
 334        la a3, .Lsecondary_park
 335        csrw CSR_TVEC, a3
 336
 337        slli a3, a0, LGREG
 338        la a1, __cpu_up_stack_pointer
 339        XIP_FIXUP_OFFSET a1
 340        la a2, __cpu_up_task_pointer
 341        XIP_FIXUP_OFFSET a2
 342        add a1, a3, a1
 343        add a2, a3, a2
 344
 345        /*
 346         * This hart didn't win the lottery, so we wait for the winning hart to
 347         * get far enough along the boot process that it should continue.
 348         */
 349.Lwait_for_cpu_up:
 350        /* FIXME: We should WFI to save some energy here. */
 351        REG_L sp, (a1)
 352        REG_L tp, (a2)
 353        beqz sp, .Lwait_for_cpu_up
 354        beqz tp, .Lwait_for_cpu_up
 355        fence
 356
 357        tail secondary_start_common
 358#endif
 359
 360END(_start_kernel)
 361
 362#ifdef CONFIG_RISCV_M_MODE
 363ENTRY(reset_regs)
 364        li      sp, 0
 365        li      gp, 0
 366        li      tp, 0
 367        li      t0, 0
 368        li      t1, 0
 369        li      t2, 0
 370        li      s0, 0
 371        li      s1, 0
 372        li      a2, 0
 373        li      a3, 0
 374        li      a4, 0
 375        li      a5, 0
 376        li      a6, 0
 377        li      a7, 0
 378        li      s2, 0
 379        li      s3, 0
 380        li      s4, 0
 381        li      s5, 0
 382        li      s6, 0
 383        li      s7, 0
 384        li      s8, 0
 385        li      s9, 0
 386        li      s10, 0
 387        li      s11, 0
 388        li      t3, 0
 389        li      t4, 0
 390        li      t5, 0
 391        li      t6, 0
 392        csrw    CSR_SCRATCH, 0
 393
 394#ifdef CONFIG_FPU
 395        csrr    t0, CSR_MISA
 396        andi    t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)
 397        beqz    t0, .Lreset_regs_done
 398
 399        li      t1, SR_FS
 400        csrs    CSR_STATUS, t1
 401        fmv.s.x f0, zero
 402        fmv.s.x f1, zero
 403        fmv.s.x f2, zero
 404        fmv.s.x f3, zero
 405        fmv.s.x f4, zero
 406        fmv.s.x f5, zero
 407        fmv.s.x f6, zero
 408        fmv.s.x f7, zero
 409        fmv.s.x f8, zero
 410        fmv.s.x f9, zero
 411        fmv.s.x f10, zero
 412        fmv.s.x f11, zero
 413        fmv.s.x f12, zero
 414        fmv.s.x f13, zero
 415        fmv.s.x f14, zero
 416        fmv.s.x f15, zero
 417        fmv.s.x f16, zero
 418        fmv.s.x f17, zero
 419        fmv.s.x f18, zero
 420        fmv.s.x f19, zero
 421        fmv.s.x f20, zero
 422        fmv.s.x f21, zero
 423        fmv.s.x f22, zero
 424        fmv.s.x f23, zero
 425        fmv.s.x f24, zero
 426        fmv.s.x f25, zero
 427        fmv.s.x f26, zero
 428        fmv.s.x f27, zero
 429        fmv.s.x f28, zero
 430        fmv.s.x f29, zero
 431        fmv.s.x f30, zero
 432        fmv.s.x f31, zero
 433        csrw    fcsr, 0
 434        /* note that the caller must clear SR_FS */
 435#endif /* CONFIG_FPU */
 436.Lreset_regs_done:
 437        ret
 438END(reset_regs)
 439#endif /* CONFIG_RISCV_M_MODE */
 440
 441__PAGE_ALIGNED_BSS
 442        /* Empty zero page */
 443        .balign PAGE_SIZE
 444