LXR linux/arch/arm64/kernel/entry-common.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Exception handling code
   4 *
   5 * Copyright (C) 2019 ARM Ltd.
   6 */
   7
   8#include <linux/context_tracking.h>
   9#include <linux/ptrace.h>
  10#include <linux/thread_info.h>
  11
  12#include <asm/cpufeature.h>
  13#include <asm/daifflags.h>
  14#include <asm/esr.h>
  15#include <asm/exception.h>
  16#include <asm/kprobes.h>
  17#include <asm/mmu.h>
  18#include <asm/sysreg.h>
  19
  20static void notrace el1_abort(struct pt_regs *regs, unsigned long esr)
  21{
  22        unsigned long far = read_sysreg(far_el1);
  23
  24        local_daif_inherit(regs);
  25        far = untagged_addr(far);
  26        do_mem_abort(far, esr, regs);
  27}
  28NOKPROBE_SYMBOL(el1_abort);
  29
  30static void notrace el1_pc(struct pt_regs *regs, unsigned long esr)
  31{
  32        unsigned long far = read_sysreg(far_el1);
  33
  34        local_daif_inherit(regs);
  35        do_sp_pc_abort(far, esr, regs);
  36}
  37NOKPROBE_SYMBOL(el1_pc);
  38
  39static void notrace el1_undef(struct pt_regs *regs)
  40{
  41        local_daif_inherit(regs);
  42        do_undefinstr(regs);
  43}
  44NOKPROBE_SYMBOL(el1_undef);
  45
  46static void notrace el1_inv(struct pt_regs *regs, unsigned long esr)
  47{
  48        local_daif_inherit(regs);
  49        bad_mode(regs, 0, esr);
  50}
  51NOKPROBE_SYMBOL(el1_inv);
  52
  53static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)
  54{
  55        unsigned long far = read_sysreg(far_el1);
  56
  57        /*
  58         * The CPU masked interrupts, and we are leaving them masked during
  59         * do_debug_exception(). Update PMR as if we had called
  60         * local_mask_daif().
  61         */
  62        if (system_uses_irq_prio_masking())
  63                gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
  64
  65        do_debug_exception(far, esr, regs);
  66}
  67NOKPROBE_SYMBOL(el1_dbg);
  68
  69asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)
  70{
  71        unsigned long esr = read_sysreg(esr_el1);
  72
  73        switch (ESR_ELx_EC(esr)) {
  74        case ESR_ELx_EC_DABT_CUR:
  75        case ESR_ELx_EC_IABT_CUR:
  76                el1_abort(regs, esr);
  77                break;
  78        /*
  79         * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
  80         * recursive exception when trying to push the initial pt_regs.
  81         */
  82        case ESR_ELx_EC_PC_ALIGN:
  83                el1_pc(regs, esr);
  84                break;
  85        case ESR_ELx_EC_SYS64:
  86        case ESR_ELx_EC_UNKNOWN:
  87                el1_undef(regs);
  88                break;
  89        case ESR_ELx_EC_BREAKPT_CUR:
  90        case ESR_ELx_EC_SOFTSTP_CUR:
  91        case ESR_ELx_EC_WATCHPT_CUR:
  92        case ESR_ELx_EC_BRK64:
  93                el1_dbg(regs, esr);
  94                break;
  95        default:
  96                el1_inv(regs, esr);
  97        };
  98}
  99NOKPROBE_SYMBOL(el1_sync_handler);
 100
 101static void notrace el0_da(struct pt_regs *regs, unsigned long esr)
 102{
 103        unsigned long far = read_sysreg(far_el1);
 104
 105        user_exit_irqoff();
 106        local_daif_restore(DAIF_PROCCTX);
 107        far = untagged_addr(far);
 108        do_mem_abort(far, esr, regs);
 109}
 110NOKPROBE_SYMBOL(el0_da);
 111
 112static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)
 113{
 114        unsigned long far = read_sysreg(far_el1);
 115
 116        /*
 117         * We've taken an instruction abort from userspace and not yet
 118         * re-enabled IRQs. If the address is a kernel address, apply
 119         * BP hardening prior to enabling IRQs and pre-emption.
 120         */
 121        if (!is_ttbr0_addr(far))
 122                arm64_apply_bp_hardening();
 123
 124        user_exit_irqoff();
 125        local_daif_restore(DAIF_PROCCTX);
 126        do_mem_abort(far, esr, regs);
 127}
 128NOKPROBE_SYMBOL(el0_ia);
 129
 130static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
 131{
 132        user_exit_irqoff();
 133        local_daif_restore(DAIF_PROCCTX);
 134        do_fpsimd_acc(esr, regs);
 135}
 136NOKPROBE_SYMBOL(el0_fpsimd_acc);
 137
 138static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr)
 139{
 140        user_exit_irqoff();
 141        local_daif_restore(DAIF_PROCCTX);
 142        do_sve_acc(esr, regs);
 143}
 144NOKPROBE_SYMBOL(el0_sve_acc);
 145
 146static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
 147{
 148        user_exit_irqoff();
 149        local_daif_restore(DAIF_PROCCTX);
 150        do_fpsimd_exc(esr, regs);
 151}
 152NOKPROBE_SYMBOL(el0_fpsimd_exc);
 153
 154static void notrace el0_sys(struct pt_regs *regs, unsigned long esr)
 155{
 156        user_exit_irqoff();
 157        local_daif_restore(DAIF_PROCCTX);
 158        do_sysinstr(esr, regs);
 159}
 160NOKPROBE_SYMBOL(el0_sys);
 161
 162static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)
 163{
 164        unsigned long far = read_sysreg(far_el1);
 165
 166        if (!is_ttbr0_addr(instruction_pointer(regs)))
 167                arm64_apply_bp_hardening();
 168
 169        user_exit_irqoff();
 170        local_daif_restore(DAIF_PROCCTX);
 171        do_sp_pc_abort(far, esr, regs);
 172}
 173NOKPROBE_SYMBOL(el0_pc);
 174
 175static void notrace el0_sp(struct pt_regs *regs, unsigned long esr)
 176{
 177        user_exit_irqoff();
 178        local_daif_restore(DAIF_PROCCTX_NOIRQ);
 179        do_sp_pc_abort(regs->sp, esr, regs);
 180}
 181NOKPROBE_SYMBOL(el0_sp);
 182
 183static void notrace el0_undef(struct pt_regs *regs)
 184{
 185        user_exit_irqoff();
 186        local_daif_restore(DAIF_PROCCTX);
 187        do_undefinstr(regs);
 188}
 189NOKPROBE_SYMBOL(el0_undef);
 190
 191static void notrace el0_inv(struct pt_regs *regs, unsigned long esr)
 192{
 193        user_exit_irqoff();
 194        local_daif_restore(DAIF_PROCCTX);
 195        bad_el0_sync(regs, 0, esr);
 196}
 197NOKPROBE_SYMBOL(el0_inv);
 198
 199static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)
 200{
 201        /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
 202        unsigned long far = read_sysreg(far_el1);
 203
 204        if (system_uses_irq_prio_masking())
 205                gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 206
 207        user_exit_irqoff();
 208        do_debug_exception(far, esr, regs);
 209        local_daif_restore(DAIF_PROCCTX_NOIRQ);
 210}
 211NOKPROBE_SYMBOL(el0_dbg);
 212
 213static void notrace el0_svc(struct pt_regs *regs)
 214{
 215        if (system_uses_irq_prio_masking())
 216                gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 217
 218        do_el0_svc(regs);
 219}
 220NOKPROBE_SYMBOL(el0_svc);
 221
 222asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
 223{
 224        unsigned long esr = read_sysreg(esr_el1);
 225
 226        switch (ESR_ELx_EC(esr)) {
 227        case ESR_ELx_EC_SVC64:
 228                el0_svc(regs);
 229                break;
 230        case ESR_ELx_EC_DABT_LOW:
 231                el0_da(regs, esr);
 232                break;
 233        case ESR_ELx_EC_IABT_LOW:
 234                el0_ia(regs, esr);
 235                break;
 236        case ESR_ELx_EC_FP_ASIMD:
 237                el0_fpsimd_acc(regs, esr);
 238                break;
 239        case ESR_ELx_EC_SVE:
 240                el0_sve_acc(regs, esr);
 241                break;
 242        case ESR_ELx_EC_FP_EXC64:
 243                el0_fpsimd_exc(regs, esr);
 244                break;
 245        case ESR_ELx_EC_SYS64:
 246        case ESR_ELx_EC_WFx:
 247                el0_sys(regs, esr);
 248                break;
 249        case ESR_ELx_EC_SP_ALIGN:
 250                el0_sp(regs, esr);
 251                break;
 252        case ESR_ELx_EC_PC_ALIGN:
 253                el0_pc(regs, esr);
 254                break;
 255        case ESR_ELx_EC_UNKNOWN:
 256                el0_undef(regs);
 257                break;
 258        case ESR_ELx_EC_BREAKPT_LOW:
 259        case ESR_ELx_EC_SOFTSTP_LOW:
 260        case ESR_ELx_EC_WATCHPT_LOW:
 261        case ESR_ELx_EC_BRK64:
 262                el0_dbg(regs, esr);
 263                break;
 264        default:
 265                el0_inv(regs, esr);
 266        }
 267}
 268NOKPROBE_SYMBOL(el0_sync_handler);
 269
 270#ifdef CONFIG_COMPAT
 271static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr)
 272{
 273        user_exit_irqoff();
 274        local_daif_restore(DAIF_PROCCTX);
 275        do_cp15instr(esr, regs);
 276}
 277NOKPROBE_SYMBOL(el0_cp15);
 278
 279static void notrace el0_svc_compat(struct pt_regs *regs)
 280{
 281        if (system_uses_irq_prio_masking())
 282                gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 283
 284        do_el0_svc_compat(regs);
 285}
 286NOKPROBE_SYMBOL(el0_svc_compat);
 287
 288asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)
 289{
 290        unsigned long esr = read_sysreg(esr_el1);
 291
 292        switch (ESR_ELx_EC(esr)) {
 293        case ESR_ELx_EC_SVC32:
 294                el0_svc_compat(regs);
 295                break;
 296        case ESR_ELx_EC_DABT_LOW:
 297                el0_da(regs, esr);
 298                break;
 299        case ESR_ELx_EC_IABT_LOW:
 300                el0_ia(regs, esr);
 301                break;
 302        case ESR_ELx_EC_FP_ASIMD:
 303                el0_fpsimd_acc(regs, esr);
 304                break;
 305        case ESR_ELx_EC_FP_EXC32:
 306                el0_fpsimd_exc(regs, esr);
 307                break;
 308        case ESR_ELx_EC_PC_ALIGN:
 309                el0_pc(regs, esr);
 310                break;
 311        case ESR_ELx_EC_UNKNOWN:
 312        case ESR_ELx_EC_CP14_MR:
 313        case ESR_ELx_EC_CP14_LS:
 314        case ESR_ELx_EC_CP14_64:
 315                el0_undef(regs);
 316                break;
 317        case ESR_ELx_EC_CP15_32:
 318        case ESR_ELx_EC_CP15_64:
 319                el0_cp15(regs, esr);
 320                break;
 321        case ESR_ELx_EC_BREAKPT_LOW:
 322        case ESR_ELx_EC_SOFTSTP_LOW:
 323        case ESR_ELx_EC_WATCHPT_LOW:
 324        case ESR_ELx_EC_BKPT32:
 325                el0_dbg(regs, esr);
 326                break;
 327        default:
 328                el0_inv(regs, esr);
 329        }
 330}
 331NOKPROBE_SYMBOL(el0_sync_compat_handler);
 332#endif /* CONFIG_COMPAT */
 333