LXR linux/arch/powerpc/kernel/mce

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Machine check exception handling CPU-side for power7 and power8
   4 *
   5 * Copyright 2013 IBM Corporation
   6 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
   7 */
   8
   9#undef DEBUG
  10#define pr_fmt(fmt) "mce_power: " fmt
  11
  12#include <linux/types.h>
  13#include <linux/ptrace.h>
  14#include <linux/extable.h>
  15#include <linux/pgtable.h>
  16#include <asm/mmu.h>
  17#include <asm/mce.h>
  18#include <asm/machdep.h>
  19#include <asm/pte-walk.h>
  20#include <asm/sstep.h>
  21#include <asm/exception-64s.h>
  22#include <asm/extable.h>
  23#include <asm/inst.h>
  24
  25/*
  26 * Convert an address related to an mm to a PFN. NOTE: we are in real
  27 * mode, we could potentially race with page table updates.
  28 */
  29unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr)
  30{
  31        pte_t *ptep, pte;
  32        unsigned int shift;
  33        unsigned long pfn, flags;
  34        struct mm_struct *mm;
  35
  36        if (user_mode(regs))
  37                mm = current->mm;
  38        else
  39                mm = &init_mm;
  40
  41        local_irq_save(flags);
  42        ptep = __find_linux_pte(mm->pgd, addr, NULL, &shift);
  43        if (!ptep) {
  44                pfn = ULONG_MAX;
  45                goto out;
  46        }
  47        pte = READ_ONCE(*ptep);
  48
  49        if (!pte_present(pte) || pte_special(pte)) {
  50                pfn = ULONG_MAX;
  51                goto out;
  52        }
  53
  54        if (shift <= PAGE_SHIFT)
  55                pfn = pte_pfn(pte);
  56        else {
  57                unsigned long rpnmask = (1ul << shift) - PAGE_SIZE;
  58                pfn = pte_pfn(__pte(pte_val(pte) | (addr & rpnmask)));
  59        }
  60out:
  61        local_irq_restore(flags);
  62        return pfn;
  63}
  64
  65/* flush SLBs and reload */
  66#ifdef CONFIG_PPC_BOOK3S_64
  67void flush_and_reload_slb(void)
  68{
  69        /* Invalidate all SLBs */
  70        slb_flush_all_realmode();
  71
  72#ifdef CONFIG_KVM_BOOK3S_HANDLER
  73        /*
  74         * If machine check is hit when in guest or in transition, we will
  75         * only flush the SLBs and continue.
  76         */
  77        if (get_paca()->kvm_hstate.in_guest)
  78                return;
  79#endif
  80        if (early_radix_enabled())
  81                return;
  82
  83        /*
  84         * This probably shouldn't happen, but it may be possible it's
  85         * called in early boot before SLB shadows are allocated.
  86         */
  87        if (!get_slb_shadow())
  88                return;
  89
  90        slb_restore_bolted_realmode();
  91}
  92#endif
  93
  94static void flush_erat(void)
  95{
  96#ifdef CONFIG_PPC_BOOK3S_64
  97        if (!early_cpu_has_feature(CPU_FTR_ARCH_300)) {
  98                flush_and_reload_slb();
  99                return;
 100        }
 101#endif
 102        asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
 103}
 104
 105#define MCE_FLUSH_SLB 1
 106#define MCE_FLUSH_TLB 2
 107#define MCE_FLUSH_ERAT 3
 108
 109static int mce_flush(int what)
 110{
 111#ifdef CONFIG_PPC_BOOK3S_64
 112        if (what == MCE_FLUSH_SLB) {
 113                flush_and_reload_slb();
 114                return 1;
 115        }
 116#endif
 117        if (what == MCE_FLUSH_ERAT) {
 118                flush_erat();
 119                return 1;
 120        }
 121        if (what == MCE_FLUSH_TLB) {
 122                tlbiel_all();
 123                return 1;
 124        }
 125
 126        return 0;
 127}
 128
 129#define SRR1_MC_LOADSTORE(srr1) ((srr1) & PPC_BIT(42))
 130
 131struct mce_ierror_table {
 132        unsigned long srr1_mask;
 133        unsigned long srr1_value;
 134        bool nip_valid; /* nip is a valid indicator of faulting address */
 135        unsigned int error_type;
 136        unsigned int error_subtype;
 137        unsigned int error_class;
 138        unsigned int initiator;
 139        unsigned int severity;
 140        bool sync_error;
 141};
 142
 143static const struct mce_ierror_table mce_p7_ierror_table[] = {
 144{ 0x00000000001c0000, 0x0000000000040000, true,
 145  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
 146  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 147{ 0x00000000001c0000, 0x0000000000080000, true,
 148  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 149  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 150{ 0x00000000001c0000, 0x00000000000c0000, true,
 151  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 152  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 153{ 0x00000000001c0000, 0x0000000000100000, true,
 154  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_INDETERMINATE, /* BOTH */
 155  MCE_ECLASS_SOFT_INDETERMINATE,
 156  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 157{ 0x00000000001c0000, 0x0000000000140000, true,
 158  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 159  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 160{ 0x00000000001c0000, 0x0000000000180000, true,
 161  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_HARDWARE,
 162  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 163{ 0x00000000001c0000, 0x00000000001c0000, true,
 164  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
 165  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 166{ 0, 0, 0, 0, 0, 0, 0 } };
 167
 168static const struct mce_ierror_table mce_p8_ierror_table[] = {
 169{ 0x00000000081c0000, 0x0000000000040000, true,
 170  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
 171  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 172{ 0x00000000081c0000, 0x0000000000080000, true,
 173  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 174  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 175{ 0x00000000081c0000, 0x00000000000c0000, true,
 176  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 177  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 178{ 0x00000000081c0000, 0x0000000000100000, true,
 179  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 180  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 181{ 0x00000000081c0000, 0x0000000000140000, true,
 182  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 183  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 184{ 0x00000000081c0000, 0x0000000000180000, true,
 185  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH,
 186  MCE_ECLASS_HARDWARE,
 187  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 188{ 0x00000000081c0000, 0x00000000001c0000, true,
 189  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
 190  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 191{ 0x00000000081c0000, 0x0000000008000000, true,
 192  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_IFETCH_TIMEOUT, MCE_ECLASS_HARDWARE,
 193  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 194{ 0x00000000081c0000, 0x0000000008040000, true,
 195  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_PAGE_TABLE_WALK_IFETCH_TIMEOUT,
 196  MCE_ECLASS_HARDWARE,
 197  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 198{ 0, 0, 0, 0, 0, 0, 0 } };
 199
 200static const struct mce_ierror_table mce_p9_ierror_table[] = {
 201{ 0x00000000081c0000, 0x0000000000040000, true,
 202  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
 203  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 204{ 0x00000000081c0000, 0x0000000000080000, true,
 205  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 206  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 207{ 0x00000000081c0000, 0x00000000000c0000, true,
 208  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 209  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 210{ 0x00000000081c0000, 0x0000000000100000, true,
 211  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 212  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 213{ 0x00000000081c0000, 0x0000000000140000, true,
 214  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 215  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 216{ 0x00000000081c0000, 0x0000000000180000, true,
 217  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_HARDWARE,
 218  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 219{ 0x00000000081c0000, 0x00000000001c0000, true,
 220  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_IFETCH_FOREIGN, MCE_ECLASS_SOFTWARE,
 221  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 222{ 0x00000000081c0000, 0x0000000008000000, true,
 223  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_IFETCH_TIMEOUT, MCE_ECLASS_HARDWARE,
 224  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 225{ 0x00000000081c0000, 0x0000000008040000, true,
 226  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_PAGE_TABLE_WALK_IFETCH_TIMEOUT,
 227  MCE_ECLASS_HARDWARE,
 228  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 229{ 0x00000000081c0000, 0x00000000080c0000, true,
 230  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_IFETCH, MCE_ECLASS_SOFTWARE,
 231  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 232{ 0x00000000081c0000, 0x0000000008100000, true,
 233  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_SOFTWARE,
 234  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 235{ 0x00000000081c0000, 0x0000000008140000, false,
 236  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_STORE, MCE_ECLASS_HARDWARE,
 237  MCE_INITIATOR_CPU,  MCE_SEV_FATAL, false }, /* ASYNC is fatal */
 238{ 0x00000000081c0000, 0x0000000008180000, false,
 239  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_STORE_TIMEOUT,
 240  MCE_INITIATOR_CPU,  MCE_SEV_FATAL, false }, /* ASYNC is fatal */
 241{ 0x00000000081c0000, 0x00000000081c0000, true, MCE_ECLASS_HARDWARE,
 242  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH_FOREIGN,
 243  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 244{ 0, 0, 0, 0, 0, 0, 0 } };
 245
 246static const struct mce_ierror_table mce_p10_ierror_table[] = {
 247{ 0x00000000081c0000, 0x0000000000040000, true,
 248  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH, MCE_ECLASS_HARDWARE,
 249  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 250{ 0x00000000081c0000, 0x0000000000080000, true,
 251  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 252  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 253{ 0x00000000081c0000, 0x00000000000c0000, true,
 254  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 255  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 256{ 0x00000000081c0000, 0x0000000000100000, true,
 257  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 258  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 259{ 0x00000000081c0000, 0x0000000000140000, true,
 260  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 261  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 262{ 0x00000000081c0000, 0x0000000000180000, true,
 263  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_HARDWARE,
 264  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 265{ 0x00000000081c0000, 0x00000000001c0000, true,
 266  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_IFETCH_FOREIGN, MCE_ECLASS_SOFTWARE,
 267  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 268{ 0x00000000081c0000, 0x0000000008080000, true,
 269  MCE_ERROR_TYPE_USER,MCE_USER_ERROR_SCV, MCE_ECLASS_SOFTWARE,
 270  MCE_INITIATOR_CPU,  MCE_SEV_WARNING, true },
 271{ 0x00000000081c0000, 0x00000000080c0000, true,
 272  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_IFETCH, MCE_ECLASS_SOFTWARE,
 273  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 274{ 0x00000000081c0000, 0x0000000008100000, true,
 275  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH, MCE_ECLASS_SOFTWARE,
 276  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 277{ 0x00000000081c0000, 0x0000000008140000, false,
 278  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_STORE, MCE_ECLASS_HARDWARE,
 279  MCE_INITIATOR_CPU,  MCE_SEV_FATAL, false }, /* ASYNC is fatal */
 280{ 0x00000000081c0000, 0x00000000081c0000, true, MCE_ECLASS_HARDWARE,
 281  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH_FOREIGN,
 282  MCE_INITIATOR_CPU,  MCE_SEV_SEVERE, true },
 283{ 0, 0, 0, 0, 0, 0, 0 } };
 284
 285struct mce_derror_table {
 286        unsigned long dsisr_value;
 287        bool dar_valid; /* dar is a valid indicator of faulting address */
 288        unsigned int error_type;
 289        unsigned int error_subtype;
 290        unsigned int error_class;
 291        unsigned int initiator;
 292        unsigned int severity;
 293        bool sync_error;
 294};
 295
 296static const struct mce_derror_table mce_p7_derror_table[] = {
 297{ 0x00008000, false,
 298  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE, MCE_ECLASS_HARDWARE,
 299  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 300{ 0x00004000, true,
 301  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 302  MCE_ECLASS_HARDWARE,
 303  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 304{ 0x00000800, true,
 305  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 306  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 307{ 0x00000400, true,
 308  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 309  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 310{ 0x00000080, true,
 311  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 312  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 313{ 0x00000100, true,
 314  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 315  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 316{ 0x00000040, true,
 317  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_INDETERMINATE, /* BOTH */
 318  MCE_ECLASS_HARD_INDETERMINATE,
 319  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 320{ 0, false, 0, 0, 0, 0, 0 } };
 321
 322static const struct mce_derror_table mce_p8_derror_table[] = {
 323{ 0x00008000, false,
 324  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE, MCE_ECLASS_HARDWARE,
 325  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 326{ 0x00004000, true,
 327  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 328  MCE_ECLASS_HARDWARE,
 329  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 330{ 0x00002000, true,
 331  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_LOAD_TIMEOUT, MCE_ECLASS_HARDWARE,
 332  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 333{ 0x00001000, true,
 334  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_PAGE_TABLE_WALK_LOAD_STORE_TIMEOUT,
 335  MCE_ECLASS_HARDWARE,
 336  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 337{ 0x00000800, true,
 338  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 339  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 340{ 0x00000400, true,
 341  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 342  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 343{ 0x00000200, true,
 344  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, /* SECONDARY ERAT */
 345  MCE_ECLASS_SOFT_INDETERMINATE,
 346  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 347{ 0x00000080, true,
 348  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT,  /* Before PARITY */
 349  MCE_ECLASS_SOFT_INDETERMINATE,
 350  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 351{ 0x00000100, true,
 352  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 353  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 354{ 0, false, 0, 0, 0, 0, 0 } };
 355
 356static const struct mce_derror_table mce_p9_derror_table[] = {
 357{ 0x00008000, false,
 358  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE, MCE_ECLASS_HARDWARE,
 359  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 360{ 0x00004000, true,
 361  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 362  MCE_ECLASS_HARDWARE,
 363  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 364{ 0x00002000, true,
 365  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_LOAD_TIMEOUT, MCE_ECLASS_HARDWARE,
 366  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 367{ 0x00001000, true,
 368  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_PAGE_TABLE_WALK_LOAD_STORE_TIMEOUT,
 369  MCE_ECLASS_HARDWARE,
 370  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 371{ 0x00000800, true,
 372  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 373  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 374{ 0x00000400, true,
 375  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 376  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 377{ 0x00000200, false,
 378  MCE_ERROR_TYPE_USER, MCE_USER_ERROR_TLBIE, MCE_ECLASS_SOFTWARE,
 379  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 380{ 0x00000080, true,
 381  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT,  /* Before PARITY */
 382  MCE_ECLASS_SOFT_INDETERMINATE,
 383  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 384{ 0x00000100, true,
 385  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 386  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 387{ 0x00000040, true,
 388  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_LOAD, MCE_ECLASS_HARDWARE,
 389  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 390{ 0x00000020, false,
 391  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 392  MCE_ECLASS_HARDWARE,
 393  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 394{ 0x00000010, false,
 395  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE_FOREIGN,
 396  MCE_ECLASS_HARDWARE,
 397  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 398{ 0x00000008, false,
 399  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_LOAD_STORE_FOREIGN, MCE_ECLASS_HARDWARE,
 400  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 401{ 0, false, 0, 0, 0, 0, 0 } };
 402
 403static const struct mce_derror_table mce_p10_derror_table[] = {
 404{ 0x00008000, false,
 405  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE, MCE_ECLASS_HARDWARE,
 406  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 407{ 0x00004000, true,
 408  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 409  MCE_ECLASS_HARDWARE,
 410  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 411{ 0x00000800, true,
 412  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 413  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 414{ 0x00000400, true,
 415  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT, MCE_ECLASS_SOFT_INDETERMINATE,
 416  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 417{ 0x00000200, false,
 418  MCE_ERROR_TYPE_USER, MCE_USER_ERROR_TLBIE, MCE_ECLASS_SOFTWARE,
 419  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 420{ 0x00000080, true,
 421  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT,  /* Before PARITY */
 422  MCE_ECLASS_SOFT_INDETERMINATE,
 423  MCE_INITIATOR_CPU,   MCE_SEV_WARNING, true },
 424{ 0x00000100, true,
 425  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY, MCE_ECLASS_HARD_INDETERMINATE,
 426  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 427{ 0x00000040, true,
 428  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_LOAD, MCE_ECLASS_HARDWARE,
 429  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 430{ 0x00000020, false,
 431  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 432  MCE_ECLASS_HARDWARE,
 433  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 434{ 0x00000010, false,
 435  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE_FOREIGN,
 436  MCE_ECLASS_HARDWARE,
 437  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 438{ 0x00000008, false,
 439  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_LOAD_STORE_FOREIGN, MCE_ECLASS_HARDWARE,
 440  MCE_INITIATOR_CPU,   MCE_SEV_SEVERE, true },
 441{ 0, false, 0, 0, 0, 0, 0 } };
 442
 443static int mce_find_instr_ea_and_phys(struct pt_regs *regs, uint64_t *addr,
 444                                        uint64_t *phys_addr)
 445{
 446        /*
 447         * Carefully look at the NIP to determine
 448         * the instruction to analyse. Reading the NIP
 449         * in real-mode is tricky and can lead to recursive
 450         * faults
 451         */
 452        struct ppc_inst instr;
 453        unsigned long pfn, instr_addr;
 454        struct instruction_op op;
 455        struct pt_regs tmp = *regs;
 456
 457        pfn = addr_to_pfn(regs, regs->nip);
 458        if (pfn != ULONG_MAX) {
 459                instr_addr = (pfn << PAGE_SHIFT) + (regs->nip & ~PAGE_MASK);
 460                instr = ppc_inst_read((struct ppc_inst *)instr_addr);
 461                if (!analyse_instr(&op, &tmp, instr)) {
 462                        pfn = addr_to_pfn(regs, op.ea);
 463                        *addr = op.ea;
 464                        *phys_addr = (pfn << PAGE_SHIFT);
 465                        return 0;
 466                }
 467                /*
 468                 * analyse_instr() might fail if the instruction
 469                 * is not a load/store, although this is unexpected
 470                 * for load/store errors or if we got the NIP
 471                 * wrong
 472                 */
 473        }
 474        *addr = 0;
 475        return -1;
 476}
 477
 478static int mce_handle_ierror(struct pt_regs *regs,
 479                const struct mce_ierror_table table[],
 480                struct mce_error_info *mce_err, uint64_t *addr,
 481                uint64_t *phys_addr)
 482{
 483        uint64_t srr1 = regs->msr;
 484        int handled = 0;
 485        int i;
 486
 487        *addr = 0;
 488
 489        for (i = 0; table[i].srr1_mask; i++) {
 490                if ((srr1 & table[i].srr1_mask) != table[i].srr1_value)
 491                        continue;
 492
 493                /* attempt to correct the error */
 494                switch (table[i].error_type) {
 495                case MCE_ERROR_TYPE_SLB:
 496                        if (local_paca->in_mce == 1)
 497                                slb_save_contents(local_paca->mce_faulty_slbs);
 498                        handled = mce_flush(MCE_FLUSH_SLB);
 499                        break;
 500                case MCE_ERROR_TYPE_ERAT:
 501                        handled = mce_flush(MCE_FLUSH_ERAT);
 502                        break;
 503                case MCE_ERROR_TYPE_TLB:
 504                        handled = mce_flush(MCE_FLUSH_TLB);
 505                        break;
 506                }
 507
 508                /* now fill in mce_error_info */
 509                mce_err->error_type = table[i].error_type;
 510                mce_err->error_class = table[i].error_class;
 511                switch (table[i].error_type) {
 512                case MCE_ERROR_TYPE_UE:
 513                        mce_err->u.ue_error_type = table[i].error_subtype;
 514                        break;
 515                case MCE_ERROR_TYPE_SLB:
 516                        mce_err->u.slb_error_type = table[i].error_subtype;
 517                        break;
 518                case MCE_ERROR_TYPE_ERAT:
 519                        mce_err->u.erat_error_type = table[i].error_subtype;
 520                        break;
 521                case MCE_ERROR_TYPE_TLB:
 522                        mce_err->u.tlb_error_type = table[i].error_subtype;
 523                        break;
 524                case MCE_ERROR_TYPE_USER:
 525                        mce_err->u.user_error_type = table[i].error_subtype;
 526                        break;
 527                case MCE_ERROR_TYPE_RA:
 528                        mce_err->u.ra_error_type = table[i].error_subtype;
 529                        break;
 530                case MCE_ERROR_TYPE_LINK:
 531                        mce_err->u.link_error_type = table[i].error_subtype;
 532                        break;
 533                }
 534                mce_err->sync_error = table[i].sync_error;
 535                mce_err->severity = table[i].severity;
 536                mce_err->initiator = table[i].initiator;
 537                if (table[i].nip_valid) {
 538                        *addr = regs->nip;
 539                        if (mce_err->sync_error &&
 540                                table[i].error_type == MCE_ERROR_TYPE_UE) {
 541                                unsigned long pfn;
 542
 543                                if (get_paca()->in_mce < MAX_MCE_DEPTH) {
 544                                        pfn = addr_to_pfn(regs, regs->nip);
 545                                        if (pfn != ULONG_MAX) {
 546                                                *phys_addr =
 547                                                        (pfn << PAGE_SHIFT);
 548                                        }
 549                                }
 550                        }
 551                }
 552                return handled;
 553        }
 554
 555        mce_err->error_type = MCE_ERROR_TYPE_UNKNOWN;
 556        mce_err->error_class = MCE_ECLASS_UNKNOWN;
 557        mce_err->severity = MCE_SEV_SEVERE;
 558        mce_err->initiator = MCE_INITIATOR_CPU;
 559        mce_err->sync_error = true;
 560
 561        return 0;
 562}
 563
 564static int mce_handle_derror(struct pt_regs *regs,
 565                const struct mce_derror_table table[],
 566                struct mce_error_info *mce_err, uint64_t *addr,
 567                uint64_t *phys_addr)
 568{
 569        uint64_t dsisr = regs->dsisr;
 570        int handled = 0;
 571        int found = 0;
 572        int i;
 573
 574        *addr = 0;
 575
 576        for (i = 0; table[i].dsisr_value; i++) {
 577                if (!(dsisr & table[i].dsisr_value))
 578                        continue;
 579
 580                /* attempt to correct the error */
 581                switch (table[i].error_type) {
 582                case MCE_ERROR_TYPE_SLB:
 583                        if (local_paca->in_mce == 1)
 584                                slb_save_contents(local_paca->mce_faulty_slbs);
 585                        if (mce_flush(MCE_FLUSH_SLB))
 586                                handled = 1;
 587                        break;
 588                case MCE_ERROR_TYPE_ERAT:
 589                        if (mce_flush(MCE_FLUSH_ERAT))
 590                                handled = 1;
 591                        break;
 592                case MCE_ERROR_TYPE_TLB:
 593                        if (mce_flush(MCE_FLUSH_TLB))
 594                                handled = 1;
 595                        break;
 596                }
 597
 598                /*
 599                 * Attempt to handle multiple conditions, but only return
 600                 * one. Ensure uncorrectable errors are first in the table
 601                 * to match.
 602                 */
 603                if (found)
 604                        continue;
 605
 606                /* now fill in mce_error_info */
 607                mce_err->error_type = table[i].error_type;
 608                mce_err->error_class = table[i].error_class;
 609                switch (table[i].error_type) {
 610                case MCE_ERROR_TYPE_UE:
 611                        mce_err->u.ue_error_type = table[i].error_subtype;
 612                        break;
 613                case MCE_ERROR_TYPE_SLB:
 614                        mce_err->u.slb_error_type = table[i].error_subtype;
 615                        break;
 616                case MCE_ERROR_TYPE_ERAT:
 617                        mce_err->u.erat_error_type = table[i].error_subtype;
 618                        break;
 619                case MCE_ERROR_TYPE_TLB:
 620                        mce_err->u.tlb_error_type = table[i].error_subtype;
 621                        break;
 622                case MCE_ERROR_TYPE_USER:
 623                        mce_err->u.user_error_type = table[i].error_subtype;
 624                        break;
 625                case MCE_ERROR_TYPE_RA:
 626                        mce_err->u.ra_error_type = table[i].error_subtype;
 627                        break;
 628                case MCE_ERROR_TYPE_LINK:
 629                        mce_err->u.link_error_type = table[i].error_subtype;
 630                        break;
 631                }
 632                mce_err->sync_error = table[i].sync_error;
 633                mce_err->severity = table[i].severity;
 634                mce_err->initiator = table[i].initiator;
 635                if (table[i].dar_valid)
 636                        *addr = regs->dar;
 637                else if (mce_err->sync_error &&
 638                                table[i].error_type == MCE_ERROR_TYPE_UE) {
 639                        /*
 640                         * We do a maximum of 4 nested MCE calls, see
 641                         * kernel/exception-64s.h
 642                         */
 643                        if (get_paca()->in_mce < MAX_MCE_DEPTH)
 644                                mce_find_instr_ea_and_phys(regs, addr,
 645                                                           phys_addr);
 646                }
 647                found = 1;
 648        }
 649
 650        if (found)
 651                return handled;
 652
 653        mce_err->error_type = MCE_ERROR_TYPE_UNKNOWN;
 654        mce_err->error_class = MCE_ECLASS_UNKNOWN;
 655        mce_err->severity = MCE_SEV_SEVERE;
 656        mce_err->initiator = MCE_INITIATOR_CPU;
 657        mce_err->sync_error = true;
 658
 659        return 0;
 660}
 661
 662static long mce_handle_ue_error(struct pt_regs *regs,
 663                                struct mce_error_info *mce_err)
 664{
 665        long handled = 0;
 666
 667        mce_common_process_ue(regs, mce_err);
 668        if (mce_err->ignore_event)
 669                return 1;
 670
 671        /*
 672         * On specific SCOM read via MMIO we may get a machine check
 673         * exception with SRR0 pointing inside opal. If that is the
 674         * case OPAL may have recovery address to re-read SCOM data in
 675         * different way and hence we can recover from this MC.
 676         */
 677
 678        if (ppc_md.mce_check_early_recovery) {
 679                if (ppc_md.mce_check_early_recovery(regs))
 680                        handled = 1;
 681        }
 682        return handled;
 683}
 684
 685static long mce_handle_error(struct pt_regs *regs,
 686                const struct mce_derror_table dtable[],
 687                const struct mce_ierror_table itable[])
 688{
 689        struct mce_error_info mce_err = { 0 };
 690        uint64_t addr, phys_addr = ULONG_MAX;
 691        uint64_t srr1 = regs->msr;
 692        long handled;
 693
 694        if (SRR1_MC_LOADSTORE(srr1))
 695                handled = mce_handle_derror(regs, dtable, &mce_err, &addr,
 696                                &phys_addr);
 697        else
 698                handled = mce_handle_ierror(regs, itable, &mce_err, &addr,
 699                                &phys_addr);
 700
 701        if (!handled && mce_err.error_type == MCE_ERROR_TYPE_UE)
 702                handled = mce_handle_ue_error(regs, &mce_err);
 703
 704        save_mce_event(regs, handled, &mce_err, regs->nip, addr, phys_addr);
 705
 706        return handled;
 707}
 708
 709long __machine_check_early_realmode_p7(struct pt_regs *regs)
 710{
 711        /* P7 DD1 leaves top bits of DSISR undefined */
 712        regs->dsisr &= 0x0000ffff;
 713
 714        return mce_handle_error(regs, mce_p7_derror_table, mce_p7_ierror_table);
 715}
 716
 717long __machine_check_early_realmode_p8(struct pt_regs *regs)
 718{
 719        return mce_handle_error(regs, mce_p8_derror_table, mce_p8_ierror_table);
 720}
 721
 722long __machine_check_early_realmode_p9(struct pt_regs *regs)
 723{
 724        /*
 725         * On POWER9 DD2.1 and below, it's possible to get a machine check
 726         * caused by a paste instruction where only DSISR bit 25 is set. This
 727         * will result in the MCE handler seeing an unknown event and the kernel
 728         * crashing. An MCE that occurs like this is spurious, so we don't need
 729         * to do anything in terms of servicing it. If there is something that
 730         * needs to be serviced, the CPU will raise the MCE again with the
 731         * correct DSISR so that it can be serviced properly. So detect this
 732         * case and mark it as handled.
 733         */
 734        if (SRR1_MC_LOADSTORE(regs->msr) && regs->dsisr == 0x02000000)
 735                return 1;
 736
 737        return mce_handle_error(regs, mce_p9_derror_table, mce_p9_ierror_table);
 738}
 739
 740long __machine_check_early_realmode_p10(struct pt_regs *regs)
 741{
 742        return mce_handle_error(regs, mce_p10_derror_table, mce_p10_ierror_table);
 743}
 744