LXR linux/arch/powerpc/kernel/mce

   1/*
   2 * Machine check exception handling CPU-side for power7 and power8
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright 2013 IBM Corporation
  19 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
  20 */
  21
  22#undef DEBUG
  23#define pr_fmt(fmt) "mce_power: " fmt
  24
  25#include <linux/types.h>
  26#include <linux/ptrace.h>
  27#include <asm/mmu.h>
  28#include <asm/mce.h>
  29#include <asm/machdep.h>
  30#include <asm/pgtable.h>
  31#include <asm/pte-walk.h>
  32#include <asm/sstep.h>
  33#include <asm/exception-64s.h>
  34
  35/*
  36 * Convert an address related to an mm to a PFN. NOTE: we are in real
  37 * mode, we could potentially race with page table updates.
  38 */
  39static unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr)
  40{
  41        pte_t *ptep;
  42        unsigned long flags;
  43        struct mm_struct *mm;
  44
  45        if (user_mode(regs))
  46                mm = current->mm;
  47        else
  48                mm = &init_mm;
  49
  50        local_irq_save(flags);
  51        if (mm == current->mm)
  52                ptep = find_current_mm_pte(mm->pgd, addr, NULL, NULL);
  53        else
  54                ptep = find_init_mm_pte(addr, NULL);
  55        local_irq_restore(flags);
  56        if (!ptep || pte_special(*ptep))
  57                return ULONG_MAX;
  58        return pte_pfn(*ptep);
  59}
  60
  61/* flush SLBs and reload */
  62#ifdef CONFIG_PPC_BOOK3S_64
  63static void flush_and_reload_slb(void)
  64{
  65        /* Invalidate all SLBs */
  66        slb_flush_all_realmode();
  67
  68#ifdef CONFIG_KVM_BOOK3S_HANDLER
  69        /*
  70         * If machine check is hit when in guest or in transition, we will
  71         * only flush the SLBs and continue.
  72         */
  73        if (get_paca()->kvm_hstate.in_guest)
  74                return;
  75#endif
  76        if (early_radix_enabled())
  77                return;
  78
  79        /*
  80         * This probably shouldn't happen, but it may be possible it's
  81         * called in early boot before SLB shadows are allocated.
  82         */
  83        if (!get_slb_shadow())
  84                return;
  85
  86        slb_restore_bolted_realmode();
  87}
  88#endif
  89
  90static void flush_erat(void)
  91{
  92        asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
  93}
  94
  95#define MCE_FLUSH_SLB 1
  96#define MCE_FLUSH_TLB 2
  97#define MCE_FLUSH_ERAT 3
  98
  99static int mce_flush(int what)
 100{
 101#ifdef CONFIG_PPC_BOOK3S_64
 102        if (what == MCE_FLUSH_SLB) {
 103                flush_and_reload_slb();
 104                return 1;
 105        }
 106#endif
 107        if (what == MCE_FLUSH_ERAT) {
 108                flush_erat();
 109                return 1;
 110        }
 111        if (what == MCE_FLUSH_TLB) {
 112                tlbiel_all();
 113                return 1;
 114        }
 115
 116        return 0;
 117}
 118
 119#define SRR1_MC_LOADSTORE(srr1) ((srr1) & PPC_BIT(42))
 120
 121struct mce_ierror_table {
 122        unsigned long srr1_mask;
 123        unsigned long srr1_value;
 124        bool nip_valid; /* nip is a valid indicator of faulting address */
 125        unsigned int error_type;
 126        unsigned int error_subtype;
 127        unsigned int initiator;
 128        unsigned int severity;
 129};
 130
 131static const struct mce_ierror_table mce_p7_ierror_table[] = {
 132{ 0x00000000001c0000, 0x0000000000040000, true,
 133  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH,
 134  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 135{ 0x00000000001c0000, 0x0000000000080000, true,
 136  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY,
 137  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 138{ 0x00000000001c0000, 0x00000000000c0000, true,
 139  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT,
 140  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 141{ 0x00000000001c0000, 0x0000000000100000, true,
 142  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_INDETERMINATE, /* BOTH */
 143  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 144{ 0x00000000001c0000, 0x0000000000140000, true,
 145  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT,
 146  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 147{ 0x00000000001c0000, 0x0000000000180000, true,
 148  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH,
 149  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 150{ 0x00000000001c0000, 0x00000000001c0000, true,
 151  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH,
 152  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 153{ 0, 0, 0, 0, 0, 0 } };
 154
 155static const struct mce_ierror_table mce_p8_ierror_table[] = {
 156{ 0x00000000081c0000, 0x0000000000040000, true,
 157  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH,
 158  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 159{ 0x00000000081c0000, 0x0000000000080000, true,
 160  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY,
 161  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 162{ 0x00000000081c0000, 0x00000000000c0000, true,
 163  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT,
 164  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 165{ 0x00000000081c0000, 0x0000000000100000, true,
 166  MCE_ERROR_TYPE_ERAT,MCE_ERAT_ERROR_MULTIHIT,
 167  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 168{ 0x00000000081c0000, 0x0000000000140000, true,
 169  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT,
 170  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 171{ 0x00000000081c0000, 0x0000000000180000, true,
 172  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH,
 173  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 174{ 0x00000000081c0000, 0x00000000001c0000, true,
 175  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH,
 176  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 177{ 0x00000000081c0000, 0x0000000008000000, true,
 178  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_IFETCH_TIMEOUT,
 179  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 180{ 0x00000000081c0000, 0x0000000008040000, true,
 181  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_PAGE_TABLE_WALK_IFETCH_TIMEOUT,
 182  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 183{ 0, 0, 0, 0, 0, 0 } };
 184
 185static const struct mce_ierror_table mce_p9_ierror_table[] = {
 186{ 0x00000000081c0000, 0x0000000000040000, true,
 187  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_IFETCH,
 188  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 189{ 0x00000000081c0000, 0x0000000000080000, true,
 190  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY,
 191  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 192{ 0x00000000081c0000, 0x00000000000c0000, true,
 193  MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT,
 194  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 195{ 0x00000000081c0000, 0x0000000000100000, true,
 196  MCE_ERROR_TYPE_ERAT,MCE_ERAT_ERROR_MULTIHIT,
 197  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 198{ 0x00000000081c0000, 0x0000000000140000, true,
 199  MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT,
 200  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 201{ 0x00000000081c0000, 0x0000000000180000, true,
 202  MCE_ERROR_TYPE_UE,  MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH,
 203  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 204{ 0x00000000081c0000, 0x00000000001c0000, true,
 205  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_IFETCH_FOREIGN,
 206  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 207{ 0x00000000081c0000, 0x0000000008000000, true,
 208  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_IFETCH_TIMEOUT,
 209  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 210{ 0x00000000081c0000, 0x0000000008040000, true,
 211  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_PAGE_TABLE_WALK_IFETCH_TIMEOUT,
 212  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 213{ 0x00000000081c0000, 0x00000000080c0000, true,
 214  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_IFETCH,
 215  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 216{ 0x00000000081c0000, 0x0000000008100000, true,
 217  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH,
 218  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 219{ 0x00000000081c0000, 0x0000000008140000, false,
 220  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_STORE,
 221  MCE_INITIATOR_CPU,  MCE_SEV_FATAL, }, /* ASYNC is fatal */
 222{ 0x00000000081c0000, 0x0000000008180000, false,
 223  MCE_ERROR_TYPE_LINK,MCE_LINK_ERROR_STORE_TIMEOUT,
 224  MCE_INITIATOR_CPU,  MCE_SEV_FATAL, }, /* ASYNC is fatal */
 225{ 0x00000000081c0000, 0x00000000081c0000, true,
 226  MCE_ERROR_TYPE_RA,  MCE_RA_ERROR_PAGE_TABLE_WALK_IFETCH_FOREIGN,
 227  MCE_INITIATOR_CPU,  MCE_SEV_ERROR_SYNC, },
 228{ 0, 0, 0, 0, 0, 0 } };
 229
 230struct mce_derror_table {
 231        unsigned long dsisr_value;
 232        bool dar_valid; /* dar is a valid indicator of faulting address */
 233        unsigned int error_type;
 234        unsigned int error_subtype;
 235        unsigned int initiator;
 236        unsigned int severity;
 237};
 238
 239static const struct mce_derror_table mce_p7_derror_table[] = {
 240{ 0x00008000, false,
 241  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE,
 242  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 243{ 0x00004000, true,
 244  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 245  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 246{ 0x00000800, true,
 247  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT,
 248  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 249{ 0x00000400, true,
 250  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT,
 251  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 252{ 0x00000080, true,
 253  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT,  /* Before PARITY */
 254  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 255{ 0x00000100, true,
 256  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY,
 257  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 258{ 0x00000040, true,
 259  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_INDETERMINATE, /* BOTH */
 260  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 261{ 0, false, 0, 0, 0, 0 } };
 262
 263static const struct mce_derror_table mce_p8_derror_table[] = {
 264{ 0x00008000, false,
 265  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE,
 266  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 267{ 0x00004000, true,
 268  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 269  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 270{ 0x00002000, true,
 271  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_LOAD_TIMEOUT,
 272  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 273{ 0x00001000, true,
 274  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_PAGE_TABLE_WALK_LOAD_STORE_TIMEOUT,
 275  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 276{ 0x00000800, true,
 277  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT,
 278  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 279{ 0x00000400, true,
 280  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT,
 281  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 282{ 0x00000200, true,
 283  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, /* SECONDARY ERAT */
 284  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 285{ 0x00000080, true,
 286  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT,  /* Before PARITY */
 287  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 288{ 0x00000100, true,
 289  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY,
 290  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 291{ 0, false, 0, 0, 0, 0 } };
 292
 293static const struct mce_derror_table mce_p9_derror_table[] = {
 294{ 0x00008000, false,
 295  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_LOAD_STORE,
 296  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 297{ 0x00004000, true,
 298  MCE_ERROR_TYPE_UE,   MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 299  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 300{ 0x00002000, true,
 301  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_LOAD_TIMEOUT,
 302  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 303{ 0x00001000, true,
 304  MCE_ERROR_TYPE_LINK, MCE_LINK_ERROR_PAGE_TABLE_WALK_LOAD_STORE_TIMEOUT,
 305  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 306{ 0x00000800, true,
 307  MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT,
 308  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 309{ 0x00000400, true,
 310  MCE_ERROR_TYPE_TLB,  MCE_TLB_ERROR_MULTIHIT,
 311  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 312{ 0x00000200, false,
 313  MCE_ERROR_TYPE_USER, MCE_USER_ERROR_TLBIE,
 314  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 315{ 0x00000080, true,
 316  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_MULTIHIT,  /* Before PARITY */
 317  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 318{ 0x00000100, true,
 319  MCE_ERROR_TYPE_SLB,  MCE_SLB_ERROR_PARITY,
 320  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 321{ 0x00000040, true,
 322  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_LOAD,
 323  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 324{ 0x00000020, false,
 325  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE,
 326  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 327{ 0x00000010, false,
 328  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_PAGE_TABLE_WALK_LOAD_STORE_FOREIGN,
 329  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 330{ 0x00000008, false,
 331  MCE_ERROR_TYPE_RA,   MCE_RA_ERROR_LOAD_STORE_FOREIGN,
 332  MCE_INITIATOR_CPU,   MCE_SEV_ERROR_SYNC, },
 333{ 0, false, 0, 0, 0, 0 } };
 334
 335static int mce_find_instr_ea_and_pfn(struct pt_regs *regs, uint64_t *addr,
 336                                        uint64_t *phys_addr)
 337{
 338        /*
 339         * Carefully look at the NIP to determine
 340         * the instruction to analyse. Reading the NIP
 341         * in real-mode is tricky and can lead to recursive
 342         * faults
 343         */
 344        int instr;
 345        unsigned long pfn, instr_addr;
 346        struct instruction_op op;
 347        struct pt_regs tmp = *regs;
 348
 349        pfn = addr_to_pfn(regs, regs->nip);
 350        if (pfn != ULONG_MAX) {
 351                instr_addr = (pfn << PAGE_SHIFT) + (regs->nip & ~PAGE_MASK);
 352                instr = *(unsigned int *)(instr_addr);
 353                if (!analyse_instr(&op, &tmp, instr)) {
 354                        pfn = addr_to_pfn(regs, op.ea);
 355                        *addr = op.ea;
 356                        *phys_addr = (pfn << PAGE_SHIFT);
 357                        return 0;
 358                }
 359                /*
 360                 * analyse_instr() might fail if the instruction
 361                 * is not a load/store, although this is unexpected
 362                 * for load/store errors or if we got the NIP
 363                 * wrong
 364                 */
 365        }
 366        *addr = 0;
 367        return -1;
 368}
 369
 370static int mce_handle_ierror(struct pt_regs *regs,
 371                const struct mce_ierror_table table[],
 372                struct mce_error_info *mce_err, uint64_t *addr,
 373                uint64_t *phys_addr)
 374{
 375        uint64_t srr1 = regs->msr;
 376        int handled = 0;
 377        int i;
 378
 379        *addr = 0;
 380
 381        for (i = 0; table[i].srr1_mask; i++) {
 382                if ((srr1 & table[i].srr1_mask) != table[i].srr1_value)
 383                        continue;
 384
 385                /* attempt to correct the error */
 386                switch (table[i].error_type) {
 387                case MCE_ERROR_TYPE_SLB:
 388                        handled = mce_flush(MCE_FLUSH_SLB);
 389                        break;
 390                case MCE_ERROR_TYPE_ERAT:
 391                        handled = mce_flush(MCE_FLUSH_ERAT);
 392                        break;
 393                case MCE_ERROR_TYPE_TLB:
 394                        handled = mce_flush(MCE_FLUSH_TLB);
 395                        break;
 396                }
 397
 398                /* now fill in mce_error_info */
 399                mce_err->error_type = table[i].error_type;
 400                switch (table[i].error_type) {
 401                case MCE_ERROR_TYPE_UE:
 402                        mce_err->u.ue_error_type = table[i].error_subtype;
 403                        break;
 404                case MCE_ERROR_TYPE_SLB:
 405                        mce_err->u.slb_error_type = table[i].error_subtype;
 406                        break;
 407                case MCE_ERROR_TYPE_ERAT:
 408                        mce_err->u.erat_error_type = table[i].error_subtype;
 409                        break;
 410                case MCE_ERROR_TYPE_TLB:
 411                        mce_err->u.tlb_error_type = table[i].error_subtype;
 412                        break;
 413                case MCE_ERROR_TYPE_USER:
 414                        mce_err->u.user_error_type = table[i].error_subtype;
 415                        break;
 416                case MCE_ERROR_TYPE_RA:
 417                        mce_err->u.ra_error_type = table[i].error_subtype;
 418                        break;
 419                case MCE_ERROR_TYPE_LINK:
 420                        mce_err->u.link_error_type = table[i].error_subtype;
 421                        break;
 422                }
 423                mce_err->severity = table[i].severity;
 424                mce_err->initiator = table[i].initiator;
 425                if (table[i].nip_valid) {
 426                        *addr = regs->nip;
 427                        if (mce_err->severity == MCE_SEV_ERROR_SYNC &&
 428                                table[i].error_type == MCE_ERROR_TYPE_UE) {
 429                                unsigned long pfn;
 430
 431                                if (get_paca()->in_mce < MAX_MCE_DEPTH) {
 432                                        pfn = addr_to_pfn(regs, regs->nip);
 433                                        if (pfn != ULONG_MAX) {
 434                                                *phys_addr =
 435                                                        (pfn << PAGE_SHIFT);
 436                                        }
 437                                }
 438                        }
 439                }
 440                return handled;
 441        }
 442
 443        mce_err->error_type = MCE_ERROR_TYPE_UNKNOWN;
 444        mce_err->severity = MCE_SEV_ERROR_SYNC;
 445        mce_err->initiator = MCE_INITIATOR_CPU;
 446
 447        return 0;
 448}
 449
 450static int mce_handle_derror(struct pt_regs *regs,
 451                const struct mce_derror_table table[],
 452                struct mce_error_info *mce_err, uint64_t *addr,
 453                uint64_t *phys_addr)
 454{
 455        uint64_t dsisr = regs->dsisr;
 456        int handled = 0;
 457        int found = 0;
 458        int i;
 459
 460        *addr = 0;
 461
 462        for (i = 0; table[i].dsisr_value; i++) {
 463                if (!(dsisr & table[i].dsisr_value))
 464                        continue;
 465
 466                /* attempt to correct the error */
 467                switch (table[i].error_type) {
 468                case MCE_ERROR_TYPE_SLB:
 469                        if (mce_flush(MCE_FLUSH_SLB))
 470                                handled = 1;
 471                        break;
 472                case MCE_ERROR_TYPE_ERAT:
 473                        if (mce_flush(MCE_FLUSH_ERAT))
 474                                handled = 1;
 475                        break;
 476                case MCE_ERROR_TYPE_TLB:
 477                        if (mce_flush(MCE_FLUSH_TLB))
 478                                handled = 1;
 479                        break;
 480                }
 481
 482                /*
 483                 * Attempt to handle multiple conditions, but only return
 484                 * one. Ensure uncorrectable errors are first in the table
 485                 * to match.
 486                 */
 487                if (found)
 488                        continue;
 489
 490                /* now fill in mce_error_info */
 491                mce_err->error_type = table[i].error_type;
 492                switch (table[i].error_type) {
 493                case MCE_ERROR_TYPE_UE:
 494                        mce_err->u.ue_error_type = table[i].error_subtype;
 495                        break;
 496                case MCE_ERROR_TYPE_SLB:
 497                        mce_err->u.slb_error_type = table[i].error_subtype;
 498                        break;
 499                case MCE_ERROR_TYPE_ERAT:
 500                        mce_err->u.erat_error_type = table[i].error_subtype;
 501                        break;
 502                case MCE_ERROR_TYPE_TLB:
 503                        mce_err->u.tlb_error_type = table[i].error_subtype;
 504                        break;
 505                case MCE_ERROR_TYPE_USER:
 506                        mce_err->u.user_error_type = table[i].error_subtype;
 507                        break;
 508                case MCE_ERROR_TYPE_RA:
 509                        mce_err->u.ra_error_type = table[i].error_subtype;
 510                        break;
 511                case MCE_ERROR_TYPE_LINK:
 512                        mce_err->u.link_error_type = table[i].error_subtype;
 513                        break;
 514                }
 515                mce_err->severity = table[i].severity;
 516                mce_err->initiator = table[i].initiator;
 517                if (table[i].dar_valid)
 518                        *addr = regs->dar;
 519                else if (mce_err->severity == MCE_SEV_ERROR_SYNC &&
 520                                table[i].error_type == MCE_ERROR_TYPE_UE) {
 521                        /*
 522                         * We do a maximum of 4 nested MCE calls, see
 523                         * kernel/exception-64s.h
 524                         */
 525                        if (get_paca()->in_mce < MAX_MCE_DEPTH)
 526                                mce_find_instr_ea_and_pfn(regs, addr, phys_addr);
 527                }
 528                found = 1;
 529        }
 530
 531        if (found)
 532                return handled;
 533
 534        mce_err->error_type = MCE_ERROR_TYPE_UNKNOWN;
 535        mce_err->severity = MCE_SEV_ERROR_SYNC;
 536        mce_err->initiator = MCE_INITIATOR_CPU;
 537
 538        return 0;
 539}
 540
 541static long mce_handle_ue_error(struct pt_regs *regs)
 542{
 543        long handled = 0;
 544
 545        /*
 546         * On specific SCOM read via MMIO we may get a machine check
 547         * exception with SRR0 pointing inside opal. If that is the
 548         * case OPAL may have recovery address to re-read SCOM data in
 549         * different way and hence we can recover from this MC.
 550         */
 551
 552        if (ppc_md.mce_check_early_recovery) {
 553                if (ppc_md.mce_check_early_recovery(regs))
 554                        handled = 1;
 555        }
 556        return handled;
 557}
 558
 559static long mce_handle_error(struct pt_regs *regs,
 560                const struct mce_derror_table dtable[],
 561                const struct mce_ierror_table itable[])
 562{
 563        struct mce_error_info mce_err = { 0 };
 564        uint64_t addr, phys_addr = ULONG_MAX;
 565        uint64_t srr1 = regs->msr;
 566        long handled;
 567
 568        if (SRR1_MC_LOADSTORE(srr1))
 569                handled = mce_handle_derror(regs, dtable, &mce_err, &addr,
 570                                &phys_addr);
 571        else
 572                handled = mce_handle_ierror(regs, itable, &mce_err, &addr,
 573                                &phys_addr);
 574
 575        if (!handled && mce_err.error_type == MCE_ERROR_TYPE_UE)
 576                handled = mce_handle_ue_error(regs);
 577
 578        save_mce_event(regs, handled, &mce_err, regs->nip, addr, phys_addr);
 579
 580        return handled;
 581}
 582
 583long __machine_check_early_realmode_p7(struct pt_regs *regs)
 584{
 585        /* P7 DD1 leaves top bits of DSISR undefined */
 586        regs->dsisr &= 0x0000ffff;
 587
 588        return mce_handle_error(regs, mce_p7_derror_table, mce_p7_ierror_table);
 589}
 590
 591long __machine_check_early_realmode_p8(struct pt_regs *regs)
 592{
 593        return mce_handle_error(regs, mce_p8_derror_table, mce_p8_ierror_table);
 594}
 595
 596long __machine_check_early_realmode_p9(struct pt_regs *regs)
 597{
 598        /*
 599         * On POWER9 DD2.1 and below, it's possible to get a machine check
 600         * caused by a paste instruction where only DSISR bit 25 is set. This
 601         * will result in the MCE handler seeing an unknown event and the kernel
 602         * crashing. An MCE that occurs like this is spurious, so we don't need
 603         * to do anything in terms of servicing it. If there is something that
 604         * needs to be serviced, the CPU will raise the MCE again with the
 605         * correct DSISR so that it can be serviced properly. So detect this
 606         * case and mark it as handled.
 607         */
 608        if (SRR1_MC_LOADSTORE(regs->msr) && regs->dsisr == 0x02000000)
 609                return 1;
 610
 611        return mce_handle_error(regs, mce_p9_derror_table, mce_p9_ierror_table);
 612}
 613