linux/arch/sparc/kernel/traps_64.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/* arch/sparc64/kernel/traps.c
   3 *
   4 * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net)
   5 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
   6 */
   7
   8/*
   9 * I like traps on v9, :))))
  10 */
  11
  12#include <linux/extable.h>
  13#include <linux/sched/mm.h>
  14#include <linux/sched/debug.h>
  15#include <linux/linkage.h>
  16#include <linux/kernel.h>
  17#include <linux/signal.h>
  18#include <linux/smp.h>
  19#include <linux/mm.h>
  20#include <linux/init.h>
  21#include <linux/kallsyms.h>
  22#include <linux/kdebug.h>
  23#include <linux/ftrace.h>
  24#include <linux/reboot.h>
  25#include <linux/gfp.h>
  26#include <linux/context_tracking.h>
  27
  28#include <asm/smp.h>
  29#include <asm/delay.h>
  30#include <asm/ptrace.h>
  31#include <asm/oplib.h>
  32#include <asm/page.h>
  33#include <asm/unistd.h>
  34#include <linux/uaccess.h>
  35#include <asm/fpumacro.h>
  36#include <asm/lsu.h>
  37#include <asm/dcu.h>
  38#include <asm/estate.h>
  39#include <asm/chafsr.h>
  40#include <asm/sfafsr.h>
  41#include <asm/psrcompat.h>
  42#include <asm/processor.h>
  43#include <asm/timer.h>
  44#include <asm/head.h>
  45#include <asm/prom.h>
  46#include <asm/memctrl.h>
  47#include <asm/cacheflush.h>
  48#include <asm/setup.h>
  49
  50#include "entry.h"
  51#include "kernel.h"
  52#include "kstack.h"
  53
  54/* When an irrecoverable trap occurs at tl > 0, the trap entry
  55 * code logs the trap state registers at every level in the trap
  56 * stack.  It is found at (pt_regs + sizeof(pt_regs)) and the layout
  57 * is as follows:
  58 */
  59struct tl1_traplog {
  60        struct {
  61                unsigned long tstate;
  62                unsigned long tpc;
  63                unsigned long tnpc;
  64                unsigned long tt;
  65        } trapstack[4];
  66        unsigned long tl;
  67};
  68
  69static void dump_tl1_traplog(struct tl1_traplog *p)
  70{
  71        int i, limit;
  72
  73        printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
  74               "dumping track stack.\n", p->tl);
  75
  76        limit = (tlb_type == hypervisor) ? 2 : 4;
  77        for (i = 0; i < limit; i++) {
  78                printk(KERN_EMERG
  79                       "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
  80                       "TNPC[%016lx] TT[%lx]\n",
  81                       i + 1,
  82                       p->trapstack[i].tstate, p->trapstack[i].tpc,
  83                       p->trapstack[i].tnpc, p->trapstack[i].tt);
  84                printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
  85        }
  86}
  87
  88void bad_trap(struct pt_regs *regs, long lvl)
  89{
  90        char buffer[36];
  91
  92        if (notify_die(DIE_TRAP, "bad trap", regs,
  93                       0, lvl, SIGTRAP) == NOTIFY_STOP)
  94                return;
  95
  96        if (lvl < 0x100) {
  97                sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
  98                die_if_kernel(buffer, regs);
  99        }
 100
 101        lvl -= 0x100;
 102        if (regs->tstate & TSTATE_PRIV) {
 103                sprintf(buffer, "Kernel bad sw trap %lx", lvl);
 104                die_if_kernel(buffer, regs);
 105        }
 106        if (test_thread_flag(TIF_32BIT)) {
 107                regs->tpc &= 0xffffffff;
 108                regs->tnpc &= 0xffffffff;
 109        }
 110        force_sig_fault_trapno(SIGILL, ILL_ILLTRP,
 111                               (void __user *)regs->tpc, lvl);
 112}
 113
 114void bad_trap_tl1(struct pt_regs *regs, long lvl)
 115{
 116        char buffer[36];
 117        
 118        if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
 119                       0, lvl, SIGTRAP) == NOTIFY_STOP)
 120                return;
 121
 122        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
 123
 124        sprintf (buffer, "Bad trap %lx at tl>0", lvl);
 125        die_if_kernel (buffer, regs);
 126}
 127
 128#ifdef CONFIG_DEBUG_BUGVERBOSE
 129void do_BUG(const char *file, int line)
 130{
 131        bust_spinlocks(1);
 132        printk("kernel BUG at %s:%d!\n", file, line);
 133}
 134EXPORT_SYMBOL(do_BUG);
 135#endif
 136
 137static DEFINE_SPINLOCK(dimm_handler_lock);
 138static dimm_printer_t dimm_handler;
 139
 140static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen)
 141{
 142        unsigned long flags;
 143        int ret = -ENODEV;
 144
 145        spin_lock_irqsave(&dimm_handler_lock, flags);
 146        if (dimm_handler) {
 147                ret = dimm_handler(synd_code, paddr, buf, buflen);
 148        } else if (tlb_type == spitfire) {
 149                if (prom_getunumber(synd_code, paddr, buf, buflen) == -1)
 150                        ret = -EINVAL;
 151                else
 152                        ret = 0;
 153        } else
 154                ret = -ENODEV;
 155        spin_unlock_irqrestore(&dimm_handler_lock, flags);
 156
 157        return ret;
 158}
 159
 160int register_dimm_printer(dimm_printer_t func)
 161{
 162        unsigned long flags;
 163        int ret = 0;
 164
 165        spin_lock_irqsave(&dimm_handler_lock, flags);
 166        if (!dimm_handler)
 167                dimm_handler = func;
 168        else
 169                ret = -EEXIST;
 170        spin_unlock_irqrestore(&dimm_handler_lock, flags);
 171
 172        return ret;
 173}
 174EXPORT_SYMBOL_GPL(register_dimm_printer);
 175
 176void unregister_dimm_printer(dimm_printer_t func)
 177{
 178        unsigned long flags;
 179
 180        spin_lock_irqsave(&dimm_handler_lock, flags);
 181        if (dimm_handler == func)
 182                dimm_handler = NULL;
 183        spin_unlock_irqrestore(&dimm_handler_lock, flags);
 184}
 185EXPORT_SYMBOL_GPL(unregister_dimm_printer);
 186
 187void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
 188{
 189        enum ctx_state prev_state = exception_enter();
 190
 191        if (notify_die(DIE_TRAP, "instruction access exception", regs,
 192                       0, 0x8, SIGTRAP) == NOTIFY_STOP)
 193                goto out;
 194
 195        if (regs->tstate & TSTATE_PRIV) {
 196                printk("spitfire_insn_access_exception: SFSR[%016lx] "
 197                       "SFAR[%016lx], going.\n", sfsr, sfar);
 198                die_if_kernel("Iax", regs);
 199        }
 200        if (test_thread_flag(TIF_32BIT)) {
 201                regs->tpc &= 0xffffffff;
 202                regs->tnpc &= 0xffffffff;
 203        }
 204        force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)regs->tpc);
 205out:
 206        exception_exit(prev_state);
 207}
 208
 209void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
 210{
 211        if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
 212                       0, 0x8, SIGTRAP) == NOTIFY_STOP)
 213                return;
 214
 215        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
 216        spitfire_insn_access_exception(regs, sfsr, sfar);
 217}
 218
 219void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
 220{
 221        unsigned short type = (type_ctx >> 16);
 222        unsigned short ctx  = (type_ctx & 0xffff);
 223
 224        if (notify_die(DIE_TRAP, "instruction access exception", regs,
 225                       0, 0x8, SIGTRAP) == NOTIFY_STOP)
 226                return;
 227
 228        if (regs->tstate & TSTATE_PRIV) {
 229                printk("sun4v_insn_access_exception: ADDR[%016lx] "
 230                       "CTX[%04x] TYPE[%04x], going.\n",
 231                       addr, ctx, type);
 232                die_if_kernel("Iax", regs);
 233        }
 234
 235        if (test_thread_flag(TIF_32BIT)) {
 236                regs->tpc &= 0xffffffff;
 237                regs->tnpc &= 0xffffffff;
 238        }
 239        force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *) addr);
 240}
 241
 242void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
 243{
 244        if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
 245                       0, 0x8, SIGTRAP) == NOTIFY_STOP)
 246                return;
 247
 248        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
 249        sun4v_insn_access_exception(regs, addr, type_ctx);
 250}
 251
 252bool is_no_fault_exception(struct pt_regs *regs)
 253{
 254        unsigned char asi;
 255        u32 insn;
 256
 257        if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT)
 258                return false;
 259
 260        /*
 261         * Must do a little instruction decoding here in order to
 262         * decide on a course of action. The bits of interest are:
 263         *  insn[31:30] = op, where 3 indicates the load/store group
 264         *  insn[24:19] = op3, which identifies individual opcodes
 265         *  insn[13] indicates an immediate offset
 266         *  op3[4]=1 identifies alternate space instructions
 267         *  op3[5:4]=3 identifies floating point instructions
 268         *  op3[2]=1 identifies stores
 269         * See "Opcode Maps" in the appendix of any Sparc V9
 270         * architecture spec for full details.
 271         */
 272        if ((insn & 0xc0800000) == 0xc0800000) {    /* op=3, op3[4]=1   */
 273                if (insn & 0x2000)                  /* immediate offset */
 274                        asi = (regs->tstate >> 24); /* saved %asi       */
 275                else
 276                        asi = (insn >> 5);          /* immediate asi    */
 277                if ((asi & 0xf6) == ASI_PNF) {
 278                        if (insn & 0x200000)        /* op3[2], stores   */
 279                                return false;
 280                        if (insn & 0x1000000)       /* op3[5:4]=3 (fp)  */
 281                                handle_ldf_stq(insn, regs);
 282                        else
 283                                handle_ld_nf(insn, regs);
 284                        return true;
 285                }
 286        }
 287        return false;
 288}
 289
 290void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
 291{
 292        enum ctx_state prev_state = exception_enter();
 293
 294        if (notify_die(DIE_TRAP, "data access exception", regs,
 295                       0, 0x30, SIGTRAP) == NOTIFY_STOP)
 296                goto out;
 297
 298        if (regs->tstate & TSTATE_PRIV) {
 299                /* Test if this comes from uaccess places. */
 300                const struct exception_table_entry *entry;
 301
 302                entry = search_exception_tables(regs->tpc);
 303                if (entry) {
 304                        /* Ouch, somebody is trying VM hole tricks on us... */
 305#ifdef DEBUG_EXCEPTIONS
 306                        printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
 307                        printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
 308                               regs->tpc, entry->fixup);
 309#endif
 310                        regs->tpc = entry->fixup;
 311                        regs->tnpc = regs->tpc + 4;
 312                        goto out;
 313                }
 314                /* Shit... */
 315                printk("spitfire_data_access_exception: SFSR[%016lx] "
 316                       "SFAR[%016lx], going.\n", sfsr, sfar);
 317                die_if_kernel("Dax", regs);
 318        }
 319
 320        if (is_no_fault_exception(regs))
 321                return;
 322
 323        force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)sfar);
 324out:
 325        exception_exit(prev_state);
 326}
 327
 328void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
 329{
 330        if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
 331                       0, 0x30, SIGTRAP) == NOTIFY_STOP)
 332                return;
 333
 334        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
 335        spitfire_data_access_exception(regs, sfsr, sfar);
 336}
 337
 338void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
 339{
 340        unsigned short type = (type_ctx >> 16);
 341        unsigned short ctx  = (type_ctx & 0xffff);
 342
 343        if (notify_die(DIE_TRAP, "data access exception", regs,
 344                       0, 0x8, SIGTRAP) == NOTIFY_STOP)
 345                return;
 346
 347        if (regs->tstate & TSTATE_PRIV) {
 348                /* Test if this comes from uaccess places. */
 349                const struct exception_table_entry *entry;
 350
 351                entry = search_exception_tables(regs->tpc);
 352                if (entry) {
 353                        /* Ouch, somebody is trying VM hole tricks on us... */
 354#ifdef DEBUG_EXCEPTIONS
 355                        printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
 356                        printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
 357                               regs->tpc, entry->fixup);
 358#endif
 359                        regs->tpc = entry->fixup;
 360                        regs->tnpc = regs->tpc + 4;
 361                        return;
 362                }
 363                printk("sun4v_data_access_exception: ADDR[%016lx] "
 364                       "CTX[%04x] TYPE[%04x], going.\n",
 365                       addr, ctx, type);
 366                die_if_kernel("Dax", regs);
 367        }
 368
 369        if (test_thread_flag(TIF_32BIT)) {
 370                regs->tpc &= 0xffffffff;
 371                regs->tnpc &= 0xffffffff;
 372        }
 373        if (is_no_fault_exception(regs))
 374                return;
 375
 376        /* MCD (Memory Corruption Detection) disabled trap (TT=0x19) in HV
 377         * is vectored thorugh data access exception trap with fault type
 378         * set to HV_FAULT_TYPE_MCD_DIS. Check for MCD disabled trap.
 379         * Accessing an address with invalid ASI for the address, for
 380         * example setting an ADI tag on an address with ASI_MCD_PRIMARY
 381         * when TTE.mcd is not set for the VA, is also vectored into
 382         * kerbel by HV as data access exception with fault type set to
 383         * HV_FAULT_TYPE_INV_ASI.
 384         */
 385        switch (type) {
 386        case HV_FAULT_TYPE_INV_ASI:
 387                force_sig_fault(SIGILL, ILL_ILLADR, (void __user *)addr);
 388                break;
 389        case HV_FAULT_TYPE_MCD_DIS:
 390                force_sig_fault(SIGSEGV, SEGV_ACCADI, (void __user *)addr);
 391                break;
 392        default:
 393                force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)addr);
 394                break;
 395        }
 396}
 397
 398void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
 399{
 400        if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
 401                       0, 0x8, SIGTRAP) == NOTIFY_STOP)
 402                return;
 403
 404        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
 405        sun4v_data_access_exception(regs, addr, type_ctx);
 406}
 407
 408#ifdef CONFIG_PCI
 409#include "pci_impl.h"
 410#endif
 411
 412/* When access exceptions happen, we must do this. */
 413static void spitfire_clean_and_reenable_l1_caches(void)
 414{
 415        unsigned long va;
 416
 417        if (tlb_type != spitfire)
 418                BUG();
 419
 420        /* Clean 'em. */
 421        for (va =  0; va < (PAGE_SIZE << 1); va += 32) {
 422                spitfire_put_icache_tag(va, 0x0);
 423                spitfire_put_dcache_tag(va, 0x0);
 424        }
 425
 426        /* Re-enable in LSU. */
 427        __asm__ __volatile__("flush %%g6\n\t"
 428                             "membar #Sync\n\t"
 429                             "stxa %0, [%%g0] %1\n\t"
 430                             "membar #Sync"
 431                             : /* no outputs */
 432                             : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
 433                                    LSU_CONTROL_IM | LSU_CONTROL_DM),
 434                             "i" (ASI_LSU_CONTROL)
 435                             : "memory");
 436}
 437
 438static void spitfire_enable_estate_errors(void)
 439{
 440        __asm__ __volatile__("stxa      %0, [%%g0] %1\n\t"
 441                             "membar    #Sync"
 442                             : /* no outputs */
 443                             : "r" (ESTATE_ERR_ALL),
 444                               "i" (ASI_ESTATE_ERROR_EN));
 445}
 446
 447static char ecc_syndrome_table[] = {
 448        0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
 449        0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
 450        0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
 451        0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
 452        0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
 453        0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
 454        0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
 455        0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
 456        0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
 457        0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
 458        0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
 459        0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
 460        0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
 461        0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
 462        0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
 463        0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
 464        0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
 465        0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
 466        0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
 467        0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
 468        0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
 469        0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
 470        0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
 471        0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
 472        0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
 473        0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
 474        0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
 475        0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
 476        0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
 477        0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
 478        0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
 479        0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
 480};
 481
 482static char *syndrome_unknown = "<Unknown>";
 483
 484static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
 485{
 486        unsigned short scode;
 487        char memmod_str[64], *p;
 488
 489        if (udbl & bit) {
 490                scode = ecc_syndrome_table[udbl & 0xff];
 491                if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
 492                        p = syndrome_unknown;
 493                else
 494                        p = memmod_str;
 495                printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
 496                       "Memory Module \"%s\"\n",
 497                       smp_processor_id(), scode, p);
 498        }
 499
 500        if (udbh & bit) {
 501                scode = ecc_syndrome_table[udbh & 0xff];
 502                if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
 503                        p = syndrome_unknown;
 504                else
 505                        p = memmod_str;
 506                printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
 507                       "Memory Module \"%s\"\n",
 508                       smp_processor_id(), scode, p);
 509        }
 510
 511}
 512
 513static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
 514{
 515
 516        printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
 517               "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
 518               smp_processor_id(), afsr, afar, udbl, udbh, tl1);
 519
 520        spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
 521
 522        /* We always log it, even if someone is listening for this
 523         * trap.
 524         */
 525        notify_die(DIE_TRAP, "Correctable ECC Error", regs,
 526                   0, TRAP_TYPE_CEE, SIGTRAP);
 527
 528        /* The Correctable ECC Error trap does not disable I/D caches.  So
 529         * we only have to restore the ESTATE Error Enable register.
 530         */
 531        spitfire_enable_estate_errors();
 532}
 533
 534static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
 535{
 536        printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
 537               "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
 538               smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
 539
 540        /* XXX add more human friendly logging of the error status
 541         * XXX as is implemented for cheetah
 542         */
 543
 544        spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
 545
 546        /* We always log it, even if someone is listening for this
 547         * trap.
 548         */
 549        notify_die(DIE_TRAP, "Uncorrectable Error", regs,
 550                   0, tt, SIGTRAP);
 551
 552        if (regs->tstate & TSTATE_PRIV) {
 553                if (tl1)
 554                        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
 555                die_if_kernel("UE", regs);
 556        }
 557
 558        /* XXX need more intelligent processing here, such as is implemented
 559         * XXX for cheetah errors, in fact if the E-cache still holds the
 560         * XXX line with bad parity this will loop
 561         */
 562
 563        spitfire_clean_and_reenable_l1_caches();
 564        spitfire_enable_estate_errors();
 565
 566        if (test_thread_flag(TIF_32BIT)) {
 567                regs->tpc &= 0xffffffff;
 568                regs->tnpc &= 0xffffffff;
 569        }
 570        force_sig_fault(SIGBUS, BUS_OBJERR, (void *)0);
 571}
 572
 573void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
 574{
 575        unsigned long afsr, tt, udbh, udbl;
 576        int tl1;
 577
 578        afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
 579        tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
 580        tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
 581        udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
 582        udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
 583
 584#ifdef CONFIG_PCI
 585        if (tt == TRAP_TYPE_DAE &&
 586            pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
 587                spitfire_clean_and_reenable_l1_caches();
 588                spitfire_enable_estate_errors();
 589
 590                pci_poke_faulted = 1;
 591                regs->tnpc = regs->tpc + 4;
 592                return;
 593        }
 594#endif
 595
 596        if (afsr & SFAFSR_UE)
 597                spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
 598
 599        if (tt == TRAP_TYPE_CEE) {
 600                /* Handle the case where we took a CEE trap, but ACK'd
 601                 * only the UE state in the UDB error registers.
 602                 */
 603                if (afsr & SFAFSR_UE) {
 604                        if (udbh & UDBE_CE) {
 605                                __asm__ __volatile__(
 606                                        "stxa   %0, [%1] %2\n\t"
 607                                        "membar #Sync"
 608                                        : /* no outputs */
 609                                        : "r" (udbh & UDBE_CE),
 610                                          "r" (0x0), "i" (ASI_UDB_ERROR_W));
 611                        }
 612                        if (udbl & UDBE_CE) {
 613                                __asm__ __volatile__(
 614                                        "stxa   %0, [%1] %2\n\t"
 615                                        "membar #Sync"
 616                                        : /* no outputs */
 617                                        : "r" (udbl & UDBE_CE),
 618                                          "r" (0x18), "i" (ASI_UDB_ERROR_W));
 619                        }
 620                }
 621
 622                spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
 623        }
 624}
 625
 626int cheetah_pcache_forced_on;
 627
 628void cheetah_enable_pcache(void)
 629{
 630        unsigned long dcr;
 631
 632        printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
 633               smp_processor_id());
 634
 635        __asm__ __volatile__("ldxa [%%g0] %1, %0"
 636                             : "=r" (dcr)
 637                             : "i" (ASI_DCU_CONTROL_REG));
 638        dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
 639        __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
 640                             "membar #Sync"
 641                             : /* no outputs */
 642                             : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
 643}
 644
 645/* Cheetah error trap handling. */
 646static unsigned long ecache_flush_physbase;
 647static unsigned long ecache_flush_linesize;
 648static unsigned long ecache_flush_size;
 649
 650/* This table is ordered in priority of errors and matches the
 651 * AFAR overwrite policy as well.
 652 */
 653
 654struct afsr_error_table {
 655        unsigned long mask;
 656        const char *name;
 657};
 658
 659static const char CHAFSR_PERR_msg[] =
 660        "System interface protocol error";
 661static const char CHAFSR_IERR_msg[] =
 662        "Internal processor error";
 663static const char CHAFSR_ISAP_msg[] =
 664        "System request parity error on incoming address";
 665static const char CHAFSR_UCU_msg[] =
 666        "Uncorrectable E-cache ECC error for ifetch/data";
 667static const char CHAFSR_UCC_msg[] =
 668        "SW Correctable E-cache ECC error for ifetch/data";
 669static const char CHAFSR_UE_msg[] =
 670        "Uncorrectable system bus data ECC error for read";
 671static const char CHAFSR_EDU_msg[] =
 672        "Uncorrectable E-cache ECC error for stmerge/blkld";
 673static const char CHAFSR_EMU_msg[] =
 674        "Uncorrectable system bus MTAG error";
 675static const char CHAFSR_WDU_msg[] =
 676        "Uncorrectable E-cache ECC error for writeback";
 677static const char CHAFSR_CPU_msg[] =
 678        "Uncorrectable ECC error for copyout";
 679static const char CHAFSR_CE_msg[] =
 680        "HW corrected system bus data ECC error for read";
 681static const char CHAFSR_EDC_msg[] =
 682        "HW corrected E-cache ECC error for stmerge/blkld";
 683static const char CHAFSR_EMC_msg[] =
 684        "HW corrected system bus MTAG ECC error";
 685static const char CHAFSR_WDC_msg[] =
 686        "HW corrected E-cache ECC error for writeback";
 687static const char CHAFSR_CPC_msg[] =
 688        "HW corrected ECC error for copyout";
 689static const char CHAFSR_TO_msg[] =
 690        "Unmapped error from system bus";
 691static const char CHAFSR_BERR_msg[] =
 692        "Bus error response from system bus";
 693static const char CHAFSR_IVC_msg[] =
 694        "HW corrected system bus data ECC error for ivec read";
 695static const char CHAFSR_IVU_msg[] =
 696        "Uncorrectable system bus data ECC error for ivec read";
 697static struct afsr_error_table __cheetah_error_table[] = {
 698        {       CHAFSR_PERR,    CHAFSR_PERR_msg         },
 699        {       CHAFSR_IERR,    CHAFSR_IERR_msg         },
 700        {       CHAFSR_ISAP,    CHAFSR_ISAP_msg         },
 701        {       CHAFSR_UCU,     CHAFSR_UCU_msg          },
 702        {       CHAFSR_UCC,     CHAFSR_UCC_msg          },
 703        {       CHAFSR_UE,      CHAFSR_UE_msg           },
 704        {       CHAFSR_EDU,     CHAFSR_EDU_msg          },
 705        {       CHAFSR_EMU,     CHAFSR_EMU_msg          },
 706        {       CHAFSR_WDU,     CHAFSR_WDU_msg          },
 707        {       CHAFSR_CPU,     CHAFSR_CPU_msg          },
 708        {       CHAFSR_CE,      CHAFSR_CE_msg           },
 709        {       CHAFSR_EDC,     CHAFSR_EDC_msg          },
 710        {       CHAFSR_EMC,     CHAFSR_EMC_msg          },
 711        {       CHAFSR_WDC,     CHAFSR_WDC_msg          },
 712        {       CHAFSR_CPC,     CHAFSR_CPC_msg          },
 713        {       CHAFSR_TO,      CHAFSR_TO_msg           },
 714        {       CHAFSR_BERR,    CHAFSR_BERR_msg         },
 715        /* These two do not update the AFAR. */
 716        {       CHAFSR_IVC,     CHAFSR_IVC_msg          },
 717        {       CHAFSR_IVU,     CHAFSR_IVU_msg          },
 718        {       0,              NULL                    },
 719};
 720static const char CHPAFSR_DTO_msg[] =
 721        "System bus unmapped error for prefetch/storequeue-read";
 722static const char CHPAFSR_DBERR_msg[] =
 723        "System bus error for prefetch/storequeue-read";
 724static const char CHPAFSR_THCE_msg[] =
 725        "Hardware corrected E-cache Tag ECC error";
 726static const char CHPAFSR_TSCE_msg[] =
 727        "SW handled correctable E-cache Tag ECC error";
 728static const char CHPAFSR_TUE_msg[] =
 729        "Uncorrectable E-cache Tag ECC error";
 730static const char CHPAFSR_DUE_msg[] =
 731        "System bus uncorrectable data ECC error due to prefetch/store-fill";
 732static struct afsr_error_table __cheetah_plus_error_table[] = {
 733        {       CHAFSR_PERR,    CHAFSR_PERR_msg         },
 734        {       CHAFSR_IERR,    CHAFSR_IERR_msg         },
 735        {       CHAFSR_ISAP,    CHAFSR_ISAP_msg         },
 736        {       CHAFSR_UCU,     CHAFSR_UCU_msg          },
 737        {       CHAFSR_UCC,     CHAFSR_UCC_msg          },
 738        {       CHAFSR_UE,      CHAFSR_UE_msg           },
 739        {       CHAFSR_EDU,     CHAFSR_EDU_msg          },
 740        {       CHAFSR_EMU,     CHAFSR_EMU_msg          },
 741        {       CHAFSR_WDU,     CHAFSR_WDU_msg          },
 742        {       CHAFSR_CPU,     CHAFSR_CPU_msg          },
 743        {       CHAFSR_CE,      CHAFSR_CE_msg           },
 744        {       CHAFSR_EDC,     CHAFSR_EDC_msg          },
 745        {       CHAFSR_EMC,     CHAFSR_EMC_msg          },
 746        {       CHAFSR_WDC,     CHAFSR_WDC_msg          },
 747        {       CHAFSR_CPC,     CHAFSR_CPC_msg          },
 748        {       CHAFSR_TO,      CHAFSR_TO_msg           },
 749        {       CHAFSR_BERR,    CHAFSR_BERR_msg         },
 750        {       CHPAFSR_DTO,    CHPAFSR_DTO_msg         },
 751        {       CHPAFSR_DBERR,  CHPAFSR_DBERR_msg       },
 752        {       CHPAFSR_THCE,   CHPAFSR_THCE_msg        },
 753        {       CHPAFSR_TSCE,   CHPAFSR_TSCE_msg        },
 754        {       CHPAFSR_TUE,    CHPAFSR_TUE_msg         },
 755        {       CHPAFSR_DUE,    CHPAFSR_DUE_msg         },
 756        /* These two do not update the AFAR. */
 757        {       CHAFSR_IVC,     CHAFSR_IVC_msg          },
 758        {       CHAFSR_IVU,     CHAFSR_IVU_msg          },
 759        {       0,              NULL                    },
 760};
 761static const char JPAFSR_JETO_msg[] =
 762        "System interface protocol error, hw timeout caused";
 763static const char JPAFSR_SCE_msg[] =
 764        "Parity error on system snoop results";
 765static const char JPAFSR_JEIC_msg[] =
 766        "System interface protocol error, illegal command detected";
 767static const char JPAFSR_JEIT_msg[] =
 768        "System interface protocol error, illegal ADTYPE detected";
 769static const char JPAFSR_OM_msg[] =
 770        "Out of range memory error has occurred";
 771static const char JPAFSR_ETP_msg[] =
 772        "Parity error on L2 cache tag SRAM";
 773static const char JPAFSR_UMS_msg[] =
 774        "Error due to unsupported store";
 775static const char JPAFSR_RUE_msg[] =
 776        "Uncorrectable ECC error from remote cache/memory";
 777static const char JPAFSR_RCE_msg[] =
 778        "Correctable ECC error from remote cache/memory";
 779static const char JPAFSR_BP_msg[] =
 780        "JBUS parity error on returned read data";
 781static const char JPAFSR_WBP_msg[] =
 782        "JBUS parity error on data for writeback or block store";
 783static const char JPAFSR_FRC_msg[] =
 784        "Foreign read to DRAM incurring correctable ECC error";
 785static const char JPAFSR_FRU_msg[] =
 786        "Foreign read to DRAM incurring uncorrectable ECC error";
 787static struct afsr_error_table __jalapeno_error_table[] = {
 788        {       JPAFSR_JETO,    JPAFSR_JETO_msg         },
 789        {       JPAFSR_SCE,     JPAFSR_SCE_msg          },
 790        {       JPAFSR_JEIC,    JPAFSR_JEIC_msg         },
 791        {       JPAFSR_JEIT,    JPAFSR_JEIT_msg         },
 792        {       CHAFSR_PERR,    CHAFSR_PERR_msg         },
 793        {       CHAFSR_IERR,    CHAFSR_IERR_msg         },
 794        {       CHAFSR_ISAP,    CHAFSR_ISAP_msg         },
 795        {       CHAFSR_UCU,     CHAFSR_UCU_msg          },
 796        {       CHAFSR_UCC,     CHAFSR_UCC_msg          },
 797        {       CHAFSR_UE,      CHAFSR_UE_msg           },
 798        {       CHAFSR_EDU,     CHAFSR_EDU_msg          },
 799        {       JPAFSR_OM,      JPAFSR_OM_msg           },
 800        {       CHAFSR_WDU,     CHAFSR_WDU_msg          },
 801        {       CHAFSR_CPU,     CHAFSR_CPU_msg          },
 802        {       CHAFSR_CE,      CHAFSR_CE_msg           },
 803        {       CHAFSR_EDC,     CHAFSR_EDC_msg          },
 804        {       JPAFSR_ETP,     JPAFSR_ETP_msg          },
 805        {       CHAFSR_WDC,     CHAFSR_WDC_msg          },
 806        {       CHAFSR_CPC,     CHAFSR_CPC_msg          },
 807        {       CHAFSR_TO,      CHAFSR_TO_msg           },
 808        {       CHAFSR_BERR,    CHAFSR_BERR_msg         },
 809        {       JPAFSR_UMS,     JPAFSR_UMS_msg          },
 810        {       JPAFSR_RUE,     JPAFSR_RUE_msg          },
 811        {       JPAFSR_RCE,     JPAFSR_RCE_msg          },
 812        {       JPAFSR_BP,      JPAFSR_BP_msg           },
 813        {       JPAFSR_WBP,     JPAFSR_WBP_msg          },
 814        {       JPAFSR_FRC,     JPAFSR_FRC_msg          },
 815        {       JPAFSR_FRU,     JPAFSR_FRU_msg          },
 816        /* These two do not update the AFAR. */
 817        {       CHAFSR_IVU,     CHAFSR_IVU_msg          },
 818        {       0,              NULL                    },
 819};
 820static struct afsr_error_table *cheetah_error_table;
 821static unsigned long cheetah_afsr_errors;
 822
 823struct cheetah_err_info *cheetah_error_log;
 824
 825static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
 826{
 827        struct cheetah_err_info *p;
 828        int cpu = smp_processor_id();
 829
 830        if (!cheetah_error_log)
 831                return NULL;
 832
 833        p = cheetah_error_log + (cpu * 2);
 834        if ((afsr & CHAFSR_TL1) != 0UL)
 835                p++;
 836
 837        return p;
 838}
 839
 840extern unsigned int tl0_icpe[], tl1_icpe[];
 841extern unsigned int tl0_dcpe[], tl1_dcpe[];
 842extern unsigned int tl0_fecc[], tl1_fecc[];
 843extern unsigned int tl0_cee[], tl1_cee[];
 844extern unsigned int tl0_iae[], tl1_iae[];
 845extern unsigned int tl0_dae[], tl1_dae[];
 846extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
 847extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
 848extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
 849extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
 850extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
 851
 852void __init cheetah_ecache_flush_init(void)
 853{
 854        unsigned long largest_size, smallest_linesize, order, ver;
 855        int i, sz;
 856
 857        /* Scan all cpu device tree nodes, note two values:
 858         * 1) largest E-cache size
 859         * 2) smallest E-cache line size
 860         */
 861        largest_size = 0UL;
 862        smallest_linesize = ~0UL;
 863
 864        for (i = 0; i < NR_CPUS; i++) {
 865                unsigned long val;
 866
 867                val = cpu_data(i).ecache_size;
 868                if (!val)
 869                        continue;
 870
 871                if (val > largest_size)
 872                        largest_size = val;
 873
 874                val = cpu_data(i).ecache_line_size;
 875                if (val < smallest_linesize)
 876                        smallest_linesize = val;
 877
 878        }
 879
 880        if (largest_size == 0UL || smallest_linesize == ~0UL) {
 881                prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
 882                            "parameters.\n");
 883                prom_halt();
 884        }
 885
 886        ecache_flush_size = (2 * largest_size);
 887        ecache_flush_linesize = smallest_linesize;
 888
 889        ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
 890
 891        if (ecache_flush_physbase == ~0UL) {
 892                prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
 893                            "contiguous physical memory.\n",
 894                            ecache_flush_size);
 895                prom_halt();
 896        }
 897
 898        /* Now allocate error trap reporting scoreboard. */
 899        sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
 900        for (order = 0; order < MAX_ORDER; order++) {
 901                if ((PAGE_SIZE << order) >= sz)
 902                        break;
 903        }
 904        cheetah_error_log = (struct cheetah_err_info *)
 905                __get_free_pages(GFP_KERNEL, order);
 906        if (!cheetah_error_log) {
 907                prom_printf("cheetah_ecache_flush_init: Failed to allocate "
 908                            "error logging scoreboard (%d bytes).\n", sz);
 909                prom_halt();
 910        }
 911        memset(cheetah_error_log, 0, PAGE_SIZE << order);
 912
 913        /* Mark all AFSRs as invalid so that the trap handler will
 914         * log new new information there.
 915         */
 916        for (i = 0; i < 2 * NR_CPUS; i++)
 917                cheetah_error_log[i].afsr = CHAFSR_INVALID;
 918
 919        __asm__ ("rdpr %%ver, %0" : "=r" (ver));
 920        if ((ver >> 32) == __JALAPENO_ID ||
 921            (ver >> 32) == __SERRANO_ID) {
 922                cheetah_error_table = &__jalapeno_error_table[0];
 923                cheetah_afsr_errors = JPAFSR_ERRORS;
 924        } else if ((ver >> 32) == 0x003e0015) {
 925                cheetah_error_table = &__cheetah_plus_error_table[0];
 926                cheetah_afsr_errors = CHPAFSR_ERRORS;
 927        } else {
 928                cheetah_error_table = &__cheetah_error_table[0];
 929                cheetah_afsr_errors = CHAFSR_ERRORS;
 930        }
 931
 932        /* Now patch trap tables. */
 933        memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
 934        memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
 935        memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
 936        memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
 937        memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
 938        memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
 939        memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
 940        memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
 941        if (tlb_type == cheetah_plus) {
 942                memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
 943                memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
 944                memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
 945                memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
 946        }
 947        flushi(PAGE_OFFSET);
 948}
 949
 950static void cheetah_flush_ecache(void)
 951{
 952        unsigned long flush_base = ecache_flush_physbase;
 953        unsigned long flush_linesize = ecache_flush_linesize;
 954        unsigned long flush_size = ecache_flush_size;
 955
 956        __asm__ __volatile__("1: subcc  %0, %4, %0\n\t"
 957                             "   bne,pt %%xcc, 1b\n\t"
 958                             "    ldxa  [%2 + %0] %3, %%g0\n\t"
 959                             : "=&r" (flush_size)
 960                             : "0" (flush_size), "r" (flush_base),
 961                               "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
 962}
 963
 964static void cheetah_flush_ecache_line(unsigned long physaddr)
 965{
 966        unsigned long alias;
 967
 968        physaddr &= ~(8UL - 1UL);
 969        physaddr = (ecache_flush_physbase +
 970                    (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
 971        alias = physaddr + (ecache_flush_size >> 1UL);
 972        __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
 973                             "ldxa [%1] %2, %%g0\n\t"
 974                             "membar #Sync"
 975                             : /* no outputs */
 976                             : "r" (physaddr), "r" (alias),
 977                               "i" (ASI_PHYS_USE_EC));
 978}
 979
 980/* Unfortunately, the diagnostic access to the I-cache tags we need to
 981 * use to clear the thing interferes with I-cache coherency transactions.
 982 *
 983 * So we must only flush the I-cache when it is disabled.
 984 */
 985static void __cheetah_flush_icache(void)
 986{
 987        unsigned int icache_size, icache_line_size;
 988        unsigned long addr;
 989
 990        icache_size = local_cpu_data().icache_size;
 991        icache_line_size = local_cpu_data().icache_line_size;
 992
 993        /* Clear the valid bits in all the tags. */
 994        for (addr = 0; addr < icache_size; addr += icache_line_size) {
 995                __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 996                                     "membar #Sync"
 997                                     : /* no outputs */
 998                                     : "r" (addr | (2 << 3)),
 999                                       "i" (ASI_IC_TAG));
1000        }
1001}
1002
1003static void cheetah_flush_icache(void)
1004{
1005        unsigned long dcu_save;
1006
1007        /* Save current DCU, disable I-cache. */
1008        __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1009                             "or %0, %2, %%g1\n\t"
1010                             "stxa %%g1, [%%g0] %1\n\t"
1011                             "membar #Sync"
1012                             : "=r" (dcu_save)
1013                             : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
1014                             : "g1");
1015
1016        __cheetah_flush_icache();
1017
1018        /* Restore DCU register */
1019        __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1020                             "membar #Sync"
1021                             : /* no outputs */
1022                             : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
1023}
1024
1025static void cheetah_flush_dcache(void)
1026{
1027        unsigned int dcache_size, dcache_line_size;
1028        unsigned long addr;
1029
1030        dcache_size = local_cpu_data().dcache_size;
1031        dcache_line_size = local_cpu_data().dcache_line_size;
1032
1033        for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1034                __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1035                                     "membar #Sync"
1036                                     : /* no outputs */
1037                                     : "r" (addr), "i" (ASI_DCACHE_TAG));
1038        }
1039}
1040
1041/* In order to make the even parity correct we must do two things.
1042 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
1043 * Next, we clear out all 32-bytes of data for that line.  Data of
1044 * all-zero + tag parity value of zero == correct parity.
1045 */
1046static void cheetah_plus_zap_dcache_parity(void)
1047{
1048        unsigned int dcache_size, dcache_line_size;
1049        unsigned long addr;
1050
1051        dcache_size = local_cpu_data().dcache_size;
1052        dcache_line_size = local_cpu_data().dcache_line_size;
1053
1054        for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1055                unsigned long tag = (addr >> 14);
1056                unsigned long line;
1057
1058                __asm__ __volatile__("membar    #Sync\n\t"
1059                                     "stxa      %0, [%1] %2\n\t"
1060                                     "membar    #Sync"
1061                                     : /* no outputs */
1062                                     : "r" (tag), "r" (addr),
1063                                       "i" (ASI_DCACHE_UTAG));
1064                for (line = addr; line < addr + dcache_line_size; line += 8)
1065                        __asm__ __volatile__("membar    #Sync\n\t"
1066                                             "stxa      %%g0, [%0] %1\n\t"
1067                                             "membar    #Sync"
1068                                             : /* no outputs */
1069                                             : "r" (line),
1070                                               "i" (ASI_DCACHE_DATA));
1071        }
1072}
1073
1074/* Conversion tables used to frob Cheetah AFSR syndrome values into
1075 * something palatable to the memory controller driver get_unumber
1076 * routine.
1077 */
1078#define MT0     137
1079#define MT1     138
1080#define MT2     139
1081#define NONE    254
1082#define MTC0    140
1083#define MTC1    141
1084#define MTC2    142
1085#define MTC3    143
1086#define C0      128
1087#define C1      129
1088#define C2      130
1089#define C3      131
1090#define C4      132
1091#define C5      133
1092#define C6      134
1093#define C7      135
1094#define C8      136
1095#define M2      144
1096#define M3      145
1097#define M4      146
1098#define M       147
1099static unsigned char cheetah_ecc_syntab[] = {
1100/*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1101/*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1102/*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1103/*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1104/*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1105/*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1106/*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1107/*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1108/*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1109/*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1110/*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1111/*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1112/*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1113/*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1114/*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1115/*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1116/*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1117/*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1118/*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1119/*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1120/*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1121/*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1122/*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1123/*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1124/*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1125/*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1126/*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1127/*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1128/*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1129/*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1130/*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1131/*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1132};
1133static unsigned char cheetah_mtag_syntab[] = {
1134       NONE, MTC0,
1135       MTC1, NONE,
1136       MTC2, NONE,
1137       NONE, MT0,
1138       MTC3, NONE,
1139       NONE, MT1,
1140       NONE, MT2,
1141       NONE, NONE
1142};
1143
1144/* Return the highest priority error conditon mentioned. */
1145static inline unsigned long cheetah_get_hipri(unsigned long afsr)
1146{
1147        unsigned long tmp = 0;
1148        int i;
1149
1150        for (i = 0; cheetah_error_table[i].mask; i++) {
1151                if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1152                        return tmp;
1153        }
1154        return tmp;
1155}
1156
1157static const char *cheetah_get_string(unsigned long bit)
1158{
1159        int i;
1160
1161        for (i = 0; cheetah_error_table[i].mask; i++) {
1162                if ((bit & cheetah_error_table[i].mask) != 0UL)
1163                        return cheetah_error_table[i].name;
1164        }
1165        return "???";
1166}
1167
1168static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1169                               unsigned long afsr, unsigned long afar, int recoverable)
1170{
1171        unsigned long hipri;
1172        char unum[256];
1173
1174        printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1175               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1176               afsr, afar,
1177               (afsr & CHAFSR_TL1) ? 1 : 0);
1178        printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1179               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1180               regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1181        printk("%s" "ERROR(%d): ",
1182               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
1183        printk("TPC<%pS>\n", (void *) regs->tpc);
1184        printk("%s" "ERROR(%d): M_SYND(%lx),  E_SYND(%lx)%s%s\n",
1185               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1186               (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1187               (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1188               (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1189               (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1190        hipri = cheetah_get_hipri(afsr);
1191        printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1192               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1193               hipri, cheetah_get_string(hipri));
1194
1195        /* Try to get unumber if relevant. */
1196#define ESYND_ERRORS    (CHAFSR_IVC | CHAFSR_IVU | \
1197                         CHAFSR_CPC | CHAFSR_CPU | \
1198                         CHAFSR_UE  | CHAFSR_CE  | \
1199                         CHAFSR_EDC | CHAFSR_EDU  | \
1200                         CHAFSR_UCC | CHAFSR_UCU  | \
1201                         CHAFSR_WDU | CHAFSR_WDC)
1202#define MSYND_ERRORS    (CHAFSR_EMC | CHAFSR_EMU)
1203        if (afsr & ESYND_ERRORS) {
1204                int syndrome;
1205                int ret;
1206
1207                syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1208                syndrome = cheetah_ecc_syntab[syndrome];
1209                ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1210                if (ret != -1)
1211                        printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1212                               (recoverable ? KERN_WARNING : KERN_CRIT),
1213                               smp_processor_id(), unum);
1214        } else if (afsr & MSYND_ERRORS) {
1215                int syndrome;
1216                int ret;
1217
1218                syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1219                syndrome = cheetah_mtag_syntab[syndrome];
1220                ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1221                if (ret != -1)
1222                        printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1223                               (recoverable ? KERN_WARNING : KERN_CRIT),
1224                               smp_processor_id(), unum);
1225        }
1226
1227        /* Now dump the cache snapshots. */
1228        printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n",
1229               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1230               (int) info->dcache_index,
1231               info->dcache_tag,
1232               info->dcache_utag,
1233               info->dcache_stag);
1234        printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1235               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1236               info->dcache_data[0],
1237               info->dcache_data[1],
1238               info->dcache_data[2],
1239               info->dcache_data[3]);
1240        printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] "
1241               "u[%016llx] l[%016llx]\n",
1242               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1243               (int) info->icache_index,
1244               info->icache_tag,
1245               info->icache_utag,
1246               info->icache_stag,
1247               info->icache_upper,
1248               info->icache_lower);
1249        printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n",
1250               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1251               info->icache_data[0],
1252               info->icache_data[1],
1253               info->icache_data[2],
1254               info->icache_data[3]);
1255        printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n",
1256               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1257               info->icache_data[4],
1258               info->icache_data[5],
1259               info->icache_data[6],
1260               info->icache_data[7]);
1261        printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n",
1262               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1263               (int) info->ecache_index, info->ecache_tag);
1264        printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1265               (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1266               info->ecache_data[0],
1267               info->ecache_data[1],
1268               info->ecache_data[2],
1269               info->ecache_data[3]);
1270
1271        afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1272        while (afsr != 0UL) {
1273                unsigned long bit = cheetah_get_hipri(afsr);
1274
1275                printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1276                       (recoverable ? KERN_WARNING : KERN_CRIT),
1277                       bit, cheetah_get_string(bit));
1278
1279                afsr &= ~bit;
1280        }
1281
1282        if (!recoverable)
1283                printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1284}
1285
1286static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1287{
1288        unsigned long afsr, afar;
1289        int ret = 0;
1290
1291        __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1292                             : "=r" (afsr)
1293                             : "i" (ASI_AFSR));
1294        if ((afsr & cheetah_afsr_errors) != 0) {
1295                if (logp != NULL) {
1296                        __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1297                                             : "=r" (afar)
1298                                             : "i" (ASI_AFAR));
1299                        logp->afsr = afsr;
1300                        logp->afar = afar;
1301                }
1302                ret = 1;
1303        }
1304        __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1305                             "membar #Sync\n\t"
1306                             : : "r" (afsr), "i" (ASI_AFSR));
1307
1308        return ret;
1309}
1310
1311void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1312{
1313        struct cheetah_err_info local_snapshot, *p;
1314        int recoverable;
1315
1316        /* Flush E-cache */
1317        cheetah_flush_ecache();
1318
1319        p = cheetah_get_error_log(afsr);
1320        if (!p) {
1321                prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1322                            afsr, afar);
1323                prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1324                            smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1325                prom_halt();
1326        }
1327
1328        /* Grab snapshot of logged error. */
1329        memcpy(&local_snapshot, p, sizeof(local_snapshot));
1330
1331        /* If the current trap snapshot does not match what the
1332         * trap handler passed along into our args, big trouble.
1333         * In such a case, mark the local copy as invalid.
1334         *
1335         * Else, it matches and we mark the afsr in the non-local
1336         * copy as invalid so we may log new error traps there.
1337         */
1338        if (p->afsr != afsr || p->afar != afar)
1339                local_snapshot.afsr = CHAFSR_INVALID;
1340        else
1341                p->afsr = CHAFSR_INVALID;
1342
1343        cheetah_flush_icache();
1344        cheetah_flush_dcache();
1345
1346        /* Re-enable I-cache/D-cache */
1347        __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1348                             "or %%g1, %1, %%g1\n\t"
1349                             "stxa %%g1, [%%g0] %0\n\t"
1350                             "membar #Sync"
1351                             : /* no outputs */
1352                             : "i" (ASI_DCU_CONTROL_REG),
1353                               "i" (DCU_DC | DCU_IC)
1354                             : "g1");
1355
1356        /* Re-enable error reporting */
1357        __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1358                             "or %%g1, %1, %%g1\n\t"
1359                             "stxa %%g1, [%%g0] %0\n\t"
1360                             "membar #Sync"
1361                             : /* no outputs */
1362                             : "i" (ASI_ESTATE_ERROR_EN),
1363                               "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1364                             : "g1");
1365
1366        /* Decide if we can continue after handling this trap and
1367         * logging the error.
1368         */
1369        recoverable = 1;
1370        if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1371                recoverable = 0;
1372
1373        /* Re-check AFSR/AFAR.  What we are looking for here is whether a new
1374         * error was logged while we had error reporting traps disabled.
1375         */
1376        if (cheetah_recheck_errors(&local_snapshot)) {
1377                unsigned long new_afsr = local_snapshot.afsr;
1378
1379                /* If we got a new asynchronous error, die... */
1380                if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1381                                CHAFSR_WDU | CHAFSR_CPU |
1382                                CHAFSR_IVU | CHAFSR_UE |
1383                                CHAFSR_BERR | CHAFSR_TO))
1384                        recoverable = 0;
1385        }
1386
1387        /* Log errors. */
1388        cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1389
1390        if (!recoverable)
1391                panic("Irrecoverable Fast-ECC error trap.\n");
1392
1393        /* Flush E-cache to kick the error trap handlers out. */
1394        cheetah_flush_ecache();
1395}
1396
1397/* Try to fix a correctable error by pushing the line out from
1398 * the E-cache.  Recheck error reporting registers to see if the
1399 * problem is intermittent.
1400 */
1401static int cheetah_fix_ce(unsigned long physaddr)
1402{
1403        unsigned long orig_estate;
1404        unsigned long alias1, alias2;
1405        int ret;
1406
1407        /* Make sure correctable error traps are disabled. */
1408        __asm__ __volatile__("ldxa      [%%g0] %2, %0\n\t"
1409                             "andn      %0, %1, %%g1\n\t"
1410                             "stxa      %%g1, [%%g0] %2\n\t"
1411                             "membar    #Sync"
1412                             : "=&r" (orig_estate)
1413                             : "i" (ESTATE_ERROR_CEEN),
1414                               "i" (ASI_ESTATE_ERROR_EN)
1415                             : "g1");
1416
1417        /* We calculate alias addresses that will force the
1418         * cache line in question out of the E-cache.  Then
1419         * we bring it back in with an atomic instruction so
1420         * that we get it in some modified/exclusive state,
1421         * then we displace it again to try and get proper ECC
1422         * pushed back into the system.
1423         */
1424        physaddr &= ~(8UL - 1UL);
1425        alias1 = (ecache_flush_physbase +
1426                  (physaddr & ((ecache_flush_size >> 1) - 1)));
1427        alias2 = alias1 + (ecache_flush_size >> 1);
1428        __asm__ __volatile__("ldxa      [%0] %3, %%g0\n\t"
1429                             "ldxa      [%1] %3, %%g0\n\t"
1430                             "casxa     [%2] %3, %%g0, %%g0\n\t"
1431                             "ldxa      [%0] %3, %%g0\n\t"
1432                             "ldxa      [%1] %3, %%g0\n\t"
1433                             "membar    #Sync"
1434                             : /* no outputs */
1435                             : "r" (alias1), "r" (alias2),
1436                               "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1437
1438        /* Did that trigger another error? */
1439        if (cheetah_recheck_errors(NULL)) {
1440                /* Try one more time. */
1441                __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1442                                     "membar #Sync"
1443                                     : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1444                if (cheetah_recheck_errors(NULL))
1445                        ret = 2;
1446                else
1447                        ret = 1;
1448        } else {
1449                /* No new error, intermittent problem. */
1450                ret = 0;
1451        }
1452
1453        /* Restore error enables. */
1454        __asm__ __volatile__("stxa      %0, [%%g0] %1\n\t"
1455                             "membar    #Sync"
1456                             : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1457
1458        return ret;
1459}
1460
1461/* Return non-zero if PADDR is a valid physical memory address. */
1462static int cheetah_check_main_memory(unsigned long paddr)
1463{
1464        unsigned long vaddr = PAGE_OFFSET + paddr;
1465
1466        if (vaddr > (unsigned long) high_memory)
1467                return 0;
1468
1469        return kern_addr_valid(vaddr);
1470}
1471
1472void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1473{
1474        struct cheetah_err_info local_snapshot, *p;
1475        int recoverable, is_memory;
1476
1477        p = cheetah_get_error_log(afsr);
1478        if (!p) {
1479                prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1480                            afsr, afar);
1481                prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1482                            smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1483                prom_halt();
1484        }
1485
1486        /* Grab snapshot of logged error. */
1487        memcpy(&local_snapshot, p, sizeof(local_snapshot));
1488
1489        /* If the current trap snapshot does not match what the
1490         * trap handler passed along into our args, big trouble.
1491         * In such a case, mark the local copy as invalid.
1492         *
1493         * Else, it matches and we mark the afsr in the non-local
1494         * copy as invalid so we may log new error traps there.
1495         */
1496        if (p->afsr != afsr || p->afar != afar)
1497                local_snapshot.afsr = CHAFSR_INVALID;
1498        else
1499                p->afsr = CHAFSR_INVALID;
1500
1501        is_memory = cheetah_check_main_memory(afar);
1502
1503        if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1504                /* XXX Might want to log the results of this operation
1505                 * XXX somewhere... -DaveM
1506                 */
1507                cheetah_fix_ce(afar);
1508        }
1509
1510        {
1511                int flush_all, flush_line;
1512
1513                flush_all = flush_line = 0;
1514                if ((afsr & CHAFSR_EDC) != 0UL) {
1515                        if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1516                                flush_line = 1;
1517                        else
1518                                flush_all = 1;
1519                } else if ((afsr & CHAFSR_CPC) != 0UL) {
1520                        if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1521                                flush_line = 1;
1522                        else
1523                                flush_all = 1;
1524                }
1525
1526                /* Trap handler only disabled I-cache, flush it. */
1527                cheetah_flush_icache();
1528
1529                /* Re-enable I-cache */
1530                __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1531                                     "or %%g1, %1, %%g1\n\t"
1532                                     "stxa %%g1, [%%g0] %0\n\t"
1533                                     "membar #Sync"
1534                                     : /* no outputs */
1535                                     : "i" (ASI_DCU_CONTROL_REG),
1536                                     "i" (DCU_IC)
1537                                     : "g1");
1538
1539                if (flush_all)
1540                        cheetah_flush_ecache();
1541                else if (flush_line)
1542                        cheetah_flush_ecache_line(afar);
1543        }
1544
1545        /* Re-enable error reporting */
1546        __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1547                             "or %%g1, %1, %%g1\n\t"
1548                             "stxa %%g1, [%%g0] %0\n\t"
1549                             "membar #Sync"
1550                             : /* no outputs */
1551                             : "i" (ASI_ESTATE_ERROR_EN),
1552                               "i" (ESTATE_ERROR_CEEN)
1553                             : "g1");
1554
1555        /* Decide if we can continue after handling this trap and
1556         * logging the error.
1557         */
1558        recoverable = 1;
1559        if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1560                recoverable = 0;
1561
1562        /* Re-check AFSR/AFAR */
1563        (void) cheetah_recheck_errors(&local_snapshot);
1564
1565        /* Log errors. */
1566        cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1567
1568        if (!recoverable)
1569                panic("Irrecoverable Correctable-ECC error trap.\n");
1570}
1571
1572void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1573{
1574        struct cheetah_err_info local_snapshot, *p;
1575        int recoverable, is_memory;
1576
1577#ifdef CONFIG_PCI
1578        /* Check for the special PCI poke sequence. */
1579        if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1580                cheetah_flush_icache();
1581                cheetah_flush_dcache();
1582
1583                /* Re-enable I-cache/D-cache */
1584                __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1585                                     "or %%g1, %1, %%g1\n\t"
1586                                     "stxa %%g1, [%%g0] %0\n\t"
1587                                     "membar #Sync"
1588                                     : /* no outputs */
1589                                     : "i" (ASI_DCU_CONTROL_REG),
1590                                       "i" (DCU_DC | DCU_IC)
1591                                     : "g1");
1592
1593                /* Re-enable error reporting */
1594                __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1595                                     "or %%g1, %1, %%g1\n\t"
1596                                     "stxa %%g1, [%%g0] %0\n\t"
1597                                     "membar #Sync"
1598                                     : /* no outputs */
1599                                     : "i" (ASI_ESTATE_ERROR_EN),
1600                                       "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1601                                     : "g1");
1602
1603                (void) cheetah_recheck_errors(NULL);
1604
1605                pci_poke_faulted = 1;
1606                regs->tpc += 4;
1607                regs->tnpc = regs->tpc + 4;
1608                return;
1609        }
1610#endif
1611
1612        p = cheetah_get_error_log(afsr);
1613        if (!p) {
1614                prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1615                            afsr, afar);
1616                prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1617                            smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1618                prom_halt();
1619        }
1620
1621        /* Grab snapshot of logged error. */
1622        memcpy(&local_snapshot, p, sizeof(local_snapshot));
1623
1624        /* If the current trap snapshot does not match what the
1625         * trap handler passed along into our args, big trouble.
1626         * In such a case, mark the local copy as invalid.
1627         *
1628         * Else, it matches and we mark the afsr in the non-local
1629         * copy as invalid so we may log new error traps there.
1630         */
1631        if (p->afsr != afsr || p->afar != afar)
1632                local_snapshot.afsr = CHAFSR_INVALID;
1633        else
1634                p->afsr = CHAFSR_INVALID;
1635
1636        is_memory = cheetah_check_main_memory(afar);
1637
1638        {
1639                int flush_all, flush_line;
1640
1641                flush_all = flush_line = 0;
1642                if ((afsr & CHAFSR_EDU) != 0UL) {
1643                        if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1644                                flush_line = 1;
1645                        else
1646                                flush_all = 1;
1647                } else if ((afsr & CHAFSR_BERR) != 0UL) {
1648                        if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1649                                flush_line = 1;
1650                        else
1651                                flush_all = 1;
1652                }
1653
1654                cheetah_flush_icache();
1655                cheetah_flush_dcache();
1656
1657                /* Re-enable I/D caches */
1658                __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1659                                     "or %%g1, %1, %%g1\n\t"
1660                                     "stxa %%g1, [%%g0] %0\n\t"
1661                                     "membar #Sync"
1662                                     : /* no outputs */
1663                                     : "i" (ASI_DCU_CONTROL_REG),
1664                                     "i" (DCU_IC | DCU_DC)
1665                                     : "g1");
1666
1667                if (flush_all)
1668                        cheetah_flush_ecache();
1669                else if (flush_line)
1670                        cheetah_flush_ecache_line(afar);
1671        }
1672
1673        /* Re-enable error reporting */
1674        __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1675                             "or %%g1, %1, %%g1\n\t"
1676                             "stxa %%g1, [%%g0] %0\n\t"
1677                             "membar #Sync"
1678                             : /* no outputs */
1679                             : "i" (ASI_ESTATE_ERROR_EN),
1680                             "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1681                             : "g1");
1682
1683        /* Decide if we can continue after handling this trap and
1684         * logging the error.
1685         */
1686        recoverable = 1;
1687        if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1688                recoverable = 0;
1689
1690        /* Re-check AFSR/AFAR.  What we are looking for here is whether a new
1691         * error was logged while we had error reporting traps disabled.
1692         */
1693        if (cheetah_recheck_errors(&local_snapshot)) {
1694                unsigned long new_afsr = local_snapshot.afsr;
1695
1696                /* If we got a new asynchronous error, die... */
1697                if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1698                                CHAFSR_WDU | CHAFSR_CPU |
1699                                CHAFSR_IVU | CHAFSR_UE |
1700                                CHAFSR_BERR | CHAFSR_TO))
1701                        recoverable = 0;
1702        }
1703
1704        /* Log errors. */
1705        cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1706
1707        /* "Recoverable" here means we try to yank the page from ever
1708         * being newly used again.  This depends upon a few things:
1709         * 1) Must be main memory, and AFAR must be valid.
1710         * 2) If we trapped from user, OK.
1711         * 3) Else, if we trapped from kernel we must find exception
1712         *    table entry (ie. we have to have been accessing user
1713         *    space).
1714         *
1715         * If AFAR is not in main memory, or we trapped from kernel
1716         * and cannot find an exception table entry, it is unacceptable
1717         * to try and continue.
1718         */
1719        if (recoverable && is_memory) {
1720                if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1721                        /* OK, usermode access. */
1722                        recoverable = 1;
1723                } else {
1724                        const struct exception_table_entry *entry;
1725
1726                        entry = search_exception_tables(regs->tpc);
1727                        if (entry) {
1728                                /* OK, kernel access to userspace. */
1729                                recoverable = 1;
1730
1731                        } else {
1732                                /* BAD, privileged state is corrupted. */
1733                                recoverable = 0;
1734                        }
1735
1736                        if (recoverable) {
1737                                if (pfn_valid(afar >> PAGE_SHIFT))
1738                                        get_page(pfn_to_page(afar >> PAGE_SHIFT));
1739                                else
1740                                        recoverable = 0;
1741
1742                                /* Only perform fixup if we still have a
1743                                 * recoverable condition.
1744                                 */
1745                                if (recoverable) {
1746                                        regs->tpc = entry->fixup;
1747                                        regs->tnpc = regs->tpc + 4;
1748                                }
1749                        }
1750                }
1751        } else {
1752                recoverable = 0;
1753        }
1754
1755        if (!recoverable)
1756                panic("Irrecoverable deferred error trap.\n");
1757}
1758
1759/* Handle a D/I cache parity error trap.  TYPE is encoded as:
1760 *
1761 * Bit0:        0=dcache,1=icache
1762 * Bit1:        0=recoverable,1=unrecoverable
1763 *
1764 * The hardware has disabled both the I-cache and D-cache in
1765 * the %dcr register.  
1766 */
1767void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1768{
1769        if (type & 0x1)
1770                __cheetah_flush_icache();
1771        else
1772                cheetah_plus_zap_dcache_parity();
1773        cheetah_flush_dcache();
1774
1775        /* Re-enable I-cache/D-cache */
1776        __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1777                             "or %%g1, %1, %%g1\n\t"
1778                             "stxa %%g1, [%%g0] %0\n\t"
1779                             "membar #Sync"
1780                             : /* no outputs */
1781                             : "i" (ASI_DCU_CONTROL_REG),
1782                               "i" (DCU_DC | DCU_IC)
1783                             : "g1");
1784
1785        if (type & 0x2) {
1786                printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1787                       smp_processor_id(),
1788                       (type & 0x1) ? 'I' : 'D',
1789                       regs->tpc);
1790                printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
1791                panic("Irrecoverable Cheetah+ parity error.");
1792        }
1793
1794        printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1795               smp_processor_id(),
1796               (type & 0x1) ? 'I' : 'D',
1797               regs->tpc);
1798        printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
1799}
1800
1801struct sun4v_error_entry {
1802        /* Unique error handle */
1803/*0x00*/u64             err_handle;
1804
1805        /* %stick value at the time of the error */
1806/*0x08*/u64             err_stick;
1807
1808/*0x10*/u8              reserved_1[3];
1809
1810        /* Error type */
1811/*0x13*/u8              err_type;
1812#define SUN4V_ERR_TYPE_UNDEFINED        0
1813#define SUN4V_ERR_TYPE_UNCORRECTED_RES  1
1814#define SUN4V_ERR_TYPE_PRECISE_NONRES   2
1815#define SUN4V_ERR_TYPE_DEFERRED_NONRES  3
1816#define SUN4V_ERR_TYPE_SHUTDOWN_RQST    4
1817#define SUN4V_ERR_TYPE_DUMP_CORE        5
1818#define SUN4V_ERR_TYPE_SP_STATE_CHANGE  6
1819#define SUN4V_ERR_TYPE_NUM              7
1820
1821        /* Error attributes */
1822/*0x14*/u32             err_attrs;
1823#define SUN4V_ERR_ATTRS_PROCESSOR       0x00000001
1824#define SUN4V_ERR_ATTRS_MEMORY          0x00000002
1825#define SUN4V_ERR_ATTRS_PIO             0x00000004
1826#define SUN4V_ERR_ATTRS_INT_REGISTERS   0x00000008
1827#define SUN4V_ERR_ATTRS_FPU_REGISTERS   0x00000010
1828#define SUN4V_ERR_ATTRS_SHUTDOWN_RQST   0x00000020
1829#define SUN4V_ERR_ATTRS_ASR             0x00000040
1830#define SUN4V_ERR_ATTRS_ASI             0x00000080
1831#define SUN4V_ERR_ATTRS_PRIV_REG        0x00000100
1832#define SUN4V_ERR_ATTRS_SPSTATE_MSK     0x00000600
1833#define SUN4V_ERR_ATTRS_MCD             0x00000800
1834#define SUN4V_ERR_ATTRS_SPSTATE_SHFT    9
1835#define SUN4V_ERR_ATTRS_MODE_MSK        0x03000000
1836#define SUN4V_ERR_ATTRS_MODE_SHFT       24
1837#define SUN4V_ERR_ATTRS_RES_QUEUE_FULL  0x80000000
1838
1839#define SUN4V_ERR_SPSTATE_FAULTED       0
1840#define SUN4V_ERR_SPSTATE_AVAILABLE     1
1841#define SUN4V_ERR_SPSTATE_NOT_PRESENT   2
1842
1843#define SUN4V_ERR_MODE_USER             1
1844#define SUN4V_ERR_MODE_PRIV             2
1845
1846        /* Real address of the memory region or PIO transaction */
1847/*0x18*/u64             err_raddr;
1848
1849        /* Size of the operation triggering the error, in bytes */
1850/*0x20*/u32             err_size;
1851
1852        /* ID of the CPU */
1853/*0x24*/u16             err_cpu;
1854
1855        /* Grace periof for shutdown, in seconds */
1856/*0x26*/u16             err_secs;
1857
1858        /* Value of the %asi register */
1859/*0x28*/u8              err_asi;
1860
1861/*0x29*/u8              reserved_2;
1862
1863        /* Value of the ASR register number */
1864/*0x2a*/u16             err_asr;
1865#define SUN4V_ERR_ASR_VALID             0x8000
1866
1867/*0x2c*/u32             reserved_3;
1868/*0x30*/u64             reserved_4;
1869/*0x38*/u64             reserved_5;
1870};
1871
1872static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1873static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1874
1875static const char *sun4v_err_type_to_str(u8 type)
1876{
1877        static const char *types[SUN4V_ERR_TYPE_NUM] = {
1878                "undefined",
1879                "uncorrected resumable",
1880                "precise nonresumable",
1881                "deferred nonresumable",
1882                "shutdown request",
1883                "dump core",
1884                "SP state change",
1885        };
1886
1887        if (type < SUN4V_ERR_TYPE_NUM)
1888                return types[type];
1889
1890        return "unknown";
1891}
1892
1893static void sun4v_emit_err_attr_strings(u32 attrs)
1894{
1895        static const char *attr_names[] = {
1896                "processor",
1897                "memory",
1898                "PIO",
1899                "int-registers",
1900                "fpu-registers",
1901                "shutdown-request",
1902                "ASR",
1903                "ASI",
1904                "priv-reg",
1905        };
1906        static const char *sp_states[] = {
1907                "sp-faulted",
1908                "sp-available",
1909                "sp-not-present",
1910                "sp-state-reserved",
1911        };
1912        static const char *modes[] = {
1913                "mode-reserved0",
1914                "user",
1915                "priv",
1916                "mode-reserved1",
1917        };
1918        u32 sp_state, mode;
1919        int i;
1920
1921        for (i = 0; i < ARRAY_SIZE(attr_names); i++) {
1922                if (attrs & (1U << i)) {
1923                        const char *s = attr_names[i];
1924
1925                        pr_cont("%s ", s);
1926                }
1927        }
1928
1929        sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >>
1930                    SUN4V_ERR_ATTRS_SPSTATE_SHFT);
1931        pr_cont("%s ", sp_states[sp_state]);
1932
1933        mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >>
1934                SUN4V_ERR_ATTRS_MODE_SHFT);
1935        pr_cont("%s ", modes[mode]);
1936
1937        if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL)
1938                pr_cont("res-queue-full ");
1939}
1940
1941/* When the report contains a real-address of "-1" it means that the
1942 * hardware did not provide the address.  So we compute the effective
1943 * address of the load or store instruction at regs->tpc and report
1944 * that.  Usually when this happens it's a PIO and in such a case we
1945 * are using physical addresses with bypass ASIs anyways, so what we
1946 * report here is exactly what we want.
1947 */
1948static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs)
1949{
1950        unsigned int insn;
1951        u64 addr;
1952
1953        if (!(regs->tstate & TSTATE_PRIV))
1954                return;
1955
1956        insn = *(unsigned int *) regs->tpc;
1957
1958        addr = compute_effective_address(regs, insn, 0);
1959
1960        printk("%s: insn effective address [0x%016llx]\n",
1961               pfx, addr);
1962}
1963
1964static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
1965                            int cpu, const char *pfx, atomic_t *ocnt)
1966{
1967        u64 *raw_ptr = (u64 *) ent;
1968        u32 attrs;
1969        int cnt;
1970
1971        printk("%s: Reporting on cpu %d\n", pfx, cpu);
1972        printk("%s: TPC [0x%016lx] <%pS>\n",
1973               pfx, regs->tpc, (void *) regs->tpc);
1974
1975        printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n",
1976               pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]);
1977        printk("%s:      %016llx:%016llx:%016llx:%016llx]\n",
1978               pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]);
1979
1980        printk("%s: handle [0x%016llx] stick [0x%016llx]\n",
1981               pfx, ent->err_handle, ent->err_stick);
1982
1983        printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type));
1984
1985        attrs = ent->err_attrs;
1986        printk("%s: attrs [0x%08x] < ", pfx, attrs);
1987        sun4v_emit_err_attr_strings(attrs);
1988        pr_cont(">\n");
1989
1990        /* Various fields in the error report are only valid if
1991         * certain attribute bits are set.
1992         */
1993        if (attrs & (SUN4V_ERR_ATTRS_MEMORY |
1994                     SUN4V_ERR_ATTRS_PIO |
1995                     SUN4V_ERR_ATTRS_ASI)) {
1996                printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr);
1997
1998                if (ent->err_raddr == ~(u64)0)
1999                        sun4v_report_real_raddr(pfx, regs);
2000        }
2001
2002        if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI))
2003                printk("%s: size [0x%x]\n", pfx, ent->err_size);
2004
2005        if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR |
2006                     SUN4V_ERR_ATTRS_INT_REGISTERS |
2007                     SUN4V_ERR_ATTRS_FPU_REGISTERS |
2008                     SUN4V_ERR_ATTRS_PRIV_REG))
2009                printk("%s: cpu[%u]\n", pfx, ent->err_cpu);
2010
2011        if (attrs & SUN4V_ERR_ATTRS_ASI)
2012                printk("%s: asi [0x%02x]\n", pfx, ent->err_asi);
2013
2014        if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS |
2015                      SUN4V_ERR_ATTRS_FPU_REGISTERS |
2016                      SUN4V_ERR_ATTRS_PRIV_REG)) &&
2017            (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0)
2018                printk("%s: reg [0x%04x]\n",
2019                       pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID);
2020
2021        show_regs(regs);
2022
2023        if ((cnt = atomic_read(ocnt)) != 0) {
2024                atomic_set(ocnt, 0);
2025                wmb();
2026                printk("%s: Queue overflowed %d times.\n",
2027                       pfx, cnt);
2028        }
2029}
2030
2031/* Handle memory corruption detected error which is vectored in
2032 * through resumable error trap.
2033 */
2034void do_mcd_err(struct pt_regs *regs, struct sun4v_error_entry ent)
2035{
2036        if (notify_die(DIE_TRAP, "MCD error", regs, 0, 0x34,
2037                       SIGSEGV) == NOTIFY_STOP)
2038                return;
2039
2040        if (regs->tstate & TSTATE_PRIV) {
2041                /* MCD exception could happen because the task was
2042                 * running a system call with MCD enabled and passed a
2043                 * non-versioned pointer or pointer with bad version
2044                 * tag to the system call. In such cases, hypervisor
2045                 * places the address of offending instruction in the
2046                 * resumable error report. This is a deferred error,
2047                 * so the read/write that caused the trap was potentially
2048                 * retired long time back and we may have no choice
2049                 * but to send SIGSEGV to the process.
2050                 */
2051                const struct exception_table_entry *entry;
2052
2053                entry = search_exception_tables(regs->tpc);
2054                if (entry) {
2055                        /* Looks like a bad syscall parameter */
2056#ifdef DEBUG_EXCEPTIONS
2057                        pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n",
2058                                 regs->tpc);
2059                        pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
2060                                 ent.err_raddr, entry->fixup);
2061#endif
2062                        regs->tpc = entry->fixup;
2063                        regs->tnpc = regs->tpc + 4;
2064                        return;
2065                }
2066        }
2067
2068        /* Send SIGSEGV to the userspace process with the right signal
2069         * code
2070         */
2071        force_sig_fault(SIGSEGV, SEGV_ADIDERR, (void __user *)ent.err_raddr);
2072}
2073
2074/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2075 * Log the event and clear the first word of the entry.
2076 */
2077void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
2078{
2079        enum ctx_state prev_state = exception_enter();
2080        struct sun4v_error_entry *ent, local_copy;
2081        struct trap_per_cpu *tb;
2082        unsigned long paddr;
2083        int cpu;
2084
2085        cpu = get_cpu();
2086
2087        tb = &trap_block[cpu];
2088        paddr = tb->resum_kernel_buf_pa + offset;
2089        ent = __va(paddr);
2090
2091        memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2092
2093        /* We have a local copy now, so release the entry.  */
2094        ent->err_handle = 0;
2095        wmb();
2096
2097        put_cpu();
2098
2099        if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) {
2100                /* We should really take the seconds field of
2101                 * the error report and use it for the shutdown
2102                 * invocation, but for now do the same thing we
2103                 * do for a DS shutdown request.
2104                 */
2105                pr_info("Shutdown request, %u seconds...\n",
2106                        local_copy.err_secs);
2107                orderly_poweroff(true);
2108                goto out;
2109        }
2110
2111        /* If this is a memory corruption detected error vectored in
2112         * by HV through resumable error trap, call the handler
2113         */
2114        if (local_copy.err_attrs & SUN4V_ERR_ATTRS_MCD) {
2115                do_mcd_err(regs, local_copy);
2116                return;
2117        }
2118
2119        sun4v_log_error(regs, &local_copy, cpu,
2120                        KERN_ERR "RESUMABLE ERROR",
2121                        &sun4v_resum_oflow_cnt);
2122out:
2123        exception_exit(prev_state);
2124}
2125
2126/* If we try to printk() we'll probably make matters worse, by trying
2127 * to retake locks this cpu already holds or causing more errors. So
2128 * just bump a counter, and we'll report these counter bumps above.
2129 */
2130void sun4v_resum_overflow(struct pt_regs *regs)
2131{
2132        atomic_inc(&sun4v_resum_oflow_cnt);
2133}
2134
2135/* Given a set of registers, get the virtual addressi that was being accessed
2136 * by the faulting instructions at tpc.
2137 */
2138static unsigned long sun4v_get_vaddr(struct pt_regs *regs)
2139{
2140        unsigned int insn;
2141
2142        if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) {
2143                return compute_effective_address(regs, insn,
2144                                                 (insn >> 25) & 0x1f);
2145        }
2146        return 0;
2147}
2148
2149/* Attempt to handle non-resumable errors generated from userspace.
2150 * Returns true if the signal was handled, false otherwise.
2151 */
2152bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
2153                                  struct sun4v_error_entry *ent) {
2154
2155        unsigned int attrs = ent->err_attrs;
2156
2157        if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
2158                unsigned long addr = ent->err_raddr;
2159
2160                if (addr == ~(u64)0) {
2161                        /* This seems highly unlikely to ever occur */
2162                        pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n");
2163                } else {
2164                        unsigned long page_cnt = DIV_ROUND_UP(ent->err_size,
2165                                                              PAGE_SIZE);
2166
2167                        /* Break the unfortunate news. */
2168                        pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n",
2169                                 addr);
2170                        pr_emerg("SUN4V NON-RECOVERABLE ERROR:   Claiming %lu ages.\n",
2171                                 page_cnt);
2172
2173                        while (page_cnt-- > 0) {
2174                                if (pfn_valid(addr >> PAGE_SHIFT))
2175                                        get_page(pfn_to_page(addr >> PAGE_SHIFT));
2176                                addr += PAGE_SIZE;
2177                        }
2178                }
2179                force_sig(SIGKILL);
2180
2181                return true;
2182        }
2183        if (attrs & SUN4V_ERR_ATTRS_PIO) {
2184                force_sig_fault(SIGBUS, BUS_ADRERR,
2185                                (void __user *)sun4v_get_vaddr(regs));
2186                return true;
2187        }
2188
2189        /* Default to doing nothing */
2190        return false;
2191}
2192
2193/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2194 * Log the event, clear the first word of the entry, and die.
2195 */
2196void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
2197{
2198        struct sun4v_error_entry *ent, local_copy;
2199        struct trap_per_cpu *tb;
2200        unsigned long paddr;
2201        int cpu;
2202
2203        cpu = get_cpu();
2204
2205        tb = &trap_block[cpu];
2206        paddr = tb->nonresum_kernel_buf_pa + offset;
2207        ent = __va(paddr);
2208
2209        memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2210
2211        /* We have a local copy now, so release the entry.  */
2212        ent->err_handle = 0;
2213        wmb();
2214
2215        put_cpu();
2216
2217        if (!(regs->tstate & TSTATE_PRIV) &&
2218            sun4v_nonresum_error_user_handled(regs, &local_copy)) {
2219                /* DON'T PANIC: This userspace error was handled. */
2220                return;
2221        }
2222
2223#ifdef CONFIG_PCI
2224        /* Check for the special PCI poke sequence. */
2225        if (pci_poke_in_progress && pci_poke_cpu == cpu) {
2226                pci_poke_faulted = 1;
2227                regs->tpc += 4;
2228                regs->tnpc = regs->tpc + 4;
2229                return;
2230        }
2231#endif
2232
2233        sun4v_log_error(regs, &local_copy, cpu,
2234                        KERN_EMERG "NON-RESUMABLE ERROR",
2235                        &sun4v_nonresum_oflow_cnt);
2236
2237        panic("Non-resumable error.");
2238}
2239
2240/* If we try to printk() we'll probably make matters worse, by trying
2241 * to retake locks this cpu already holds or causing more errors. So
2242 * just bump a counter, and we'll report these counter bumps above.
2243 */
2244void sun4v_nonresum_overflow(struct pt_regs *regs)
2245{
2246        /* XXX Actually even this can make not that much sense.  Perhaps
2247         * XXX we should just pull the plug and panic directly from here?
2248         */
2249        atomic_inc(&sun4v_nonresum_oflow_cnt);
2250}
2251
2252static void sun4v_tlb_error(struct pt_regs *regs)
2253{
2254        die_if_kernel("TLB/TSB error", regs);
2255}
2256
2257unsigned long sun4v_err_itlb_vaddr;
2258unsigned long sun4v_err_itlb_ctx;
2259unsigned long sun4v_err_itlb_pte;
2260unsigned long sun4v_err_itlb_error;
2261
2262void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2263{
2264        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2265
2266        printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2267               regs->tpc, tl);
2268        printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
2269        printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2270        printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
2271               (void *) regs->u_regs[UREG_I7]);
2272        printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
2273               "pte[%lx] error[%lx]\n",
2274               sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2275               sun4v_err_itlb_pte, sun4v_err_itlb_error);
2276
2277        sun4v_tlb_error(regs);
2278}
2279
2280unsigned long sun4v_err_dtlb_vaddr;
2281unsigned long sun4v_err_dtlb_ctx;
2282unsigned long sun4v_err_dtlb_pte;
2283unsigned long sun4v_err_dtlb_error;
2284
2285void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2286{
2287        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2288
2289        printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2290               regs->tpc, tl);
2291        printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
2292        printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2293        printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
2294               (void *) regs->u_regs[UREG_I7]);
2295        printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
2296               "pte[%lx] error[%lx]\n",
2297               sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2298               sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2299
2300        sun4v_tlb_error(regs);
2301}
2302
2303void hypervisor_tlbop_error(unsigned long err, unsigned long op)
2304{
2305        printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
2306               err, op);
2307}
2308
2309void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
2310{
2311        printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
2312               err, op);
2313}
2314
2315static void do_fpe_common(struct pt_regs *regs)
2316{
2317        if (regs->tstate & TSTATE_PRIV) {
2318                regs->tpc = regs->tnpc;
2319                regs->tnpc += 4;
2320        } else {
2321                unsigned long fsr = current_thread_info()->xfsr[0];
2322                int code;
2323
2324                if (test_thread_flag(TIF_32BIT)) {
2325                        regs->tpc &= 0xffffffff;
2326                        regs->tnpc &= 0xffffffff;
2327                }
2328                code = FPE_FLTUNK;
2329                if ((fsr & 0x1c000) == (1 << 14)) {
2330                        if (fsr & 0x10)
2331                                code = FPE_FLTINV;
2332                        else if (fsr & 0x08)
2333                                code = FPE_FLTOVF;
2334                        else if (fsr & 0x04)
2335                                code = FPE_FLTUND;
2336                        else if (fsr & 0x02)
2337                                code = FPE_FLTDIV;
2338                        else if (fsr & 0x01)
2339                                code = FPE_FLTRES;
2340                }
2341                force_sig_fault(SIGFPE, code, (void __user *)regs->tpc);
2342        }
2343}
2344
2345void do_fpieee(struct pt_regs *regs)
2346{
2347        enum ctx_state prev_state = exception_enter();
2348
2349        if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2350                       0, 0x24, SIGFPE) == NOTIFY_STOP)
2351                goto out;
2352
2353        do_fpe_common(regs);
2354out:
2355        exception_exit(prev_state);
2356}
2357
2358void do_fpother(struct pt_regs *regs)
2359{
2360        enum ctx_state prev_state = exception_enter();
2361        struct fpustate *f = FPUSTATE;
2362        int ret = 0;
2363
2364        if (notify_die(DIE_TRAP, "fpu exception other", regs,
2365                       0, 0x25, SIGFPE) == NOTIFY_STOP)
2366                goto out;
2367
2368        switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2369        case (2 << 14): /* unfinished_FPop */
2370        case (3 << 14): /* unimplemented_FPop */
2371                ret = do_mathemu(regs, f, false);
2372                break;
2373        }
2374        if (ret)
2375                goto out;
2376        do_fpe_common(regs);
2377out:
2378        exception_exit(prev_state);
2379}
2380
2381void do_tof(struct pt_regs *regs)
2382{
2383        enum ctx_state prev_state = exception_enter();
2384
2385        if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2386                       0, 0x26, SIGEMT) == NOTIFY_STOP)
2387                goto out;
2388
2389        if (regs->tstate & TSTATE_PRIV)
2390                die_if_kernel("Penguin overflow trap from kernel mode", regs);
2391        if (test_thread_flag(TIF_32BIT)) {
2392                regs->tpc &= 0xffffffff;
2393                regs->tnpc &= 0xffffffff;
2394        }
2395        force_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)regs->tpc);
2396out:
2397        exception_exit(prev_state);
2398}
2399
2400void do_div0(struct pt_regs *regs)
2401{
2402        enum ctx_state prev_state = exception_enter();
2403
2404        if (notify_die(DIE_TRAP, "integer division by zero", regs,
2405                       0, 0x28, SIGFPE) == NOTIFY_STOP)
2406                goto out;
2407
2408        if (regs->tstate & TSTATE_PRIV)
2409                die_if_kernel("TL0: Kernel divide by zero.", regs);
2410        if (test_thread_flag(TIF_32BIT)) {
2411                regs->tpc &= 0xffffffff;
2412                regs->tnpc &= 0xffffffff;
2413        }
2414        force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->tpc);
2415out:
2416        exception_exit(prev_state);
2417}
2418
2419static void instruction_dump(unsigned int *pc)
2420{
2421        int i;
2422
2423        if ((((unsigned long) pc) & 3))
2424                return;
2425
2426        printk("Instruction DUMP:");
2427        for (i = -3; i < 6; i++)
2428                printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2429        printk("\n");
2430}
2431
2432static void user_instruction_dump(unsigned int __user *pc)
2433{
2434        int i;
2435        unsigned int buf[9];
2436        
2437        if ((((unsigned long) pc) & 3))
2438                return;
2439                
2440        if (copy_from_user(buf, pc - 3, sizeof(buf)))
2441                return;
2442
2443        printk("Instruction DUMP:");
2444        for (i = 0; i < 9; i++)
2445                printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2446        printk("\n");
2447}
2448
2449void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl)
2450{
2451        unsigned long fp, ksp;
2452        struct thread_info *tp;
2453        int count = 0;
2454#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2455        int graph = 0;
2456#endif
2457
2458        ksp = (unsigned long) _ksp;
2459        if (!tsk)
2460                tsk = current;
2461        tp = task_thread_info(tsk);
2462        if (ksp == 0UL) {
2463                if (tsk == current)
2464                        asm("mov %%fp, %0" : "=r" (ksp));
2465                else
2466                        ksp = tp->ksp;
2467        }
2468        if (tp == current_thread_info())
2469                flushw_all();
2470
2471        fp = ksp + STACK_BIAS;
2472
2473        printk("%sCall Trace:\n", loglvl);
2474        do {
2475                struct sparc_stackf *sf;
2476                struct pt_regs *regs;
2477                unsigned long pc;
2478
2479                if (!kstack_valid(tp, fp))
2480                        break;
2481                sf = (struct sparc_stackf *) fp;
2482                regs = (struct pt_regs *) (sf + 1);
2483
2484                if (kstack_is_trap_frame(tp, regs)) {
2485                        if (!(regs->tstate & TSTATE_PRIV))
2486                                break;
2487                        pc = regs->tpc;
2488                        fp = regs->u_regs[UREG_I6] + STACK_BIAS;
2489                } else {
2490                        pc = sf->callers_pc;
2491                        fp = (unsigned long)sf->fp + STACK_BIAS;
2492                }
2493
2494                print_ip_sym(loglvl, pc);
2495#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2496                if ((pc + 8UL) == (unsigned long) &return_to_handler) {
2497                        struct ftrace_ret_stack *ret_stack;
2498                        ret_stack = ftrace_graph_get_ret_stack(tsk, graph);
2499                        if (ret_stack) {
2500                                pc = ret_stack->ret;
2501                                print_ip_sym(loglvl, pc);
2502                                graph++;
2503                        }
2504                }
2505#endif
2506        } while (++count < 16);
2507}
2508
2509static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2510{
2511        unsigned long fp = rw->ins[6];
2512
2513        if (!fp)
2514                return NULL;
2515
2516        return (struct reg_window *) (fp + STACK_BIAS);
2517}
2518
2519void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
2520{
2521        static int die_counter;
2522        int count = 0;
2523        
2524        /* Amuse the user. */
2525        printk(
2526"              \\|/ ____ \\|/\n"
2527"              \"@'/ .. \\`@\"\n"
2528"              /_| \\__/ |_\\\n"
2529"                 \\__U_/\n");
2530
2531        printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter);
2532        notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2533        __asm__ __volatile__("flushw");
2534        show_regs(regs);
2535        add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
2536        if (regs->tstate & TSTATE_PRIV) {
2537                struct thread_info *tp = current_thread_info();
2538                struct reg_window *rw = (struct reg_window *)
2539                        (regs->u_regs[UREG_FP] + STACK_BIAS);
2540
2541                /* Stop the back trace when we hit userland or we
2542                 * find some badly aligned kernel stack.
2543                 */
2544                while (rw &&
2545                       count++ < 30 &&
2546                       kstack_valid(tp, (unsigned long) rw)) {
2547                        printk("Caller[%016lx]: %pS\n", rw->ins[7],
2548                               (void *) rw->ins[7]);
2549
2550                        rw = kernel_stack_up(rw);
2551                }
2552                instruction_dump ((unsigned int *) regs->tpc);
2553        } else {
2554                if (test_thread_flag(TIF_32BIT)) {
2555                        regs->tpc &= 0xffffffff;
2556                        regs->tnpc &= 0xffffffff;
2557                }
2558                user_instruction_dump ((unsigned int __user *) regs->tpc);
2559        }
2560        if (panic_on_oops)
2561                panic("Fatal exception");
2562        make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV);
2563}
2564EXPORT_SYMBOL(die_if_kernel);
2565
2566#define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19))
2567#define VIS_OPCODE_VAL  ((0x2 << 30) | (0x36 << 19))
2568
2569void do_illegal_instruction(struct pt_regs *regs)
2570{
2571        enum ctx_state prev_state = exception_enter();
2572        unsigned long pc = regs->tpc;
2573        unsigned long tstate = regs->tstate;
2574        u32 insn;
2575
2576        if (notify_die(DIE_TRAP, "illegal instruction", regs,
2577                       0, 0x10, SIGILL) == NOTIFY_STOP)
2578                goto out;
2579
2580        if (tstate & TSTATE_PRIV)
2581                die_if_kernel("Kernel illegal instruction", regs);
2582        if (test_thread_flag(TIF_32BIT))
2583                pc = (u32)pc;
2584        if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2585                if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2586                        if (handle_popc(insn, regs))
2587                                goto out;
2588                } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2589                        if (handle_ldf_stq(insn, regs))
2590                                goto out;
2591                } else if (tlb_type == hypervisor) {
2592                        if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
2593                                if (!vis_emul(regs, insn))
2594                                        goto out;
2595                        } else {
2596                                struct fpustate *f = FPUSTATE;
2597
2598                                /* On UltraSPARC T2 and later, FPU insns which
2599                                 * are not implemented in HW signal an illegal
2600                                 * instruction trap and do not set the FP Trap
2601                                 * Trap in the %fsr to unimplemented_FPop.
2602                                 */
2603                                if (do_mathemu(regs, f, true))
2604                                        goto out;
2605                        }
2606                }
2607        }
2608        force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc);
2609out:
2610        exception_exit(prev_state);
2611}
2612
2613void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2614{
2615        enum ctx_state prev_state = exception_enter();
2616
2617        if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2618                       0, 0x34, SIGSEGV) == NOTIFY_STOP)
2619                goto out;
2620
2621        if (regs->tstate & TSTATE_PRIV) {
2622                kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2623                goto out;
2624        }
2625        if (is_no_fault_exception(regs))
2626                return;
2627
2628        force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)sfar);
2629out:
2630        exception_exit(prev_state);
2631}
2632
2633void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
2634{
2635        if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2636                       0, 0x34, SIGSEGV) == NOTIFY_STOP)
2637                return;
2638
2639        if (regs->tstate & TSTATE_PRIV) {
2640                kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2641                return;
2642        }
2643        if (is_no_fault_exception(regs))
2644                return;
2645
2646        force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) addr);
2647}
2648
2649/* sun4v_mem_corrupt_detect_precise() - Handle precise exception on an ADI
2650 * tag mismatch.
2651 *
2652 * ADI version tag mismatch on a load from memory always results in a
2653 * precise exception. Tag mismatch on a store to memory will result in
2654 * precise exception if MCDPER or PMCDPER is set to 1.
2655 */
2656void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, unsigned long addr,
2657                                      unsigned long context)
2658{
2659        if (notify_die(DIE_TRAP, "memory corruption precise exception", regs,
2660                       0, 0x8, SIGSEGV) == NOTIFY_STOP)
2661                return;
2662
2663        if (regs->tstate & TSTATE_PRIV) {
2664                /* MCD exception could happen because the task was running
2665                 * a system call with MCD enabled and passed a non-versioned
2666                 * pointer or pointer with bad version tag to  the system
2667                 * call.
2668                 */
2669                const struct exception_table_entry *entry;
2670
2671                entry = search_exception_tables(regs->tpc);
2672                if (entry) {
2673                        /* Looks like a bad syscall parameter */
2674#ifdef DEBUG_EXCEPTIONS
2675                        pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n",
2676                                 regs->tpc);
2677                        pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
2678                                 regs->tpc, entry->fixup);
2679#endif
2680                        regs->tpc = entry->fixup;
2681                        regs->tnpc = regs->tpc + 4;
2682                        return;
2683                }
2684                pr_emerg("%s: ADDR[%016lx] CTX[%lx], going.\n",
2685                         __func__, addr, context);
2686                die_if_kernel("MCD precise", regs);
2687        }
2688
2689        if (test_thread_flag(TIF_32BIT)) {
2690                regs->tpc &= 0xffffffff;
2691                regs->tnpc &= 0xffffffff;
2692        }
2693        force_sig_fault(SIGSEGV, SEGV_ADIPERR, (void __user *)addr);
2694}
2695
2696void do_privop(struct pt_regs *regs)
2697{
2698        enum ctx_state prev_state = exception_enter();
2699
2700        if (notify_die(DIE_TRAP, "privileged operation", regs,
2701                       0, 0x11, SIGILL) == NOTIFY_STOP)
2702                goto out;
2703
2704        if (test_thread_flag(TIF_32BIT)) {
2705                regs->tpc &= 0xffffffff;
2706                regs->tnpc &= 0xffffffff;
2707        }
2708        force_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)regs->tpc);
2709out:
2710        exception_exit(prev_state);
2711}
2712
2713void do_privact(struct pt_regs *regs)
2714{
2715        do_privop(regs);
2716}
2717
2718/* Trap level 1 stuff or other traps we should never see... */
2719void do_cee(struct pt_regs *regs)
2720{
2721        exception_enter();
2722        die_if_kernel("TL0: Cache Error Exception", regs);
2723}
2724
2725void do_div0_tl1(struct pt_regs *regs)
2726{
2727        exception_enter();
2728        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2729        die_if_kernel("TL1: DIV0 Exception", regs);
2730}
2731
2732void do_fpieee_tl1(struct pt_regs *regs)
2733{
2734        exception_enter();
2735        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2736        die_if_kernel("TL1: FPU IEEE Exception", regs);
2737}
2738
2739void do_fpother_tl1(struct pt_regs *regs)
2740{
2741        exception_enter();
2742        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2743        die_if_kernel("TL1: FPU Other Exception", regs);
2744}
2745
2746void do_ill_tl1(struct pt_regs *regs)
2747{
2748        exception_enter();
2749        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2750        die_if_kernel("TL1: Illegal Instruction Exception", regs);
2751}
2752
2753void do_irq_tl1(struct pt_regs *regs)
2754{
2755        exception_enter();
2756        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2757        die_if_kernel("TL1: IRQ Exception", regs);
2758}
2759
2760void do_lddfmna_tl1(struct pt_regs *regs)
2761{
2762        exception_enter();
2763        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2764        die_if_kernel("TL1: LDDF Exception", regs);
2765}
2766
2767void do_stdfmna_tl1(struct pt_regs *regs)
2768{
2769        exception_enter();
2770        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2771        die_if_kernel("TL1: STDF Exception", regs);
2772}
2773
2774void do_paw(struct pt_regs *regs)
2775{
2776        exception_enter();
2777        die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2778}
2779
2780void do_paw_tl1(struct pt_regs *regs)
2781{
2782        exception_enter();
2783        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2784        die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2785}
2786
2787void do_vaw(struct pt_regs *regs)
2788{
2789        exception_enter();
2790        die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2791}
2792
2793void do_vaw_tl1(struct pt_regs *regs)
2794{
2795        exception_enter();
2796        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2797        die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2798}
2799
2800void do_tof_tl1(struct pt_regs *regs)
2801{
2802        exception_enter();
2803        dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2804        die_if_kernel("TL1: Tag Overflow Exception", regs);
2805}
2806
2807void do_getpsr(struct pt_regs *regs)
2808{
2809        regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2810        regs->tpc   = regs->tnpc;
2811        regs->tnpc += 4;
2812        if (test_thread_flag(TIF_32BIT)) {
2813                regs->tpc &= 0xffffffff;
2814                regs->tnpc &= 0xffffffff;
2815        }
2816}
2817
2818u64 cpu_mondo_counter[NR_CPUS] = {0};
2819struct trap_per_cpu trap_block[NR_CPUS];
2820EXPORT_SYMBOL(trap_block);
2821
2822/* This can get invoked before sched_init() so play it super safe
2823 * and use hard_smp_processor_id().
2824 */
2825void notrace init_cur_cpu_trap(struct thread_info *t)
2826{
2827        int cpu = hard_smp_processor_id();
2828        struct trap_per_cpu *p = &trap_block[cpu];
2829
2830        p->thread = t;
2831        p->pgd_paddr = 0;
2832}
2833
2834extern void thread_info_offsets_are_bolixed_dave(void);
2835extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2836extern void tsb_config_offsets_are_bolixed_dave(void);
2837
2838/* Only invoked on boot processor. */
2839void __init trap_init(void)
2840{
2841        /* Compile time sanity check. */
2842        BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) ||
2843                     TI_FLAGS != offsetof(struct thread_info, flags) ||
2844                     TI_CPU != offsetof(struct thread_info, cpu) ||
2845                     TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2846                     TI_KSP != offsetof(struct thread_info, ksp) ||
2847                     TI_FAULT_ADDR != offsetof(struct thread_info,
2848                                               fault_address) ||
2849                     TI_KREGS != offsetof(struct thread_info, kregs) ||
2850                     TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2851                     TI_REG_WINDOW != offsetof(struct thread_info,
2852                                               reg_window) ||
2853                     TI_RWIN_SPTRS != offsetof(struct thread_info,
2854                                               rwbuf_stkptrs) ||
2855                     TI_GSR != offsetof(struct thread_info, gsr) ||
2856                     TI_XFSR != offsetof(struct thread_info, xfsr) ||
2857                     TI_PRE_COUNT != offsetof(struct thread_info,
2858                                              preempt_count) ||
2859                     TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2860                     TI_CURRENT_DS != offsetof(struct thread_info,
2861                                                current_ds) ||
2862                     TI_KUNA_REGS != offsetof(struct thread_info,
2863                                              kern_una_regs) ||
2864                     TI_KUNA_INSN != offsetof(struct thread_info,
2865                                              kern_una_insn) ||
2866                     TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2867                     (TI_FPREGS & (64 - 1)));
2868
2869        BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu,
2870                                                     thread) ||
2871                     (TRAP_PER_CPU_PGD_PADDR !=
2872                      offsetof(struct trap_per_cpu, pgd_paddr)) ||
2873                     (TRAP_PER_CPU_CPU_MONDO_PA !=
2874                      offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2875                     (TRAP_PER_CPU_DEV_MONDO_PA !=
2876                      offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2877                     (TRAP_PER_CPU_RESUM_MONDO_PA !=
2878                      offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
2879                     (TRAP_PER_CPU_RESUM_KBUF_PA !=
2880                      offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
2881                     (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2882                      offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
2883                     (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2884                      offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
2885                     (TRAP_PER_CPU_FAULT_INFO !=
2886                      offsetof(struct trap_per_cpu, fault_info)) ||
2887                     (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2888                      offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2889                     (TRAP_PER_CPU_CPU_LIST_PA !=
2890                      offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2891                     (TRAP_PER_CPU_TSB_HUGE !=
2892                      offsetof(struct trap_per_cpu, tsb_huge)) ||
2893                     (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2894                      offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2895                     (TRAP_PER_CPU_IRQ_WORKLIST_PA !=
2896                      offsetof(struct trap_per_cpu, irq_worklist_pa)) ||
2897                     (TRAP_PER_CPU_CPU_MONDO_QMASK !=
2898                      offsetof(struct trap_per_cpu, cpu_mondo_qmask)) ||
2899                     (TRAP_PER_CPU_DEV_MONDO_QMASK !=
2900                      offsetof(struct trap_per_cpu, dev_mondo_qmask)) ||
2901                     (TRAP_PER_CPU_RESUM_QMASK !=
2902                      offsetof(struct trap_per_cpu, resum_qmask)) ||
2903                     (TRAP_PER_CPU_NONRESUM_QMASK !=
2904                      offsetof(struct trap_per_cpu, nonresum_qmask)) ||
2905                     (TRAP_PER_CPU_PER_CPU_BASE !=
2906                      offsetof(struct trap_per_cpu, __per_cpu_base)));
2907
2908        BUILD_BUG_ON((TSB_CONFIG_TSB !=
2909                      offsetof(struct tsb_config, tsb)) ||
2910                     (TSB_CONFIG_RSS_LIMIT !=
2911                      offsetof(struct tsb_config, tsb_rss_limit)) ||
2912                     (TSB_CONFIG_NENTRIES !=
2913                      offsetof(struct tsb_config, tsb_nentries)) ||
2914                     (TSB_CONFIG_REG_VAL !=
2915                      offsetof(struct tsb_config, tsb_reg_val)) ||
2916                     (TSB_CONFIG_MAP_VADDR !=
2917                      offsetof(struct tsb_config, tsb_map_vaddr)) ||
2918                     (TSB_CONFIG_MAP_PTE !=
2919                      offsetof(struct tsb_config, tsb_map_pte)));
2920
2921        /* Attach to the address space of init_task.  On SMP we
2922         * do this in smp.c:smp_callin for other cpus.
2923         */
2924        mmgrab(&init_mm);
2925        current->active_mm = &init_mm;
2926}
2927