linux/arch/powerpc/kernel/exceptions-64s.S
<<
>>
Prefs 
   1/*
   2 * This file contains the 64-bit "server" PowerPC variant
   3 * of the low level exception handling including exception
   4 * vectors, exception return, part of the slb and stab
   5 * handling and other fixed offset specific things.
   6 *
   7 * This file is meant to be #included from head_64.S due to
   8 * position dependent assembly.
   9 *
  10 * Most of this originates from head_64.S and thus has the same
  11 * copyright history.
  12 *
  13 */
  14
  15#include <asm/hw_irq.h>
  16#include <asm/exception-64s.h>
  17#include <asm/ptrace.h>
  18
  19/*
  20 * We layout physical memory as follows:
  21 * 0x0000 - 0x00ff : Secondary processor spin code
  22 * 0x0100 - 0x17ff : pSeries Interrupt prologs
  23 * 0x1800 - 0x4000 : interrupt support common interrupt prologs
  24 * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
  25 * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
  26 * 0x7000 - 0x7fff : FWNMI data area
  27 * 0x8000 - 0x8fff : Initial (CPU0) segment table
  28 * 0x9000 -        : Early init and support code
  29 */
  30        /* Syscall routine is used twice, in reloc-off and reloc-on paths */
  31#define SYSCALL_PSERIES_1                                       \
  32BEGIN_FTR_SECTION                                               \
  33        cmpdi   r0,0x1ebe ;                                     \
  34        beq-    1f ;                                            \
  35END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)                          \
  36        mr      r9,r13 ;                                        \
  37        GET_PACA(r13) ;                                         \
  38        mfspr   r11,SPRN_SRR0 ;                                 \
  390:
  40
  41#define SYSCALL_PSERIES_2_RFID                                  \
  42        mfspr   r12,SPRN_SRR1 ;                                 \
  43        ld      r10,PACAKBASE(r13) ;                            \
  44        LOAD_HANDLER(r10, system_call_entry) ;                  \
  45        mtspr   SPRN_SRR0,r10 ;                                 \
  46        ld      r10,PACAKMSR(r13) ;                             \
  47        mtspr   SPRN_SRR1,r10 ;                                 \
  48        rfid ;                                                  \
  49        b       . ;     /* prevent speculative execution */
  50
  51#define SYSCALL_PSERIES_3                                       \
  52        /* Fast LE/BE switch system call */                     \
  531:      mfspr   r12,SPRN_SRR1 ;                                 \
  54        xori    r12,r12,MSR_LE ;                                \
  55        mtspr   SPRN_SRR1,r12 ;                                 \
  56        rfid ;          /* return to userspace */               \
  57        b       . ;                                             \
  582:      mfspr   r12,SPRN_SRR1 ;                                 \
  59        andi.   r12,r12,MSR_PR ;                                \
  60        bne     0b ;                                            \
  61        mtspr   SPRN_SRR0,r3 ;                                  \
  62        mtspr   SPRN_SRR1,r4 ;                                  \
  63        mtspr   SPRN_SDR1,r5 ;                                  \
  64        rfid ;                                                  \
  65        b       . ;     /* prevent speculative execution */
  66
  67#if defined(CONFIG_RELOCATABLE)
  68        /*
  69         * We can't branch directly; in the direct case we use LR
  70         * and system_call_entry restores LR.  (We thus need to move
  71         * LR to r10 in the RFID case too.)
  72         */
  73#define SYSCALL_PSERIES_2_DIRECT                                \
  74        mflr    r10 ;                                           \
  75        ld      r12,PACAKBASE(r13) ;                            \
  76        LOAD_HANDLER(r12, system_call_entry_direct) ;           \
  77        mtctr   r12 ;                                           \
  78        mfspr   r12,SPRN_SRR1 ;                                 \
  79        /* Re-use of r13... No spare regs to do this */ \
  80        li      r13,MSR_RI ;                                    \
  81        mtmsrd  r13,1 ;                                         \
  82        GET_PACA(r13) ; /* get r13 back */                      \
  83        bctr ;
  84#else
  85        /* We can branch directly */
  86#define SYSCALL_PSERIES_2_DIRECT                                \
  87        mfspr   r12,SPRN_SRR1 ;                                 \
  88        li      r10,MSR_RI ;                                    \
  89        mtmsrd  r10,1 ;                 /* Set RI (EE=0) */     \
  90        b       system_call_entry_direct ;
  91#endif
  92
  93/*
  94 * This is the start of the interrupt handlers for pSeries
  95 * This code runs with relocation off.
  96 * Code from here to __end_interrupts gets copied down to real
  97 * address 0x100 when we are running a relocatable kernel.
  98 * Therefore any relative branches in this section must only
  99 * branch to labels in this section.
 100 */
 101        . = 0x100
 102        .globl __start_interrupts
 103__start_interrupts:
 104
 105        .globl system_reset_pSeries;
 106system_reset_pSeries:
 107        HMT_MEDIUM_PPR_DISCARD
 108        SET_SCRATCH0(r13)
 109#ifdef CONFIG_PPC_P7_NAP
 110BEGIN_FTR_SECTION
 111        /* Running native on arch 2.06 or later, check if we are
 112         * waking up from nap. We only handle no state loss and
 113         * supervisor state loss. We do -not- handle hypervisor
 114         * state loss at this time.
 115         */
 116        mfspr   r13,SPRN_SRR1
 117        rlwinm. r13,r13,47-31,30,31
 118        beq     9f
 119
 120        /* waking up from powersave (nap) state */
 121        cmpwi   cr1,r13,2
 122        /* Total loss of HV state is fatal, we could try to use the
 123         * PIR to locate a PACA, then use an emergency stack etc...
 124         * but for now, let's just stay stuck here
 125         */
 126        bgt     cr1,.
 127        GET_PACA(r13)
 128
 129#ifdef CONFIG_KVM_BOOK3S_64_HV
 130        li      r0,KVM_HWTHREAD_IN_KERNEL
 131        stb     r0,HSTATE_HWTHREAD_STATE(r13)
 132        /* Order setting hwthread_state vs. testing hwthread_req */
 133        sync
 134        lbz     r0,HSTATE_HWTHREAD_REQ(r13)
 135        cmpwi   r0,0
 136        beq     1f
 137        b       kvm_start_guest
 1381:
 139#endif
 140
 141        beq     cr1,2f
 142        b       .power7_wakeup_noloss
 1432:      b       .power7_wakeup_loss
 1449:
 145END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 146#endif /* CONFIG_PPC_P7_NAP */
 147        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
 148                                 NOTEST, 0x100)
 149
 150        . = 0x200
 151machine_check_pSeries_1:
 152        /* This is moved out of line as it can be patched by FW, but
 153         * some code path might still want to branch into the original
 154         * vector
 155         */
 156        HMT_MEDIUM_PPR_DISCARD
 157        SET_SCRATCH0(r13)               /* save r13 */
 158        EXCEPTION_PROLOG_0(PACA_EXMC)
 159        b       machine_check_pSeries_0
 160
 161        . = 0x300
 162        .globl data_access_pSeries
 163data_access_pSeries:
 164        HMT_MEDIUM_PPR_DISCARD
 165        SET_SCRATCH0(r13)
 166BEGIN_FTR_SECTION
 167        b       data_access_check_stab
 168data_access_not_stab:
 169END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
 170        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
 171                                 KVMTEST, 0x300)
 172
 173        . = 0x380
 174        .globl data_access_slb_pSeries
 175data_access_slb_pSeries:
 176        HMT_MEDIUM_PPR_DISCARD
 177        SET_SCRATCH0(r13)
 178        EXCEPTION_PROLOG_0(PACA_EXSLB)
 179        EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
 180        std     r3,PACA_EXSLB+EX_R3(r13)
 181        mfspr   r3,SPRN_DAR
 182#ifdef __DISABLED__
 183        /* Keep that around for when we re-implement dynamic VSIDs */
 184        cmpdi   r3,0
 185        bge     slb_miss_user_pseries
 186#endif /* __DISABLED__ */
 187        mfspr   r12,SPRN_SRR1
 188#ifndef CONFIG_RELOCATABLE
 189        b       .slb_miss_realmode
 190#else
 191        /*
 192         * We can't just use a direct branch to .slb_miss_realmode
 193         * because the distance from here to there depends on where
 194         * the kernel ends up being put.
 195         */
 196        mfctr   r11
 197        ld      r10,PACAKBASE(r13)
 198        LOAD_HANDLER(r10, .slb_miss_realmode)
 199        mtctr   r10
 200        bctr
 201#endif
 202
 203        STD_EXCEPTION_PSERIES(0x400, 0x400, instruction_access)
 204
 205        . = 0x480
 206        .globl instruction_access_slb_pSeries
 207instruction_access_slb_pSeries:
 208        HMT_MEDIUM_PPR_DISCARD
 209        SET_SCRATCH0(r13)
 210        EXCEPTION_PROLOG_0(PACA_EXSLB)
 211        EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
 212        std     r3,PACA_EXSLB+EX_R3(r13)
 213        mfspr   r3,SPRN_SRR0            /* SRR0 is faulting address */
 214#ifdef __DISABLED__
 215        /* Keep that around for when we re-implement dynamic VSIDs */
 216        cmpdi   r3,0
 217        bge     slb_miss_user_pseries
 218#endif /* __DISABLED__ */
 219        mfspr   r12,SPRN_SRR1
 220#ifndef CONFIG_RELOCATABLE
 221        b       .slb_miss_realmode
 222#else
 223        mfctr   r11
 224        ld      r10,PACAKBASE(r13)
 225        LOAD_HANDLER(r10, .slb_miss_realmode)
 226        mtctr   r10
 227        bctr
 228#endif
 229
 230        /* We open code these as we can't have a ". = x" (even with
 231         * x = "." within a feature section
 232         */
 233        . = 0x500;
 234        .globl hardware_interrupt_pSeries;
 235        .globl hardware_interrupt_hv;
 236hardware_interrupt_pSeries:
 237hardware_interrupt_hv:
 238        HMT_MEDIUM_PPR_DISCARD
 239        BEGIN_FTR_SECTION
 240                _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
 241                                            EXC_HV, SOFTEN_TEST_HV)
 242                KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
 243        FTR_SECTION_ELSE
 244                _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
 245                                            EXC_STD, SOFTEN_TEST_HV_201)
 246                KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
 247        ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 248
 249        STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
 250        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x600)
 251
 252        STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
 253        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x700)
 254
 255        STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
 256        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
 257
 258        . = 0x900
 259        .globl decrementer_pSeries
 260decrementer_pSeries:
 261        _MASKABLE_EXCEPTION_PSERIES(0x900, decrementer, EXC_STD, SOFTEN_TEST_PR)
 262
 263        STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
 264
 265        MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
 266        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
 267
 268        STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
 269        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xb00)
 270
 271        . = 0xc00
 272        .globl  system_call_pSeries
 273system_call_pSeries:
 274        HMT_MEDIUM
 275#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 276        SET_SCRATCH0(r13)
 277        GET_PACA(r13)
 278        std     r9,PACA_EXGEN+EX_R9(r13)
 279        std     r10,PACA_EXGEN+EX_R10(r13)
 280        mfcr    r9
 281        KVMTEST(0xc00)
 282        GET_SCRATCH0(r13)
 283#endif
 284        SYSCALL_PSERIES_1
 285        SYSCALL_PSERIES_2_RFID
 286        SYSCALL_PSERIES_3
 287        KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
 288
 289        STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
 290        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xd00)
 291
 292        /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
 293         * out of line to handle them
 294         */
 295        . = 0xe00
 296hv_data_storage_trampoline:
 297        SET_SCRATCH0(r13)
 298        EXCEPTION_PROLOG_0(PACA_EXGEN)
 299        b       h_data_storage_hv
 300
 301        . = 0xe20
 302hv_instr_storage_trampoline:
 303        SET_SCRATCH0(r13)
 304        EXCEPTION_PROLOG_0(PACA_EXGEN)
 305        b       h_instr_storage_hv
 306
 307        . = 0xe40
 308emulation_assist_trampoline:
 309        SET_SCRATCH0(r13)
 310        EXCEPTION_PROLOG_0(PACA_EXGEN)
 311        b       emulation_assist_hv
 312
 313        . = 0xe60
 314hv_exception_trampoline:
 315        SET_SCRATCH0(r13)
 316        EXCEPTION_PROLOG_0(PACA_EXGEN)
 317        b       hmi_exception_hv
 318
 319        . = 0xe80
 320hv_doorbell_trampoline:
 321        SET_SCRATCH0(r13)
 322        EXCEPTION_PROLOG_0(PACA_EXGEN)
 323        b       h_doorbell_hv
 324
 325        /* We need to deal with the Altivec unavailable exception
 326         * here which is at 0xf20, thus in the middle of the
 327         * prolog code of the PerformanceMonitor one. A little
 328         * trickery is thus necessary
 329         */
 330        . = 0xf00
 331performance_monitor_pseries_trampoline:
 332        SET_SCRATCH0(r13)
 333        EXCEPTION_PROLOG_0(PACA_EXGEN)
 334        b       performance_monitor_pSeries
 335
 336        . = 0xf20
 337altivec_unavailable_pseries_trampoline:
 338        SET_SCRATCH0(r13)
 339        EXCEPTION_PROLOG_0(PACA_EXGEN)
 340        b       altivec_unavailable_pSeries
 341
 342        . = 0xf40
 343vsx_unavailable_pseries_trampoline:
 344        SET_SCRATCH0(r13)
 345        EXCEPTION_PROLOG_0(PACA_EXGEN)
 346        b       vsx_unavailable_pSeries
 347
 348        . = 0xf60
 349facility_unavailable_trampoline:
 350        SET_SCRATCH0(r13)
 351        EXCEPTION_PROLOG_0(PACA_EXGEN)
 352        b       facility_unavailable_pSeries
 353
 354        . = 0xf80
 355hv_facility_unavailable_trampoline:
 356        SET_SCRATCH0(r13)
 357        EXCEPTION_PROLOG_0(PACA_EXGEN)
 358        b       facility_unavailable_hv
 359
 360#ifdef CONFIG_CBE_RAS
 361        STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
 362        KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
 363#endif /* CONFIG_CBE_RAS */
 364
 365        STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
 366        KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
 367
 368        . = 0x1500
 369        .global denorm_exception_hv
 370denorm_exception_hv:
 371        HMT_MEDIUM_PPR_DISCARD
 372        mtspr   SPRN_SPRG_HSCRATCH0,r13
 373        EXCEPTION_PROLOG_0(PACA_EXGEN)
 374        EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500)
 375
 376#ifdef CONFIG_PPC_DENORMALISATION
 377        mfspr   r10,SPRN_HSRR1
 378        mfspr   r11,SPRN_HSRR0          /* save HSRR0 */
 379        andis.  r10,r10,(HSRR1_DENORM)@h /* denorm? */
 380        addi    r11,r11,-4              /* HSRR0 is next instruction */
 381        bne+    denorm_assist
 382#endif
 383
 384        KVMTEST(0x1500)
 385        EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
 386        KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)
 387
 388#ifdef CONFIG_CBE_RAS
 389        STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
 390        KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
 391#endif /* CONFIG_CBE_RAS */
 392
 393        STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
 394        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x1700)
 395
 396#ifdef CONFIG_CBE_RAS
 397        STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
 398        KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
 399#else
 400        . = 0x1800
 401#endif /* CONFIG_CBE_RAS */
 402
 403
 404/*** Out of line interrupts support ***/
 405
 406        .align  7
 407        /* moved from 0x200 */
 408machine_check_pSeries:
 409        .globl machine_check_fwnmi
 410machine_check_fwnmi:
 411        HMT_MEDIUM_PPR_DISCARD
 412        SET_SCRATCH0(r13)               /* save r13 */
 413        EXCEPTION_PROLOG_0(PACA_EXMC)
 414machine_check_pSeries_0:
 415        EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST, 0x200)
 416        EXCEPTION_PROLOG_PSERIES_1(machine_check_common, EXC_STD)
 417        KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
 418
 419        /* moved from 0x300 */
 420data_access_check_stab:
 421        GET_PACA(r13)
 422        std     r9,PACA_EXSLB+EX_R9(r13)
 423        std     r10,PACA_EXSLB+EX_R10(r13)
 424        mfspr   r10,SPRN_DAR
 425        mfspr   r9,SPRN_DSISR
 426        srdi    r10,r10,60
 427        rlwimi  r10,r9,16,0x20
 428#ifdef CONFIG_KVM_BOOK3S_PR
 429        lbz     r9,HSTATE_IN_GUEST(r13)
 430        rlwimi  r10,r9,8,0x300
 431#endif
 432        mfcr    r9
 433        cmpwi   r10,0x2c
 434        beq     do_stab_bolted_pSeries
 435        mtcrf   0x80,r9
 436        ld      r9,PACA_EXSLB+EX_R9(r13)
 437        ld      r10,PACA_EXSLB+EX_R10(r13)
 438        b       data_access_not_stab
 439do_stab_bolted_pSeries:
 440        std     r11,PACA_EXSLB+EX_R11(r13)
 441        std     r12,PACA_EXSLB+EX_R12(r13)
 442        GET_SCRATCH0(r10)
 443        std     r10,PACA_EXSLB+EX_R13(r13)
 444        EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
 445
 446        KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
 447        KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
 448        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x400)
 449        KVM_HANDLER_PR(PACA_EXSLB, EXC_STD, 0x480)
 450        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
 451        KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
 452
 453#ifdef CONFIG_PPC_DENORMALISATION
 454denorm_assist:
 455BEGIN_FTR_SECTION
 456/*
 457 * To denormalise we need to move a copy of the register to itself.
 458 * For POWER6 do that here for all FP regs.
 459 */
 460        mfmsr   r10
 461        ori     r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
 462        xori    r10,r10,(MSR_FE0|MSR_FE1)
 463        mtmsrd  r10
 464        sync
 465
 466#define FMR2(n)  fmr (n), (n) ; fmr n+1, n+1
 467#define FMR4(n)  FMR2(n) ; FMR2(n+2)
 468#define FMR8(n)  FMR4(n) ; FMR4(n+4)
 469#define FMR16(n) FMR8(n) ; FMR8(n+8)
 470#define FMR32(n) FMR16(n) ; FMR16(n+16)
 471        FMR32(0)
 472
 473FTR_SECTION_ELSE
 474/*
 475 * To denormalise we need to move a copy of the register to itself.
 476 * For POWER7 do that here for the first 32 VSX registers only.
 477 */
 478        mfmsr   r10
 479        oris    r10,r10,MSR_VSX@h
 480        mtmsrd  r10
 481        sync
 482
 483#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
 484#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
 485#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
 486#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
 487#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
 488        XVCPSGNDP32(0)
 489
 490ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
 491
 492BEGIN_FTR_SECTION
 493        b       denorm_done
 494END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
 495/*
 496 * To denormalise we need to move a copy of the register to itself.
 497 * For POWER8 we need to do that for all 64 VSX registers
 498 */
 499        XVCPSGNDP32(32)
 500denorm_done:
 501        mtspr   SPRN_HSRR0,r11
 502        mtcrf   0x80,r9
 503        ld      r9,PACA_EXGEN+EX_R9(r13)
 504        RESTORE_PPR_PACA(PACA_EXGEN, r10)
 505BEGIN_FTR_SECTION
 506        ld      r10,PACA_EXGEN+EX_CFAR(r13)
 507        mtspr   SPRN_CFAR,r10
 508END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 509        ld      r10,PACA_EXGEN+EX_R10(r13)
 510        ld      r11,PACA_EXGEN+EX_R11(r13)
 511        ld      r12,PACA_EXGEN+EX_R12(r13)
 512        ld      r13,PACA_EXGEN+EX_R13(r13)
 513        HRFID
 514        b       .
 515#endif
 516
 517        .align  7
 518        /* moved from 0xe00 */
 519        STD_EXCEPTION_HV_OOL(0xe02, h_data_storage)
 520        KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
 521        STD_EXCEPTION_HV_OOL(0xe22, h_instr_storage)
 522        KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
 523        STD_EXCEPTION_HV_OOL(0xe42, emulation_assist)
 524        KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
 525        STD_EXCEPTION_HV_OOL(0xe62, hmi_exception) /* need to flush cache ? */
 526        KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
 527        MASKABLE_EXCEPTION_HV_OOL(0xe82, h_doorbell)
 528        KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe82)
 529
 530        /* moved from 0xf00 */
 531        STD_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
 532        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf00)
 533        STD_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
 534        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf20)
 535        STD_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
 536        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
 537        STD_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
 538        KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf60)
 539        STD_EXCEPTION_HV_OOL(0xf82, facility_unavailable)
 540        KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xf82)
 541
 542/*
 543 * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
 544 * - If it was a decrementer interrupt, we bump the dec to max and and return.
 545 * - If it was a doorbell we return immediately since doorbells are edge
 546 *   triggered and won't automatically refire.
 547 * - else we hard disable and return.
 548 * This is called with r10 containing the value to OR to the paca field.
 549 */
 550#define MASKED_INTERRUPT(_H)                            \
 551masked_##_H##interrupt:                                 \
 552        std     r11,PACA_EXGEN+EX_R11(r13);             \
 553        lbz     r11,PACAIRQHAPPENED(r13);               \
 554        or      r11,r11,r10;                            \
 555        stb     r11,PACAIRQHAPPENED(r13);               \
 556        cmpwi   r10,PACA_IRQ_DEC;                       \
 557        bne     1f;                                     \
 558        lis     r10,0x7fff;                             \
 559        ori     r10,r10,0xffff;                         \
 560        mtspr   SPRN_DEC,r10;                           \
 561        b       2f;                                     \
 5621:      cmpwi   r10,PACA_IRQ_DBELL;                     \
 563        beq     2f;                                     \
 564        mfspr   r10,SPRN_##_H##SRR1;                    \
 565        rldicl  r10,r10,48,1; /* clear MSR_EE */        \
 566        rotldi  r10,r10,16;                             \
 567        mtspr   SPRN_##_H##SRR1,r10;                    \
 5682:      mtcrf   0x80,r9;                                \
 569        ld      r9,PACA_EXGEN+EX_R9(r13);               \
 570        ld      r10,PACA_EXGEN+EX_R10(r13);             \
 571        ld      r11,PACA_EXGEN+EX_R11(r13);             \
 572        GET_SCRATCH0(r13);                              \
 573        ##_H##rfid;                                     \
 574        b       .
 575        
 576        MASKED_INTERRUPT()
 577        MASKED_INTERRUPT(H)
 578
 579/*
 580 * Called from arch_local_irq_enable when an interrupt needs
 581 * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
 582 * which kind of interrupt. MSR:EE is already off. We generate a
 583 * stackframe like if a real interrupt had happened.
 584 *
 585 * Note: While MSR:EE is off, we need to make sure that _MSR
 586 * in the generated frame has EE set to 1 or the exception
 587 * handler will not properly re-enable them.
 588 */
 589_GLOBAL(__replay_interrupt)
 590        /* We are going to jump to the exception common code which
 591         * will retrieve various register values from the PACA which
 592         * we don't give a damn about, so we don't bother storing them.
 593         */
 594        mfmsr   r12
 595        mflr    r11
 596        mfcr    r9
 597        ori     r12,r12,MSR_EE
 598        cmpwi   r3,0x900
 599        beq     decrementer_common
 600        cmpwi   r3,0x500
 601        beq     hardware_interrupt_common
 602BEGIN_FTR_SECTION
 603        cmpwi   r3,0xe80
 604        beq     h_doorbell_common
 605FTR_SECTION_ELSE
 606        cmpwi   r3,0xa00
 607        beq     doorbell_super_common
 608ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
 609        blr
 610
 611#ifdef CONFIG_PPC_PSERIES
 612/*
 613 * Vectors for the FWNMI option.  Share common code.
 614 */
 615        .globl system_reset_fwnmi
 616      .align 7
 617system_reset_fwnmi:
 618        HMT_MEDIUM_PPR_DISCARD
 619        SET_SCRATCH0(r13)               /* save r13 */
 620        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
 621                                 NOTEST, 0x100)
 622
 623#endif /* CONFIG_PPC_PSERIES */
 624
 625#ifdef __DISABLED__
 626/*
 627 * This is used for when the SLB miss handler has to go virtual,
 628 * which doesn't happen for now anymore but will once we re-implement
 629 * dynamic VSIDs for shared page tables
 630 */
 631slb_miss_user_pseries:
 632        std     r10,PACA_EXGEN+EX_R10(r13)
 633        std     r11,PACA_EXGEN+EX_R11(r13)
 634        std     r12,PACA_EXGEN+EX_R12(r13)
 635        GET_SCRATCH0(r10)
 636        ld      r11,PACA_EXSLB+EX_R9(r13)
 637        ld      r12,PACA_EXSLB+EX_R3(r13)
 638        std     r10,PACA_EXGEN+EX_R13(r13)
 639        std     r11,PACA_EXGEN+EX_R9(r13)
 640        std     r12,PACA_EXGEN+EX_R3(r13)
 641        clrrdi  r12,r13,32
 642        mfmsr   r10
 643        mfspr   r11,SRR0                        /* save SRR0 */
 644        ori     r12,r12,slb_miss_user_common@l  /* virt addr of handler */
 645        ori     r10,r10,MSR_IR|MSR_DR|MSR_RI
 646        mtspr   SRR0,r12
 647        mfspr   r12,SRR1                        /* and SRR1 */
 648        mtspr   SRR1,r10
 649        rfid
 650        b       .                               /* prevent spec. execution */
 651#endif /* __DISABLED__ */
 652
 653/*
 654 * Code from here down to __end_handlers is invoked from the
 655 * exception prologs above.  Because the prologs assemble the
 656 * addresses of these handlers using the LOAD_HANDLER macro,
 657 * which uses an ori instruction, these handlers must be in
 658 * the first 64k of the kernel image.
 659 */
 660
 661/*** Common interrupt handlers ***/
 662
 663        STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
 664
 665        /*
 666         * Machine check is different because we use a different
 667         * save area: PACA_EXMC instead of PACA_EXGEN.
 668         */
 669        .align  7
 670        .globl machine_check_common
 671machine_check_common:
 672
 673        mfspr   r10,SPRN_DAR
 674        std     r10,PACA_EXGEN+EX_DAR(r13)
 675        mfspr   r10,SPRN_DSISR
 676        stw     r10,PACA_EXGEN+EX_DSISR(r13)
 677        EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
 678        FINISH_NAP
 679        DISABLE_INTS
 680        ld      r3,PACA_EXGEN+EX_DAR(r13)
 681        lwz     r4,PACA_EXGEN+EX_DSISR(r13)
 682        std     r3,_DAR(r1)
 683        std     r4,_DSISR(r1)
 684        bl      .save_nvgprs
 685        addi    r3,r1,STACK_FRAME_OVERHEAD
 686        bl      .machine_check_exception
 687        b       .ret_from_except
 688
 689        STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
 690        STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, .timer_interrupt)
 691        STD_EXCEPTION_COMMON(0x980, hdecrementer, .hdec_interrupt)
 692#ifdef CONFIG_PPC_DOORBELL
 693        STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, .doorbell_exception)
 694#else
 695        STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, .unknown_exception)
 696#endif
 697        STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
 698        STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
 699        STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
 700        STD_EXCEPTION_COMMON(0xe40, emulation_assist, .emulation_assist_interrupt)
 701        STD_EXCEPTION_COMMON(0xe60, hmi_exception, .unknown_exception)
 702#ifdef CONFIG_PPC_DOORBELL
 703        STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, .doorbell_exception)
 704#else
 705        STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, .unknown_exception)
 706#endif
 707        STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, .performance_monitor_exception)
 708        STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
 709        STD_EXCEPTION_COMMON(0x1502, denorm, .unknown_exception)
 710#ifdef CONFIG_ALTIVEC
 711        STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
 712#else
 713        STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
 714#endif
 715#ifdef CONFIG_CBE_RAS
 716        STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
 717        STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
 718        STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
 719#endif /* CONFIG_CBE_RAS */
 720
 721        /*
 722         * Relocation-on interrupts: A subset of the interrupts can be delivered
 723         * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
 724         * it.  Addresses are the same as the original interrupt addresses, but
 725         * offset by 0xc000000000004000.
 726         * It's impossible to receive interrupts below 0x300 via this mechanism.
 727         * KVM: None of these traps are from the guest ; anything that escalated
 728         * to HV=1 from HV=0 is delivered via real mode handlers.
 729         */
 730
 731        /*
 732         * This uses the standard macro, since the original 0x300 vector
 733         * only has extra guff for STAB-based processors -- which never
 734         * come here.
 735         */
 736        STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
 737        . = 0x4380
 738        .globl data_access_slb_relon_pSeries
 739data_access_slb_relon_pSeries:
 740        SET_SCRATCH0(r13)
 741        EXCEPTION_PROLOG_0(PACA_EXSLB)
 742        EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
 743        std     r3,PACA_EXSLB+EX_R3(r13)
 744        mfspr   r3,SPRN_DAR
 745        mfspr   r12,SPRN_SRR1
 746#ifndef CONFIG_RELOCATABLE
 747        b       .slb_miss_realmode
 748#else
 749        /*
 750         * We can't just use a direct branch to .slb_miss_realmode
 751         * because the distance from here to there depends on where
 752         * the kernel ends up being put.
 753         */
 754        mfctr   r11
 755        ld      r10,PACAKBASE(r13)
 756        LOAD_HANDLER(r10, .slb_miss_realmode)
 757        mtctr   r10
 758        bctr
 759#endif
 760
 761        STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
 762        . = 0x4480
 763        .globl instruction_access_slb_relon_pSeries
 764instruction_access_slb_relon_pSeries:
 765        SET_SCRATCH0(r13)
 766        EXCEPTION_PROLOG_0(PACA_EXSLB)
 767        EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
 768        std     r3,PACA_EXSLB+EX_R3(r13)
 769        mfspr   r3,SPRN_SRR0            /* SRR0 is faulting address */
 770        mfspr   r12,SPRN_SRR1
 771#ifndef CONFIG_RELOCATABLE
 772        b       .slb_miss_realmode
 773#else
 774        mfctr   r11
 775        ld      r10,PACAKBASE(r13)
 776        LOAD_HANDLER(r10, .slb_miss_realmode)
 777        mtctr   r10
 778        bctr
 779#endif
 780
 781        . = 0x4500
 782        .globl hardware_interrupt_relon_pSeries;
 783        .globl hardware_interrupt_relon_hv;
 784hardware_interrupt_relon_pSeries:
 785hardware_interrupt_relon_hv:
 786        BEGIN_FTR_SECTION
 787                _MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
 788        FTR_SECTION_ELSE
 789                _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
 790        ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
 791        STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
 792        STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
 793        STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
 794        MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
 795        STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
 796        MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
 797        STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
 798
 799        . = 0x4c00
 800        .globl system_call_relon_pSeries
 801system_call_relon_pSeries:
 802        HMT_MEDIUM
 803        SYSCALL_PSERIES_1
 804        SYSCALL_PSERIES_2_DIRECT
 805        SYSCALL_PSERIES_3
 806
 807        STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)
 808
 809        . = 0x4e00
 810        b       .       /* Can't happen, see v2.07 Book III-S section 6.5 */
 811
 812        . = 0x4e20
 813        b       .       /* Can't happen, see v2.07 Book III-S section 6.5 */
 814
 815        . = 0x4e40
 816emulation_assist_relon_trampoline:
 817        SET_SCRATCH0(r13)
 818        EXCEPTION_PROLOG_0(PACA_EXGEN)
 819        b       emulation_assist_relon_hv
 820
 821        . = 0x4e60
 822        b       .       /* Can't happen, see v2.07 Book III-S section 6.5 */
 823
 824        . = 0x4e80
 825h_doorbell_relon_trampoline:
 826        SET_SCRATCH0(r13)
 827        EXCEPTION_PROLOG_0(PACA_EXGEN)
 828        b       h_doorbell_relon_hv
 829
 830        . = 0x4f00
 831performance_monitor_relon_pseries_trampoline:
 832        SET_SCRATCH0(r13)
 833        EXCEPTION_PROLOG_0(PACA_EXGEN)
 834        b       performance_monitor_relon_pSeries
 835
 836        . = 0x4f20
 837altivec_unavailable_relon_pseries_trampoline:
 838        SET_SCRATCH0(r13)
 839        EXCEPTION_PROLOG_0(PACA_EXGEN)
 840        b       altivec_unavailable_relon_pSeries
 841
 842        . = 0x4f40
 843vsx_unavailable_relon_pseries_trampoline:
 844        SET_SCRATCH0(r13)
 845        EXCEPTION_PROLOG_0(PACA_EXGEN)
 846        b       vsx_unavailable_relon_pSeries
 847
 848        . = 0x4f60
 849facility_unavailable_relon_trampoline:
 850        SET_SCRATCH0(r13)
 851        EXCEPTION_PROLOG_0(PACA_EXGEN)
 852        b       facility_unavailable_relon_pSeries
 853
 854        . = 0x4f80
 855hv_facility_unavailable_relon_trampoline:
 856        SET_SCRATCH0(r13)
 857        EXCEPTION_PROLOG_0(PACA_EXGEN)
 858        b       hv_facility_unavailable_relon_hv
 859
 860        STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
 861#ifdef CONFIG_PPC_DENORMALISATION
 862        . = 0x5500
 863        b       denorm_exception_hv
 864#endif
 865        STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)
 866
 867        /* Other future vectors */
 868        .align  7
 869        .globl  __end_interrupts
 870__end_interrupts:
 871
 872        .align  7
 873system_call_entry_direct:
 874#if defined(CONFIG_RELOCATABLE)
 875        /* The first level prologue may have used LR to get here, saving
 876         * orig in r10.  To save hacking/ifdeffing common code, restore here.
 877         */
 878        mtlr    r10
 879#endif
 880system_call_entry:
 881        b       system_call_common
 882
 883ppc64_runlatch_on_trampoline:
 884        b       .__ppc64_runlatch_on
 885
 886/*
 887 * Here we have detected that the kernel stack pointer is bad.
 888 * R9 contains the saved CR, r13 points to the paca,
 889 * r10 contains the (bad) kernel stack pointer,
 890 * r11 and r12 contain the saved SRR0 and SRR1.
 891 * We switch to using an emergency stack, save the registers there,
 892 * and call kernel_bad_stack(), which panics.
 893 */
 894bad_stack:
 895        ld      r1,PACAEMERGSP(r13)
 896        subi    r1,r1,64+INT_FRAME_SIZE
 897        std     r9,_CCR(r1)
 898        std     r10,GPR1(r1)
 899        std     r11,_NIP(r1)
 900        std     r12,_MSR(r1)
 901        mfspr   r11,SPRN_DAR
 902        mfspr   r12,SPRN_DSISR
 903        std     r11,_DAR(r1)
 904        std     r12,_DSISR(r1)
 905        mflr    r10
 906        mfctr   r11
 907        mfxer   r12
 908        std     r10,_LINK(r1)
 909        std     r11,_CTR(r1)
 910        std     r12,_XER(r1)
 911        SAVE_GPR(0,r1)
 912        SAVE_GPR(2,r1)
 913        ld      r10,EX_R3(r3)
 914        std     r10,GPR3(r1)
 915        SAVE_GPR(4,r1)
 916        SAVE_4GPRS(5,r1)
 917        ld      r9,EX_R9(r3)
 918        ld      r10,EX_R10(r3)
 919        SAVE_2GPRS(9,r1)
 920        ld      r9,EX_R11(r3)
 921        ld      r10,EX_R12(r3)
 922        ld      r11,EX_R13(r3)
 923        std     r9,GPR11(r1)
 924        std     r10,GPR12(r1)
 925        std     r11,GPR13(r1)
 926BEGIN_FTR_SECTION
 927        ld      r10,EX_CFAR(r3)
 928        std     r10,ORIG_GPR3(r1)
 929END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 930        SAVE_8GPRS(14,r1)
 931        SAVE_10GPRS(22,r1)
 932        lhz     r12,PACA_TRAP_SAVE(r13)
 933        std     r12,_TRAP(r1)
 934        addi    r11,r1,INT_FRAME_SIZE
 935        std     r11,0(r1)
 936        li      r12,0
 937        std     r12,0(r11)
 938        ld      r2,PACATOC(r13)
 939        ld      r11,exception_marker@toc(r2)
 940        std     r12,RESULT(r1)
 941        std     r11,STACK_FRAME_OVERHEAD-16(r1)
 9421:      addi    r3,r1,STACK_FRAME_OVERHEAD
 943        bl      .kernel_bad_stack
 944        b       1b
 945
 946/*
 947 * Here r13 points to the paca, r9 contains the saved CR,
 948 * SRR0 and SRR1 are saved in r11 and r12,
 949 * r9 - r13 are saved in paca->exgen.
 950 */
 951        .align  7
 952        .globl data_access_common
 953data_access_common:
 954        mfspr   r10,SPRN_DAR
 955        std     r10,PACA_EXGEN+EX_DAR(r13)
 956        mfspr   r10,SPRN_DSISR
 957        stw     r10,PACA_EXGEN+EX_DSISR(r13)
 958        EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
 959        DISABLE_INTS
 960        ld      r12,_MSR(r1)
 961        ld      r3,PACA_EXGEN+EX_DAR(r13)
 962        lwz     r4,PACA_EXGEN+EX_DSISR(r13)
 963        li      r5,0x300
 964        b       .do_hash_page           /* Try to handle as hpte fault */
 965
 966        .align  7
 967        .globl  h_data_storage_common
 968h_data_storage_common:
 969        mfspr   r10,SPRN_HDAR
 970        std     r10,PACA_EXGEN+EX_DAR(r13)
 971        mfspr   r10,SPRN_HDSISR
 972        stw     r10,PACA_EXGEN+EX_DSISR(r13)
 973        EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
 974        bl      .save_nvgprs
 975        DISABLE_INTS
 976        addi    r3,r1,STACK_FRAME_OVERHEAD
 977        bl      .unknown_exception
 978        b       .ret_from_except
 979
 980        .align  7
 981        .globl instruction_access_common
 982instruction_access_common:
 983        EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
 984        DISABLE_INTS
 985        ld      r12,_MSR(r1)
 986        ld      r3,_NIP(r1)
 987        andis.  r4,r12,0x5820
 988        li      r5,0x400
 989        b       .do_hash_page           /* Try to handle as hpte fault */
 990
 991        STD_EXCEPTION_COMMON(0xe20, h_instr_storage, .unknown_exception)
 992
 993/*
 994 * Here is the common SLB miss user that is used when going to virtual
 995 * mode for SLB misses, that is currently not used
 996 */
 997#ifdef __DISABLED__
 998        .align  7
 999        .globl  slb_miss_user_common
1000slb_miss_user_common:

1001        mflr    r10
1002        std     r3,PACA_EXGEN+EX_DAR(r13)
1003        stw     r9,PACA_EXGEN+EX_CCR(r13)
1004        std     r10,PACA_EXGEN+EX_LR(r13)
1005        std     r11,PACA_EXGEN+EX_SRR0(r13)
1006        bl      .slb_allocate_user
1007
1008        ld      r10,PACA_EXGEN+EX_LR(r13)
1009        ld      r3,PACA_EXGEN+EX_R3(r13)
1010        lwz     r9,PACA_EXGEN+EX_CCR(r13)
1011        ld      r11,PACA_EXGEN+EX_SRR0(r13)
1012        mtlr    r10
1013        beq-    slb_miss_fault
1014
1015        andi.   r10,r12,MSR_RI          /* check for unrecoverable exception */
1016        beq-    unrecov_user_slb
1017        mfmsr   r10
1018
1019.machine push
1020.machine "power4"
1021        mtcrf   0x80,r9
1022.machine pop
1023
1024        clrrdi  r10,r10,2               /* clear RI before setting SRR0/1 */
1025        mtmsrd  r10,1
1026
1027        mtspr   SRR0,r11
1028        mtspr   SRR1,r12
1029
1030        ld      r9,PACA_EXGEN+EX_R9(r13)
1031        ld      r10,PACA_EXGEN+EX_R10(r13)
1032        ld      r11,PACA_EXGEN+EX_R11(r13)
1033        ld      r12,PACA_EXGEN+EX_R12(r13)
1034        ld      r13,PACA_EXGEN+EX_R13(r13)
1035        rfid
1036        b       .
1037
1038slb_miss_fault:
1039        EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
1040        ld      r4,PACA_EXGEN+EX_DAR(r13)
1041        li      r5,0
1042        std     r4,_DAR(r1)
1043        std     r5,_DSISR(r1)
1044        b       handle_page_fault
1045
1046unrecov_user_slb:
1047        EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
1048        DISABLE_INTS
1049        bl      .save_nvgprs
10501:      addi    r3,r1,STACK_FRAME_OVERHEAD
1051        bl      .unrecoverable_exception
1052        b       1b
1053
1054#endif /* __DISABLED__ */
1055
1056
1057        .align  7
1058        .globl alignment_common
1059alignment_common:
1060        mfspr   r10,SPRN_DAR
1061        std     r10,PACA_EXGEN+EX_DAR(r13)
1062        mfspr   r10,SPRN_DSISR
1063        stw     r10,PACA_EXGEN+EX_DSISR(r13)
1064        EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
1065        ld      r3,PACA_EXGEN+EX_DAR(r13)
1066        lwz     r4,PACA_EXGEN+EX_DSISR(r13)
1067        std     r3,_DAR(r1)
1068        std     r4,_DSISR(r1)
1069        bl      .save_nvgprs
1070        DISABLE_INTS
1071        addi    r3,r1,STACK_FRAME_OVERHEAD
1072        bl      .alignment_exception
1073        b       .ret_from_except
1074
1075        .align  7
1076        .globl program_check_common
1077program_check_common:
1078        EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1079        bl      .save_nvgprs
1080        DISABLE_INTS
1081        addi    r3,r1,STACK_FRAME_OVERHEAD
1082        bl      .program_check_exception
1083        b       .ret_from_except
1084
1085        .align  7
1086        .globl fp_unavailable_common
1087fp_unavailable_common:
1088        EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
1089        bne     1f                      /* if from user, just load it up */
1090        bl      .save_nvgprs
1091        DISABLE_INTS
1092        addi    r3,r1,STACK_FRAME_OVERHEAD
1093        bl      .kernel_fp_unavailable_exception
1094        BUG_OPCODE
10951:
1096#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1097BEGIN_FTR_SECTION
1098        /* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1099         * transaction), go do TM stuff
1100         */
1101        rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1102        bne-    2f
1103END_FTR_SECTION_IFSET(CPU_FTR_TM)
1104#endif
1105        bl      .load_up_fpu
1106        b       fast_exception_return
1107#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
11082:      /* User process was in a transaction */
1109        bl      .save_nvgprs
1110        DISABLE_INTS
1111        addi    r3,r1,STACK_FRAME_OVERHEAD
1112        bl      .fp_unavailable_tm
1113        b       .ret_from_except
1114#endif
1115        .align  7
1116        .globl altivec_unavailable_common
1117altivec_unavailable_common:
1118        EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
1119#ifdef CONFIG_ALTIVEC
1120BEGIN_FTR_SECTION
1121        beq     1f
1122#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1123  BEGIN_FTR_SECTION_NESTED(69)
1124        /* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1125         * transaction), go do TM stuff
1126         */
1127        rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1128        bne-    2f
1129  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1130#endif
1131        bl      .load_up_altivec
1132        b       fast_exception_return
1133#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
11342:      /* User process was in a transaction */
1135        bl      .save_nvgprs
1136        DISABLE_INTS
1137        addi    r3,r1,STACK_FRAME_OVERHEAD
1138        bl      .altivec_unavailable_tm
1139        b       .ret_from_except
1140#endif
11411:
1142END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1143#endif
1144        bl      .save_nvgprs
1145        DISABLE_INTS
1146        addi    r3,r1,STACK_FRAME_OVERHEAD
1147        bl      .altivec_unavailable_exception
1148        b       .ret_from_except
1149
1150        .align  7
1151        .globl vsx_unavailable_common
1152vsx_unavailable_common:
1153        EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
1154#ifdef CONFIG_VSX
1155BEGIN_FTR_SECTION
1156        beq     1f
1157#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1158  BEGIN_FTR_SECTION_NESTED(69)
1159        /* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1160         * transaction), go do TM stuff
1161         */
1162        rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1163        bne-    2f
1164  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1165#endif
1166        b       .load_up_vsx
1167#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
11682:      /* User process was in a transaction */
1169        bl      .save_nvgprs
1170        DISABLE_INTS
1171        addi    r3,r1,STACK_FRAME_OVERHEAD
1172        bl      .vsx_unavailable_tm
1173        b       .ret_from_except
1174#endif
11751:
1176END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1177#endif
1178        bl      .save_nvgprs
1179        DISABLE_INTS
1180        addi    r3,r1,STACK_FRAME_OVERHEAD
1181        bl      .vsx_unavailable_exception
1182        b       .ret_from_except
1183
1184        STD_EXCEPTION_COMMON(0xf60, facility_unavailable, .facility_unavailable_exception)
1185        STD_EXCEPTION_COMMON(0xf80, hv_facility_unavailable, .facility_unavailable_exception)
1186
1187        .align  7
1188        .globl  __end_handlers
1189__end_handlers:
1190
1191        /* Equivalents to the above handlers for relocation-on interrupt vectors */
1192        STD_RELON_EXCEPTION_HV_OOL(0xe40, emulation_assist)
1193        MASKABLE_RELON_EXCEPTION_HV_OOL(0xe80, h_doorbell)
1194
1195        STD_RELON_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
1196        STD_RELON_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
1197        STD_RELON_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
1198        STD_RELON_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
1199        STD_RELON_EXCEPTION_HV_OOL(0xf80, hv_facility_unavailable)
1200
1201#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
1202/*
1203 * Data area reserved for FWNMI option.
1204 * This address (0x7000) is fixed by the RPA.
1205 */
1206        .= 0x7000
1207        .globl fwnmi_data_area
1208fwnmi_data_area:
1209
1210        /* pseries and powernv need to keep the whole page from
1211         * 0x7000 to 0x8000 free for use by the firmware
1212         */
1213        . = 0x8000
1214#endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */
1215
1216/* Space for CPU0's segment table */
1217        .balign 4096
1218        .globl initial_stab
1219initial_stab:
1220        .space  4096
1221
1222#ifdef CONFIG_PPC_POWERNV
1223_GLOBAL(opal_mc_secondary_handler)
1224        HMT_MEDIUM_PPR_DISCARD
1225        SET_SCRATCH0(r13)
1226        GET_PACA(r13)
1227        clrldi  r3,r3,2
1228        tovirt(r3,r3)
1229        std     r3,PACA_OPAL_MC_EVT(r13)
1230        ld      r13,OPAL_MC_SRR0(r3)
1231        mtspr   SPRN_SRR0,r13
1232        ld      r13,OPAL_MC_SRR1(r3)
1233        mtspr   SPRN_SRR1,r13
1234        ld      r3,OPAL_MC_GPR3(r3)
1235        GET_SCRATCH0(r13)
1236        b       machine_check_pSeries
1237#endif /* CONFIG_PPC_POWERNV */
1238
1239
1240/*
1241 * r13 points to the PACA, r9 contains the saved CR,
1242 * r12 contain the saved SRR1, SRR0 is still ready for return
1243 * r3 has the faulting address
1244 * r9 - r13 are saved in paca->exslb.
1245 * r3 is saved in paca->slb_r3
1246 * We assume we aren't going to take any exceptions during this procedure.
1247 */
1248_GLOBAL(slb_miss_realmode)
1249        mflr    r10
1250#ifdef CONFIG_RELOCATABLE
1251        mtctr   r11
1252#endif
1253
1254        stw     r9,PACA_EXSLB+EX_CCR(r13)       /* save CR in exc. frame */
1255        std     r10,PACA_EXSLB+EX_LR(r13)       /* save LR */
1256
1257        bl      .slb_allocate_realmode
1258
1259        /* All done -- return from exception. */
1260
1261        ld      r10,PACA_EXSLB+EX_LR(r13)
1262        ld      r3,PACA_EXSLB+EX_R3(r13)
1263        lwz     r9,PACA_EXSLB+EX_CCR(r13)       /* get saved CR */
1264
1265        mtlr    r10
1266
1267        andi.   r10,r12,MSR_RI  /* check for unrecoverable exception */
1268        beq-    2f
1269
1270.machine        push
1271.machine        "power4"
1272        mtcrf   0x80,r9
1273        mtcrf   0x01,r9         /* slb_allocate uses cr0 and cr7 */
1274.machine        pop
1275
1276        RESTORE_PPR_PACA(PACA_EXSLB, r9)
1277        ld      r9,PACA_EXSLB+EX_R9(r13)
1278        ld      r10,PACA_EXSLB+EX_R10(r13)
1279        ld      r11,PACA_EXSLB+EX_R11(r13)
1280        ld      r12,PACA_EXSLB+EX_R12(r13)
1281        ld      r13,PACA_EXSLB+EX_R13(r13)
1282        rfid
1283        b       .       /* prevent speculative execution */
1284
12852:      mfspr   r11,SPRN_SRR0
1286        ld      r10,PACAKBASE(r13)
1287        LOAD_HANDLER(r10,unrecov_slb)
1288        mtspr   SPRN_SRR0,r10
1289        ld      r10,PACAKMSR(r13)
1290        mtspr   SPRN_SRR1,r10
1291        rfid
1292        b       .
1293
1294unrecov_slb:
1295        EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
1296        DISABLE_INTS
1297        bl      .save_nvgprs
12981:      addi    r3,r1,STACK_FRAME_OVERHEAD
1299        bl      .unrecoverable_exception
1300        b       1b
1301
1302
1303#ifdef CONFIG_PPC_970_NAP
1304power4_fixup_nap:
1305        andc    r9,r9,r10
1306        std     r9,TI_LOCAL_FLAGS(r11)
1307        ld      r10,_LINK(r1)           /* make idle task do the */
1308        std     r10,_NIP(r1)            /* equivalent of a blr */
1309        blr
1310#endif
1311
1312/*
1313 * Hash table stuff
1314 */
1315        .align  7
1316_STATIC(do_hash_page)
1317        std     r3,_DAR(r1)
1318        std     r4,_DSISR(r1)
1319
1320        andis.  r0,r4,0xa410            /* weird error? */
1321        bne-    handle_page_fault       /* if not, try to insert a HPTE */
1322        andis.  r0,r4,DSISR_DABRMATCH@h
1323        bne-    handle_dabr_fault
1324
1325BEGIN_FTR_SECTION
1326        andis.  r0,r4,0x0020            /* Is it a segment table fault? */
1327        bne-    do_ste_alloc            /* If so handle it */
1328END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
1329
1330        CURRENT_THREAD_INFO(r11, r1)
1331        lwz     r0,TI_PREEMPT(r11)      /* If we're in an "NMI" */
1332        andis.  r0,r0,NMI_MASK@h        /* (i.e. an irq when soft-disabled) */
1333        bne     77f                     /* then don't call hash_page now */
1334        /*
1335         * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
1336         * accessing a userspace segment (even from the kernel). We assume
1337         * kernel addresses always have the high bit set.
1338         */
1339        rlwinm  r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
1340        rotldi  r0,r3,15                /* Move high bit into MSR_PR posn */
1341        orc     r0,r12,r0               /* MSR_PR | ~high_bit */
1342        rlwimi  r4,r0,32-13,30,30       /* becomes _PAGE_USER access bit */
1343        ori     r4,r4,1                 /* add _PAGE_PRESENT */
1344        rlwimi  r4,r5,22+2,31-2,31-2    /* Set _PAGE_EXEC if trap is 0x400 */
1345
1346        /*
1347         * r3 contains the faulting address
1348         * r4 contains the required access permissions
1349         * r5 contains the trap number
1350         *
1351         * at return r3 = 0 for success, 1 for page fault, negative for error
1352         */
1353        bl      .hash_page              /* build HPTE if possible */
1354        cmpdi   r3,0                    /* see if hash_page succeeded */
1355
1356        /* Success */
1357        beq     fast_exc_return_irq     /* Return from exception on success */
1358
1359        /* Error */
1360        blt-    13f
1361
1362/* Here we have a page fault that hash_page can't handle. */
1363handle_page_fault:
136411:     ld      r4,_DAR(r1)
1365        ld      r5,_DSISR(r1)
1366        addi    r3,r1,STACK_FRAME_OVERHEAD
1367        bl      .do_page_fault
1368        cmpdi   r3,0
1369        beq+    12f
1370        bl      .save_nvgprs
1371        mr      r5,r3
1372        addi    r3,r1,STACK_FRAME_OVERHEAD
1373        lwz     r4,_DAR(r1)
1374        bl      .bad_page_fault
1375        b       .ret_from_except
1376
1377/* We have a data breakpoint exception - handle it */
1378handle_dabr_fault:
1379        bl      .save_nvgprs
1380        ld      r4,_DAR(r1)
1381        ld      r5,_DSISR(r1)
1382        addi    r3,r1,STACK_FRAME_OVERHEAD
1383        bl      .do_break
138412:     b       .ret_from_except_lite
1385
1386
1387/* We have a page fault that hash_page could handle but HV refused
1388 * the PTE insertion
1389 */
139013:     bl      .save_nvgprs
1391        mr      r5,r3
1392        addi    r3,r1,STACK_FRAME_OVERHEAD
1393        ld      r4,_DAR(r1)
1394        bl      .low_hash_fault
1395        b       .ret_from_except
1396
1397/*
1398 * We come here as a result of a DSI at a point where we don't want
1399 * to call hash_page, such as when we are accessing memory (possibly
1400 * user memory) inside a PMU interrupt that occurred while interrupts
1401 * were soft-disabled.  We want to invoke the exception handler for
1402 * the access, or panic if there isn't a handler.
1403 */
140477:     bl      .save_nvgprs
1405        mr      r4,r3
1406        addi    r3,r1,STACK_FRAME_OVERHEAD
1407        li      r5,SIGSEGV
1408        bl      .bad_page_fault
1409        b       .ret_from_except
1410
1411        /* here we have a segment miss */
1412do_ste_alloc:
1413        bl      .ste_allocate           /* try to insert stab entry */
1414        cmpdi   r3,0
1415        bne-    handle_page_fault
1416        b       fast_exception_return
1417
1418/*
1419 * r13 points to the PACA, r9 contains the saved CR,
1420 * r11 and r12 contain the saved SRR0 and SRR1.
1421 * r9 - r13 are saved in paca->exslb.
1422 * We assume we aren't going to take any exceptions during this procedure.
1423 * We assume (DAR >> 60) == 0xc.
1424 */
1425        .align  7
1426_GLOBAL(do_stab_bolted)
1427        stw     r9,PACA_EXSLB+EX_CCR(r13)       /* save CR in exc. frame */
1428        std     r11,PACA_EXSLB+EX_SRR0(r13)     /* save SRR0 in exc. frame */
1429        mfspr   r11,SPRN_DAR                    /* ea */
1430
1431        /*
1432         * check for bad kernel/user address
1433         * (ea & ~REGION_MASK) >= PGTABLE_RANGE
1434         */
1435        rldicr. r9,r11,4,(63 - 46 - 4)
1436        li      r9,0    /* VSID = 0 for bad address */
1437        bne-    0f
1438
1439        /*
1440         * Calculate VSID:
1441         * This is the kernel vsid, we take the top for context from
1442         * the range. context = (MAX_USER_CONTEXT) + ((ea >> 60) - 0xc) + 1
1443         * Here we know that (ea >> 60) == 0xc
1444         */
1445        lis     r9,(MAX_USER_CONTEXT + 1)@ha
1446        addi    r9,r9,(MAX_USER_CONTEXT + 1)@l
1447
1448        srdi    r10,r11,SID_SHIFT
1449        rldimi  r10,r9,ESID_BITS,0 /* proto vsid */
1450        ASM_VSID_SCRAMBLE(r10, r9, 256M)
1451        rldic   r9,r10,12,16    /* r9 = vsid << 12 */
1452
14530:
1454        /* Hash to the primary group */
1455        ld      r10,PACASTABVIRT(r13)
1456        srdi    r11,r11,SID_SHIFT
1457        rldimi  r10,r11,7,52    /* r10 = first ste of the group */
1458
1459        /* Search the primary group for a free entry */
14601:      ld      r11,0(r10)      /* Test valid bit of the current ste    */
1461        andi.   r11,r11,0x80
1462        beq     2f
1463        addi    r10,r10,16
1464        andi.   r11,r10,0x70
1465        bne     1b
1466
1467        /* Stick for only searching the primary group for now.          */
1468        /* At least for now, we use a very simple random castout scheme */
1469        /* Use the TB as a random number ;  OR in 1 to avoid entry 0    */
1470        mftb    r11
1471        rldic   r11,r11,4,57    /* r11 = (r11 << 4) & 0x70 */
1472        ori     r11,r11,0x10
1473
1474        /* r10 currently points to an ste one past the group of interest */
1475        /* make it point to the randomly selected entry                 */
1476        subi    r10,r10,128
1477        or      r10,r10,r11     /* r10 is the entry to invalidate       */
1478
1479        isync                   /* mark the entry invalid               */
1480        ld      r11,0(r10)
1481        rldicl  r11,r11,56,1    /* clear the valid bit */
1482        rotldi  r11,r11,8
1483        std     r11,0(r10)
1484        sync
1485
1486        clrrdi  r11,r11,28      /* Get the esid part of the ste         */
1487        slbie   r11
1488
14892:      std     r9,8(r10)       /* Store the vsid part of the ste       */
1490        eieio
1491
1492        mfspr   r11,SPRN_DAR            /* Get the new esid                     */
1493        clrrdi  r11,r11,28      /* Permits a full 32b of ESID           */
1494        ori     r11,r11,0x90    /* Turn on valid and kp                 */
1495        std     r11,0(r10)      /* Put new entry back into the stab     */
1496
1497        sync
1498
1499        /* All done -- return from exception. */
1500        lwz     r9,PACA_EXSLB+EX_CCR(r13)       /* get saved CR */
1501        ld      r11,PACA_EXSLB+EX_SRR0(r13)     /* get saved SRR0 */
1502
1503        andi.   r10,r12,MSR_RI
1504        beq-    unrecov_slb
1505
1506        mtcrf   0x80,r9                 /* restore CR */
1507
1508        mfmsr   r10
1509        clrrdi  r10,r10,2
1510        mtmsrd  r10,1
1511
1512        mtspr   SPRN_SRR0,r11
1513        mtspr   SPRN_SRR1,r12
1514        ld      r9,PACA_EXSLB+EX_R9(r13)
1515        ld      r10,PACA_EXSLB+EX_R10(r13)
1516        ld      r11,PACA_EXSLB+EX_R11(r13)
1517        ld      r12,PACA_EXSLB+EX_R12(r13)
1518        ld      r13,PACA_EXSLB+EX_R13(r13)
1519        rfid
1520        b       .       /* prevent speculative execution */
1521
Hosted by Missing Link Electronics. The original LXR software by the LXR community, this experimental version by lxr@linux.no.