LXR linux/arch/sparc/kernel/sun4v_tlb

   1/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
   2 *
   3 * Copyright (C) 2006 <davem@davemloft.net>
   4 */
   5
   6        .text
   7        .align  32
   8
   9        /* Load ITLB fault information into VADDR and CTX, using BASE.  */
  10#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
  11        ldx     [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
  12        ldx     [BASE + HV_FAULT_I_CTX_OFFSET], CTX;
  13
  14        /* Load DTLB fault information into VADDR and CTX, using BASE.  */
  15#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
  16        ldx     [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
  17        ldx     [BASE + HV_FAULT_D_CTX_OFFSET], CTX;
  18
  19        /* DEST = (VADDR >> 22)
  20         *
  21         * Branch to ZERO_CTX_LABEL if context is zero.
  22         */
  23#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
  24        srlx    VADDR, 22, DEST; \
  25        brz,pn  CTX, ZERO_CTX_LABEL; \
  26         nop;
  27
  28        /* Create TSB pointer.  This is something like:
  29         *
  30         * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
  31         * tsb_base = tsb_reg & ~0x7UL;
  32         * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
  33         * tsb_ptr = tsb_base + (tsb_index * 16);
  34         */
  35#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
  36        and     TSB_PTR, 0x7, TMP1;                     \
  37        mov     512, TMP2;                              \
  38        andn    TSB_PTR, 0x7, TSB_PTR;                  \
  39        sllx    TMP2, TMP1, TMP2;                       \
  40        srlx    VADDR, HASH_SHIFT, TMP1;                \
  41        sub     TMP2, 1, TMP2;                          \
  42        and     TMP1, TMP2, TMP1;                       \
  43        sllx    TMP1, 4, TMP1;                          \
  44        add     TSB_PTR, TMP1, TSB_PTR;
  45
  46sun4v_itlb_miss:
  47        /* Load MMU Miss base into %g2.  */
  48        ldxa    [%g0] ASI_SCRATCHPAD, %g2
  49        
  50        /* Load UTSB reg into %g1.  */
  51        mov     SCRATCHPAD_UTSBREG1, %g1
  52        ldxa    [%g1] ASI_SCRATCHPAD, %g1
  53
  54        LOAD_ITLB_INFO(%g2, %g4, %g5)
  55        COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
  56        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
  57
  58        /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
  59        ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
  60        cmp     %g2, %g6
  61        bne,a,pn %xcc, tsb_miss_page_table_walk
  62         mov    FAULT_CODE_ITLB, %g3
  63        andcc   %g3, _PAGE_EXEC_4V, %g0
  64        be,a,pn %xcc, tsb_do_fault
  65         mov    FAULT_CODE_ITLB, %g3
  66
  67        /* We have a valid entry, make hypervisor call to load
  68         * I-TLB and return from trap.
  69         *
  70         * %g3: PTE
  71         * %g4: vaddr
  72         */
  73sun4v_itlb_load:
  74        ldxa    [%g0] ASI_SCRATCHPAD, %g6
  75        mov     %o0, %g1                ! save %o0
  76        mov     %o1, %g2                ! save %o1
  77        mov     %o2, %g5                ! save %o2
  78        mov     %o3, %g7                ! save %o3
  79        mov     %g4, %o0                ! vaddr
  80        ldx     [%g6 + HV_FAULT_I_CTX_OFFSET], %o1      ! ctx
  81        mov     %g3, %o2                ! PTE
  82        mov     HV_MMU_IMMU, %o3        ! flags
  83        ta      HV_MMU_MAP_ADDR_TRAP
  84        brnz,pn %o0, sun4v_itlb_error
  85         mov    %g2, %o1                ! restore %o1
  86        mov     %g1, %o0                ! restore %o0
  87        mov     %g5, %o2                ! restore %o2
  88        mov     %g7, %o3                ! restore %o3
  89
  90        retry
  91
  92sun4v_dtlb_miss:
  93        /* Load MMU Miss base into %g2.  */
  94        ldxa    [%g0] ASI_SCRATCHPAD, %g2
  95        
  96        /* Load UTSB reg into %g1.  */
  97        mov     SCRATCHPAD_UTSBREG1, %g1
  98        ldxa    [%g1] ASI_SCRATCHPAD, %g1
  99
 100        LOAD_DTLB_INFO(%g2, %g4, %g5)
 101        COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
 102        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
 103
 104        /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
 105        ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
 106        cmp     %g2, %g6
 107        bne,a,pn %xcc, tsb_miss_page_table_walk
 108         mov    FAULT_CODE_DTLB, %g3
 109
 110        /* We have a valid entry, make hypervisor call to load
 111         * D-TLB and return from trap.
 112         *
 113         * %g3: PTE
 114         * %g4: vaddr
 115         */
 116sun4v_dtlb_load:
 117        ldxa    [%g0] ASI_SCRATCHPAD, %g6
 118        mov     %o0, %g1                ! save %o0
 119        mov     %o1, %g2                ! save %o1
 120        mov     %o2, %g5                ! save %o2
 121        mov     %o3, %g7                ! save %o3
 122        mov     %g4, %o0                ! vaddr
 123        ldx     [%g6 + HV_FAULT_D_CTX_OFFSET], %o1      ! ctx
 124        mov     %g3, %o2                ! PTE
 125        mov     HV_MMU_DMMU, %o3        ! flags
 126        ta      HV_MMU_MAP_ADDR_TRAP
 127        brnz,pn %o0, sun4v_dtlb_error
 128         mov    %g2, %o1                ! restore %o1
 129        mov     %g1, %o0                ! restore %o0
 130        mov     %g5, %o2                ! restore %o2
 131        mov     %g7, %o3                ! restore %o3
 132
 133        retry
 134
 135sun4v_dtlb_prot:
 136        SET_GL(1)
 137
 138        /* Load MMU Miss base into %g5.  */
 139        ldxa    [%g0] ASI_SCRATCHPAD, %g5
 140        
 141        ldx     [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
 142        rdpr    %tl, %g1
 143        cmp     %g1, 1
 144        bgu,pn  %xcc, winfix_trampoline
 145         mov    FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
 146        ba,pt   %xcc, sparc64_realfault_common
 147         nop
 148
 149        /* Called from trap table:
 150         * %g4: vaddr
 151         * %g5: context
 152         * %g6: TAG TARGET
 153         */
 154sun4v_itsb_miss:
 155        mov     SCRATCHPAD_UTSBREG1, %g1
 156        ldxa    [%g1] ASI_SCRATCHPAD, %g1
 157        brz,pn  %g5, kvmap_itlb_4v
 158         mov    FAULT_CODE_ITLB, %g3
 159        ba,a,pt %xcc, sun4v_tsb_miss_common
 160
 161        /* Called from trap table:
 162         * %g4: vaddr
 163         * %g5: context
 164         * %g6: TAG TARGET
 165         */
 166sun4v_dtsb_miss:
 167        mov     SCRATCHPAD_UTSBREG1, %g1
 168        ldxa    [%g1] ASI_SCRATCHPAD, %g1
 169        brz,pn  %g5, kvmap_dtlb_4v
 170         mov    FAULT_CODE_DTLB, %g3
 171
 172        /* fallthrough */
 173
 174sun4v_tsb_miss_common:
 175        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
 176
 177        sub     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
 178
 179#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 180        mov     SCRATCHPAD_UTSBREG2, %g5
 181        ldxa    [%g5] ASI_SCRATCHPAD, %g5
 182        cmp     %g5, -1
 183        be,pt   %xcc, 80f
 184         nop
 185        COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7)
 186
 187        /* That clobbered %g2, reload it.  */
 188        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 189        sub     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
 190
 19180:     stx     %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
 192#endif
 193
 194        ba,pt   %xcc, tsb_miss_page_table_walk_sun4v_fastpath
 195         ldx    [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
 196
 197sun4v_itlb_error:
 198        sethi   %hi(sun4v_err_itlb_vaddr), %g1
 199        stx     %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
 200        sethi   %hi(sun4v_err_itlb_ctx), %g1
 201        ldxa    [%g0] ASI_SCRATCHPAD, %g6
 202        ldx     [%g6 + HV_FAULT_I_CTX_OFFSET], %o1
 203        stx     %o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
 204        sethi   %hi(sun4v_err_itlb_pte), %g1
 205        stx     %g3, [%g1 + %lo(sun4v_err_itlb_pte)]
 206        sethi   %hi(sun4v_err_itlb_error), %g1
 207        stx     %o0, [%g1 + %lo(sun4v_err_itlb_error)]
 208
 209        rdpr    %tl, %g4
 210        cmp     %g4, 1
 211        ble,pt  %icc, 1f
 212         sethi  %hi(2f), %g7
 213        ba,pt   %xcc, etraptl1
 214         or     %g7, %lo(2f), %g7
 215
 2161:      ba,pt   %xcc, etrap
 2172:       or     %g7, %lo(2b), %g7
 218        mov     %l4, %o1
 219        call    sun4v_itlb_error_report
 220         add    %sp, PTREGS_OFF, %o0
 221
 222        /* NOTREACHED */
 223
 224sun4v_dtlb_error:
 225        sethi   %hi(sun4v_err_dtlb_vaddr), %g1
 226        stx     %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
 227        sethi   %hi(sun4v_err_dtlb_ctx), %g1
 228        ldxa    [%g0] ASI_SCRATCHPAD, %g6
 229        ldx     [%g6 + HV_FAULT_D_CTX_OFFSET], %o1
 230        stx     %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
 231        sethi   %hi(sun4v_err_dtlb_pte), %g1
 232        stx     %g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
 233        sethi   %hi(sun4v_err_dtlb_error), %g1
 234        stx     %o0, [%g1 + %lo(sun4v_err_dtlb_error)]
 235
 236        rdpr    %tl, %g4
 237        cmp     %g4, 1
 238        ble,pt  %icc, 1f
 239         sethi  %hi(2f), %g7
 240        ba,pt   %xcc, etraptl1
 241         or     %g7, %lo(2f), %g7
 242
 2431:      ba,pt   %xcc, etrap
 2442:       or     %g7, %lo(2b), %g7
 245        mov     %l4, %o1
 246        call    sun4v_dtlb_error_report
 247         add    %sp, PTREGS_OFF, %o0
 248
 249        /* NOTREACHED */
 250
 251        /* Instruction Access Exception, tl0. */
 252sun4v_iacc:
 253        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 254        ldx     [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
 255        ldx     [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
 256        ldx     [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
 257        sllx    %g3, 16, %g3
 258        or      %g5, %g3, %g5
 259        ba,pt   %xcc, etrap
 260         rd     %pc, %g7
 261        mov     %l4, %o1
 262        mov     %l5, %o2
 263        call    sun4v_insn_access_exception
 264         add    %sp, PTREGS_OFF, %o0
 265        ba,a,pt %xcc, rtrap
 266
 267        /* Instruction Access Exception, tl1. */
 268sun4v_iacc_tl1:
 269        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 270        ldx     [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
 271        ldx     [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
 272        ldx     [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
 273        sllx    %g3, 16, %g3
 274        or      %g5, %g3, %g5
 275        ba,pt   %xcc, etraptl1
 276         rd     %pc, %g7
 277        mov     %l4, %o1
 278        mov     %l5, %o2
 279        call    sun4v_insn_access_exception_tl1
 280         add    %sp, PTREGS_OFF, %o0
 281        ba,a,pt %xcc, rtrap
 282
 283        /* Data Access Exception, tl0. */
 284sun4v_dacc:
 285        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 286        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 287        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 288        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 289        sllx    %g3, 16, %g3
 290        or      %g5, %g3, %g5
 291        ba,pt   %xcc, etrap
 292         rd     %pc, %g7
 293        mov     %l4, %o1
 294        mov     %l5, %o2
 295        call    sun4v_data_access_exception
 296         add    %sp, PTREGS_OFF, %o0
 297        ba,a,pt %xcc, rtrap
 298
 299        /* Data Access Exception, tl1. */
 300sun4v_dacc_tl1:
 301        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 302        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 303        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 304        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 305        sllx    %g3, 16, %g3
 306        or      %g5, %g3, %g5
 307        ba,pt   %xcc, etraptl1
 308         rd     %pc, %g7
 309        mov     %l4, %o1
 310        mov     %l5, %o2
 311        call    sun4v_data_access_exception_tl1
 312         add    %sp, PTREGS_OFF, %o0
 313        ba,a,pt %xcc, rtrap
 314
 315        /* Memory Address Unaligned.  */
 316sun4v_mna:
 317        /* Window fixup? */
 318        rdpr    %tl, %g2
 319        cmp     %g2, 1
 320        ble,pt  %icc, 1f
 321         nop
 322
 323        SET_GL(1)
 324        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 325        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
 326        mov     HV_FAULT_TYPE_UNALIGNED, %g3
 327        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g4
 328        sllx    %g3, 16, %g3
 329        or      %g4, %g3, %g4
 330        ba,pt   %xcc, winfix_mna
 331         rdpr   %tpc, %g3
 332        /* not reached */
 333
 3341:      ldxa    [%g0] ASI_SCRATCHPAD, %g2
 335        mov     HV_FAULT_TYPE_UNALIGNED, %g3
 336        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 337        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 338        sllx    %g3, 16, %g3
 339        or      %g5, %g3, %g5
 340
 341        ba,pt   %xcc, etrap
 342         rd     %pc, %g7
 343        mov     %l4, %o1
 344        mov     %l5, %o2
 345        call    sun4v_do_mna
 346         add    %sp, PTREGS_OFF, %o0
 347        ba,a,pt %xcc, rtrap
 348
 349        /* Privileged Action.  */
 350sun4v_privact:
 351        ba,pt   %xcc, etrap
 352         rd     %pc, %g7
 353        call    do_privact
 354         add    %sp, PTREGS_OFF, %o0
 355        ba,a,pt %xcc, rtrap
 356
 357        /* Unaligned ldd float, tl0. */
 358sun4v_lddfmna:
 359        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 360        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 361        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 362        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 363        sllx    %g3, 16, %g3
 364        or      %g5, %g3, %g5
 365        ba,pt   %xcc, etrap
 366         rd     %pc, %g7
 367        mov     %l4, %o1
 368        mov     %l5, %o2
 369        call    handle_lddfmna
 370         add    %sp, PTREGS_OFF, %o0
 371        ba,a,pt %xcc, rtrap
 372
 373        /* Unaligned std float, tl0. */
 374sun4v_stdfmna:
 375        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 376        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 377        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 378        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 379        sllx    %g3, 16, %g3
 380        or      %g5, %g3, %g5
 381        ba,pt   %xcc, etrap
 382         rd     %pc, %g7
 383        mov     %l4, %o1
 384        mov     %l5, %o2
 385        call    handle_stdfmna
 386         add    %sp, PTREGS_OFF, %o0
 387        ba,a,pt %xcc, rtrap
 388
 389#define BRANCH_ALWAYS   0x10680000
 390#define NOP             0x01000000
 391#define SUN4V_DO_PATCH(OLD, NEW)        \
 392        sethi   %hi(NEW), %g1; \
 393        or      %g1, %lo(NEW), %g1; \
 394        sethi   %hi(OLD), %g2; \
 395        or      %g2, %lo(OLD), %g2; \
 396        sub     %g1, %g2, %g1; \
 397        sethi   %hi(BRANCH_ALWAYS), %g3; \
 398        sll     %g1, 11, %g1; \
 399        srl     %g1, 11 + 2, %g1; \
 400        or      %g3, %lo(BRANCH_ALWAYS), %g3; \
 401        or      %g3, %g1, %g3; \
 402        stw     %g3, [%g2]; \
 403        sethi   %hi(NOP), %g3; \
 404        or      %g3, %lo(NOP), %g3; \
 405        stw     %g3, [%g2 + 0x4]; \
 406        flush   %g2;
 407
 408        .globl  sun4v_patch_tlb_handlers
 409        .type   sun4v_patch_tlb_handlers,#function
 410sun4v_patch_tlb_handlers:
 411        SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
 412        SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
 413        SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
 414        SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
 415        SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
 416        SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
 417        SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
 418        SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
 419        SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
 420        SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
 421        SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
 422        SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
 423        SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
 424        SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
 425        SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
 426        retl
 427         nop
 428        .size   sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
 429