LXR linux/arch/sparc/kernel/sun4v_tlb

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
   3 *
   4 * Copyright (C) 2006 <davem@davemloft.net>
   5 */
   6
   7        .text
   8        .align  32
   9
  10        /* Load ITLB fault information into VADDR and CTX, using BASE.  */
  11#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
  12        ldx     [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
  13        ldx     [BASE + HV_FAULT_I_CTX_OFFSET], CTX;
  14
  15        /* Load DTLB fault information into VADDR and CTX, using BASE.  */
  16#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
  17        ldx     [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
  18        ldx     [BASE + HV_FAULT_D_CTX_OFFSET], CTX;
  19
  20        /* DEST = (VADDR >> 22)
  21         *
  22         * Branch to ZERO_CTX_LABEL if context is zero.
  23         */
  24#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
  25        srlx    VADDR, 22, DEST; \
  26        brz,pn  CTX, ZERO_CTX_LABEL; \
  27         nop;
  28
  29        /* Create TSB pointer.  This is something like:
  30         *
  31         * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
  32         * tsb_base = tsb_reg & ~0x7UL;
  33         * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
  34         * tsb_ptr = tsb_base + (tsb_index * 16);
  35         */
  36#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
  37        and     TSB_PTR, 0x7, TMP1;                     \
  38        mov     512, TMP2;                              \
  39        andn    TSB_PTR, 0x7, TSB_PTR;                  \
  40        sllx    TMP2, TMP1, TMP2;                       \
  41        srlx    VADDR, HASH_SHIFT, TMP1;                \
  42        sub     TMP2, 1, TMP2;                          \
  43        and     TMP1, TMP2, TMP1;                       \
  44        sllx    TMP1, 4, TMP1;                          \
  45        add     TSB_PTR, TMP1, TSB_PTR;
  46
  47sun4v_itlb_miss:
  48        /* Load MMU Miss base into %g2.  */
  49        ldxa    [%g0] ASI_SCRATCHPAD, %g2
  50        
  51        /* Load UTSB reg into %g1.  */
  52        mov     SCRATCHPAD_UTSBREG1, %g1
  53        ldxa    [%g1] ASI_SCRATCHPAD, %g1
  54
  55        LOAD_ITLB_INFO(%g2, %g4, %g5)
  56        COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
  57        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
  58
  59        /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
  60        ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
  61        cmp     %g2, %g6
  62        bne,a,pn %xcc, tsb_miss_page_table_walk
  63         mov    FAULT_CODE_ITLB, %g3
  64        andcc   %g3, _PAGE_EXEC_4V, %g0
  65        be,a,pn %xcc, tsb_do_fault
  66         mov    FAULT_CODE_ITLB, %g3
  67
  68        /* We have a valid entry, make hypervisor call to load
  69         * I-TLB and return from trap.
  70         *
  71         * %g3: PTE
  72         * %g4: vaddr
  73         */
  74sun4v_itlb_load:
  75        ldxa    [%g0] ASI_SCRATCHPAD, %g6
  76        mov     %o0, %g1                ! save %o0
  77        mov     %o1, %g2                ! save %o1
  78        mov     %o2, %g5                ! save %o2
  79        mov     %o3, %g7                ! save %o3
  80        mov     %g4, %o0                ! vaddr
  81        ldx     [%g6 + HV_FAULT_I_CTX_OFFSET], %o1      ! ctx
  82        mov     %g3, %o2                ! PTE
  83        mov     HV_MMU_IMMU, %o3        ! flags
  84        ta      HV_MMU_MAP_ADDR_TRAP
  85        brnz,pn %o0, sun4v_itlb_error
  86         mov    %g2, %o1                ! restore %o1
  87        mov     %g1, %o0                ! restore %o0
  88        mov     %g5, %o2                ! restore %o2
  89        mov     %g7, %o3                ! restore %o3
  90
  91        retry
  92
  93sun4v_dtlb_miss:
  94        /* Load MMU Miss base into %g2.  */
  95        ldxa    [%g0] ASI_SCRATCHPAD, %g2
  96        
  97        /* Load UTSB reg into %g1.  */
  98        mov     SCRATCHPAD_UTSBREG1, %g1
  99        ldxa    [%g1] ASI_SCRATCHPAD, %g1
 100
 101        LOAD_DTLB_INFO(%g2, %g4, %g5)
 102        COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
 103        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
 104
 105        /* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
 106        ldda    [%g1] ASI_QUAD_LDD_PHYS_4V, %g2
 107        cmp     %g2, %g6
 108        bne,a,pn %xcc, tsb_miss_page_table_walk
 109         mov    FAULT_CODE_DTLB, %g3
 110
 111        /* We have a valid entry, make hypervisor call to load
 112         * D-TLB and return from trap.
 113         *
 114         * %g3: PTE
 115         * %g4: vaddr
 116         */
 117sun4v_dtlb_load:
 118        ldxa    [%g0] ASI_SCRATCHPAD, %g6
 119        mov     %o0, %g1                ! save %o0
 120        mov     %o1, %g2                ! save %o1
 121        mov     %o2, %g5                ! save %o2
 122        mov     %o3, %g7                ! save %o3
 123        mov     %g4, %o0                ! vaddr
 124        ldx     [%g6 + HV_FAULT_D_CTX_OFFSET], %o1      ! ctx
 125        mov     %g3, %o2                ! PTE
 126        mov     HV_MMU_DMMU, %o3        ! flags
 127        ta      HV_MMU_MAP_ADDR_TRAP
 128        brnz,pn %o0, sun4v_dtlb_error
 129         mov    %g2, %o1                ! restore %o1
 130        mov     %g1, %o0                ! restore %o0
 131        mov     %g5, %o2                ! restore %o2
 132        mov     %g7, %o3                ! restore %o3
 133
 134        retry
 135
 136sun4v_dtlb_prot:
 137        SET_GL(1)
 138
 139        /* Load MMU Miss base into %g5.  */
 140        ldxa    [%g0] ASI_SCRATCHPAD, %g5
 141        
 142        ldx     [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
 143        rdpr    %tl, %g1
 144        cmp     %g1, 1
 145        bgu,pn  %xcc, winfix_trampoline
 146         mov    FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
 147        ba,pt   %xcc, sparc64_realfault_common
 148         nop
 149
 150        /* Called from trap table:
 151         * %g4: vaddr
 152         * %g5: context
 153         * %g6: TAG TARGET
 154         */
 155sun4v_itsb_miss:
 156        mov     SCRATCHPAD_UTSBREG1, %g1
 157        ldxa    [%g1] ASI_SCRATCHPAD, %g1
 158        brz,pn  %g5, kvmap_itlb_4v
 159         mov    FAULT_CODE_ITLB, %g3
 160        ba,a,pt %xcc, sun4v_tsb_miss_common
 161
 162        /* Called from trap table:
 163         * %g4: vaddr
 164         * %g5: context
 165         * %g6: TAG TARGET
 166         */
 167sun4v_dtsb_miss:
 168        mov     SCRATCHPAD_UTSBREG1, %g1
 169        ldxa    [%g1] ASI_SCRATCHPAD, %g1
 170        brz,pn  %g5, kvmap_dtlb_4v
 171         mov    FAULT_CODE_DTLB, %g3
 172
 173        /* fallthrough */
 174
 175sun4v_tsb_miss_common:
 176        COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
 177
 178        sub     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
 179
 180#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 181        mov     SCRATCHPAD_UTSBREG2, %g5
 182        ldxa    [%g5] ASI_SCRATCHPAD, %g5
 183        cmp     %g5, -1
 184        be,pt   %xcc, 80f
 185         nop
 186        COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7)
 187
 188        /* That clobbered %g2, reload it.  */
 189        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 190        sub     %g2, TRAP_PER_CPU_FAULT_INFO, %g2
 191
 19280:     stx     %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
 193#endif
 194
 195        ba,pt   %xcc, tsb_miss_page_table_walk_sun4v_fastpath
 196         ldx    [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
 197
 198sun4v_itlb_error:
 199        rdpr    %tl, %g1
 200        cmp     %g1, 1
 201        ble,pt  %icc, sun4v_bad_ra
 202         or     %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1
 203
 204        sethi   %hi(sun4v_err_itlb_vaddr), %g1
 205        stx     %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
 206        sethi   %hi(sun4v_err_itlb_ctx), %g1
 207        ldxa    [%g0] ASI_SCRATCHPAD, %g6
 208        ldx     [%g6 + HV_FAULT_I_CTX_OFFSET], %o1
 209        stx     %o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
 210        sethi   %hi(sun4v_err_itlb_pte), %g1
 211        stx     %g3, [%g1 + %lo(sun4v_err_itlb_pte)]
 212        sethi   %hi(sun4v_err_itlb_error), %g1
 213        stx     %o0, [%g1 + %lo(sun4v_err_itlb_error)]
 214
 215        sethi   %hi(1f), %g7
 216        rdpr    %tl, %g4
 217        ba,pt   %xcc, etraptl1
 2181:       or     %g7, %lo(1f), %g7
 219        mov     %l4, %o1
 220        call    sun4v_itlb_error_report
 221         add    %sp, PTREGS_OFF, %o0
 222
 223        /* NOTREACHED */
 224
 225sun4v_dtlb_error:
 226        rdpr    %tl, %g1
 227        cmp     %g1, 1
 228        ble,pt  %icc, sun4v_bad_ra
 229         or     %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1
 230
 231        sethi   %hi(sun4v_err_dtlb_vaddr), %g1
 232        stx     %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
 233        sethi   %hi(sun4v_err_dtlb_ctx), %g1
 234        ldxa    [%g0] ASI_SCRATCHPAD, %g6
 235        ldx     [%g6 + HV_FAULT_D_CTX_OFFSET], %o1
 236        stx     %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
 237        sethi   %hi(sun4v_err_dtlb_pte), %g1
 238        stx     %g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
 239        sethi   %hi(sun4v_err_dtlb_error), %g1
 240        stx     %o0, [%g1 + %lo(sun4v_err_dtlb_error)]
 241
 242        sethi   %hi(1f), %g7
 243        rdpr    %tl, %g4
 244        ba,pt   %xcc, etraptl1
 2451:       or     %g7, %lo(1f), %g7
 246        mov     %l4, %o1
 247        call    sun4v_dtlb_error_report
 248         add    %sp, PTREGS_OFF, %o0
 249
 250        /* NOTREACHED */
 251
 252sun4v_bad_ra:
 253        or      %g0, %g4, %g5
 254        ba,pt   %xcc, sparc64_realfault_common
 255         or     %g1, %g0, %g4
 256
 257        /* NOTREACHED */
 258
 259        /* Instruction Access Exception, tl0. */
 260sun4v_iacc:
 261        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 262        ldx     [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
 263        ldx     [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
 264        ldx     [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
 265        sllx    %g3, 16, %g3
 266        or      %g5, %g3, %g5
 267        ba,pt   %xcc, etrap
 268         rd     %pc, %g7
 269        mov     %l4, %o1
 270        mov     %l5, %o2
 271        call    sun4v_insn_access_exception
 272         add    %sp, PTREGS_OFF, %o0
 273        ba,a,pt %xcc, rtrap
 274
 275        /* Instruction Access Exception, tl1. */
 276sun4v_iacc_tl1:
 277        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 278        ldx     [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
 279        ldx     [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
 280        ldx     [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
 281        sllx    %g3, 16, %g3
 282        or      %g5, %g3, %g5
 283        ba,pt   %xcc, etraptl1
 284         rd     %pc, %g7
 285        mov     %l4, %o1
 286        mov     %l5, %o2
 287        call    sun4v_insn_access_exception_tl1
 288         add    %sp, PTREGS_OFF, %o0
 289        ba,a,pt %xcc, rtrap
 290
 291        /* Data Access Exception, tl0. */
 292sun4v_dacc:
 293        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 294        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 295        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 296        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 297        sllx    %g3, 16, %g3
 298        or      %g5, %g3, %g5
 299        ba,pt   %xcc, etrap
 300         rd     %pc, %g7
 301        mov     %l4, %o1
 302        mov     %l5, %o2
 303        call    sun4v_data_access_exception
 304         add    %sp, PTREGS_OFF, %o0
 305        ba,a,pt %xcc, rtrap
 306
 307        /* Data Access Exception, tl1. */
 308sun4v_dacc_tl1:
 309        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 310        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 311        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 312        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 313        sllx    %g3, 16, %g3
 314        or      %g5, %g3, %g5
 315        ba,pt   %xcc, etraptl1
 316         rd     %pc, %g7
 317        mov     %l4, %o1
 318        mov     %l5, %o2
 319        call    sun4v_data_access_exception_tl1
 320         add    %sp, PTREGS_OFF, %o0
 321        ba,a,pt %xcc, rtrap
 322
 323        /* Memory Address Unaligned.  */
 324sun4v_mna:
 325        /* Window fixup? */
 326        rdpr    %tl, %g2
 327        cmp     %g2, 1
 328        ble,pt  %icc, 1f
 329         nop
 330
 331        SET_GL(1)
 332        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 333        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
 334        mov     HV_FAULT_TYPE_UNALIGNED, %g3
 335        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g4
 336        sllx    %g3, 16, %g3
 337        or      %g4, %g3, %g4
 338        ba,pt   %xcc, winfix_mna
 339         rdpr   %tpc, %g3
 340        /* not reached */
 341
 3421:      ldxa    [%g0] ASI_SCRATCHPAD, %g2
 343        mov     HV_FAULT_TYPE_UNALIGNED, %g3
 344        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 345        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 346        sllx    %g3, 16, %g3
 347        or      %g5, %g3, %g5
 348
 349        ba,pt   %xcc, etrap
 350         rd     %pc, %g7
 351        mov     %l4, %o1
 352        mov     %l5, %o2
 353        call    sun4v_do_mna
 354         add    %sp, PTREGS_OFF, %o0
 355        ba,a,pt %xcc, rtrap
 356         nop
 357
 358        /* Privileged Action.  */
 359sun4v_privact:
 360        ba,pt   %xcc, etrap
 361         rd     %pc, %g7
 362        call    do_privact
 363         add    %sp, PTREGS_OFF, %o0
 364        ba,a,pt %xcc, rtrap
 365
 366        /* Unaligned ldd float, tl0. */
 367sun4v_lddfmna:
 368        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 369        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 370        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 371        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 372        sllx    %g3, 16, %g3
 373        or      %g5, %g3, %g5
 374        ba,pt   %xcc, etrap
 375         rd     %pc, %g7
 376        mov     %l4, %o1
 377        mov     %l5, %o2
 378        call    handle_lddfmna
 379         add    %sp, PTREGS_OFF, %o0
 380        ba,a,pt %xcc, rtrap
 381
 382        /* Unaligned std float, tl0. */
 383sun4v_stdfmna:
 384        ldxa    [%g0] ASI_SCRATCHPAD, %g2
 385        ldx     [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
 386        ldx     [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
 387        ldx     [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
 388        sllx    %g3, 16, %g3
 389        or      %g5, %g3, %g5
 390        ba,pt   %xcc, etrap
 391         rd     %pc, %g7
 392        mov     %l4, %o1
 393        mov     %l5, %o2
 394        call    handle_stdfmna
 395         add    %sp, PTREGS_OFF, %o0
 396        ba,a,pt %xcc, rtrap
 397
 398#define BRANCH_ALWAYS   0x10680000
 399#define NOP             0x01000000
 400#define SUN4V_DO_PATCH(OLD, NEW)        \
 401        sethi   %hi(NEW), %g1; \
 402        or      %g1, %lo(NEW), %g1; \
 403        sethi   %hi(OLD), %g2; \
 404        or      %g2, %lo(OLD), %g2; \
 405        sub     %g1, %g2, %g1; \
 406        sethi   %hi(BRANCH_ALWAYS), %g3; \
 407        sll     %g1, 11, %g1; \
 408        srl     %g1, 11 + 2, %g1; \
 409        or      %g3, %lo(BRANCH_ALWAYS), %g3; \
 410        or      %g3, %g1, %g3; \
 411        stw     %g3, [%g2]; \
 412        sethi   %hi(NOP), %g3; \
 413        or      %g3, %lo(NOP), %g3; \
 414        stw     %g3, [%g2 + 0x4]; \
 415        flush   %g2;
 416
 417        .globl  sun4v_patch_tlb_handlers
 418        .type   sun4v_patch_tlb_handlers,#function
 419sun4v_patch_tlb_handlers:
 420        SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
 421        SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
 422        SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
 423        SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
 424        SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
 425        SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
 426        SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
 427        SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
 428        SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
 429        SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
 430        SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
 431        SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
 432        SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
 433        SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
 434        SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
 435        retl
 436         nop
 437        .size   sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
 438