LXR linux/arch/sparc/kernel/ktlb.S

   1/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
   2 *
   3 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
   4 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
   5 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
   6 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
   7 */
   8
   9#include <asm/head.h>
  10#include <asm/asi.h>
  11#include <asm/page.h>
  12#include <asm/pgtable.h>
  13#include <asm/tsb.h>
  14
  15        .text
  16        .align          32
  17
  18kvmap_itlb:
  19        /* g6: TAG TARGET */
  20        mov             TLB_TAG_ACCESS, %g4
  21        ldxa            [%g4] ASI_IMMU, %g4
  22
  23        /* sun4v_itlb_miss branches here with the missing virtual
  24         * address already loaded into %g4
  25         */
  26kvmap_itlb_4v:
  27
  28kvmap_itlb_nonlinear:
  29        /* Catch kernel NULL pointer calls.  */
  30        sethi           %hi(PAGE_SIZE), %g5
  31        cmp             %g4, %g5
  32        bleu,pn         %xcc, kvmap_dtlb_longpath
  33         nop
  34
  35        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
  36
  37kvmap_itlb_tsb_miss:
  38        sethi           %hi(LOW_OBP_ADDRESS), %g5
  39        cmp             %g4, %g5
  40        blu,pn          %xcc, kvmap_itlb_vmalloc_addr
  41         mov            0x1, %g5
  42        sllx            %g5, 32, %g5
  43        cmp             %g4, %g5
  44        blu,pn          %xcc, kvmap_itlb_obp
  45         nop
  46
  47kvmap_itlb_vmalloc_addr:
  48        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
  49
  50        KTSB_LOCK_TAG(%g1, %g2, %g7)
  51
  52        /* Load and check PTE.  */
  53        ldxa            [%g5] ASI_PHYS_USE_EC, %g5
  54        mov             1, %g7
  55        sllx            %g7, TSB_TAG_INVALID_BIT, %g7
  56        brgez,a,pn      %g5, kvmap_itlb_longpath
  57         KTSB_STORE(%g1, %g7)
  58
  59        KTSB_WRITE(%g1, %g5, %g6)
  60
  61        /* fallthrough to TLB load */
  62
  63kvmap_itlb_load:
  64
  65661:    stxa            %g5, [%g0] ASI_ITLB_DATA_IN
  66        retry
  67        .section        .sun4v_2insn_patch, "ax"
  68        .word           661b
  69        nop
  70        nop
  71        .previous
  72
  73        /* For sun4v the ASI_ITLB_DATA_IN store and the retry
  74         * instruction get nop'd out and we get here to branch
  75         * to the sun4v tlb load code.  The registers are setup
  76         * as follows:
  77         *
  78         * %g4: vaddr
  79         * %g5: PTE
  80         * %g6: TAG
  81         *
  82         * The sun4v TLB load wants the PTE in %g3 so we fix that
  83         * up here.
  84         */
  85        ba,pt           %xcc, sun4v_itlb_load
  86         mov            %g5, %g3
  87
  88kvmap_itlb_longpath:
  89
  90661:    rdpr    %pstate, %g5
  91        wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
  92        .section .sun4v_2insn_patch, "ax"
  93        .word   661b
  94        SET_GL(1)
  95        nop
  96        .previous
  97
  98        rdpr    %tpc, %g5
  99        ba,pt   %xcc, sparc64_realfault_common
 100         mov    FAULT_CODE_ITLB, %g4
 101
 102kvmap_itlb_obp:
 103        OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
 104
 105        KTSB_LOCK_TAG(%g1, %g2, %g7)
 106
 107        KTSB_WRITE(%g1, %g5, %g6)
 108
 109        ba,pt           %xcc, kvmap_itlb_load
 110         nop
 111
 112kvmap_dtlb_obp:
 113        OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
 114
 115        KTSB_LOCK_TAG(%g1, %g2, %g7)
 116
 117        KTSB_WRITE(%g1, %g5, %g6)
 118
 119        ba,pt           %xcc, kvmap_dtlb_load
 120         nop
 121
 122        .align          32
 123kvmap_dtlb_tsb4m_load:
 124        KTSB_LOCK_TAG(%g1, %g2, %g7)
 125        KTSB_WRITE(%g1, %g5, %g6)
 126        ba,pt           %xcc, kvmap_dtlb_load
 127         nop
 128
 129kvmap_dtlb:
 130        /* %g6: TAG TARGET */
 131        mov             TLB_TAG_ACCESS, %g4
 132        ldxa            [%g4] ASI_DMMU, %g4
 133
 134        /* sun4v_dtlb_miss branches here with the missing virtual
 135         * address already loaded into %g4
 136         */
 137kvmap_dtlb_4v:
 138        brgez,pn        %g4, kvmap_dtlb_nonlinear
 139         nop
 140
 141#ifdef CONFIG_DEBUG_PAGEALLOC
 142        /* Index through the base page size TSB even for linear
 143         * mappings when using page allocation debugging.
 144         */
 145        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
 146#else
 147        /* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
 148        KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
 149#endif
 150        /* TSB entry address left in %g1, lookup linear PTE.
 151         * Must preserve %g1 and %g6 (TAG).
 152         */
 153kvmap_dtlb_tsb4m_miss:
 154        /* Clear the PAGE_OFFSET top virtual bits, shift
 155         * down to get PFN, and make sure PFN is in range.
 156         */
 157        sllx            %g4, 21, %g5
 158
 159        /* Check to see if we know about valid memory at the 4MB
 160         * chunk this physical address will reside within.
 161         */
 162        srlx            %g5, 21 + 41, %g2
 163        brnz,pn         %g2, kvmap_dtlb_longpath
 164         nop
 165
 166        /* This unconditional branch and delay-slot nop gets patched
 167         * by the sethi sequence once the bitmap is properly setup.
 168         */
 169        .globl          valid_addr_bitmap_insn
 170valid_addr_bitmap_insn:
 171        ba,pt           %xcc, 2f
 172         nop
 173        .subsection     2
 174        .globl          valid_addr_bitmap_patch
 175valid_addr_bitmap_patch:
 176        sethi           %hi(sparc64_valid_addr_bitmap), %g7
 177        or              %g7, %lo(sparc64_valid_addr_bitmap), %g7
 178        .previous
 179
 180        srlx            %g5, 21 + 22, %g2
 181        srlx            %g2, 6, %g5
 182        and             %g2, 63, %g2
 183        sllx            %g5, 3, %g5
 184        ldx             [%g7 + %g5], %g5
 185        mov             1, %g7
 186        sllx            %g7, %g2, %g7
 187        andcc           %g5, %g7, %g0
 188        be,pn           %xcc, kvmap_dtlb_longpath
 189
 1902:       sethi          %hi(kpte_linear_bitmap), %g2
 191        or              %g2, %lo(kpte_linear_bitmap), %g2
 192
 193        /* Get the 256MB physical address index. */
 194        sllx            %g4, 21, %g5
 195        mov             1, %g7
 196        srlx            %g5, 21 + 28, %g5
 197
 198        /* Don't try this at home kids... this depends upon srlx
 199         * only taking the low 6 bits of the shift count in %g5.
 200         */
 201        sllx            %g7, %g5, %g7
 202
 203        /* Divide by 64 to get the offset into the bitmask.  */
 204        srlx            %g5, 6, %g5
 205        sllx            %g5, 3, %g5
 206
 207        /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */
 208        ldx             [%g2 + %g5], %g2
 209        andcc           %g2, %g7, %g0
 210        sethi           %hi(kern_linear_pte_xor), %g5
 211        or              %g5, %lo(kern_linear_pte_xor), %g5
 212        bne,a,pt        %xcc, 1f
 213         add            %g5, 8, %g5
 214
 2151:      ldx             [%g5], %g2
 216
 217        .globl          kvmap_linear_patch
 218kvmap_linear_patch:
 219        ba,pt           %xcc, kvmap_dtlb_tsb4m_load
 220         xor            %g2, %g4, %g5
 221
 222kvmap_dtlb_vmalloc_addr:
 223        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
 224
 225        KTSB_LOCK_TAG(%g1, %g2, %g7)
 226
 227        /* Load and check PTE.  */
 228        ldxa            [%g5] ASI_PHYS_USE_EC, %g5
 229        mov             1, %g7
 230        sllx            %g7, TSB_TAG_INVALID_BIT, %g7
 231        brgez,a,pn      %g5, kvmap_dtlb_longpath
 232         KTSB_STORE(%g1, %g7)
 233
 234        KTSB_WRITE(%g1, %g5, %g6)
 235
 236        /* fallthrough to TLB load */
 237
 238kvmap_dtlb_load:
 239
 240661:    stxa            %g5, [%g0] ASI_DTLB_DATA_IN     ! Reload TLB
 241        retry
 242        .section        .sun4v_2insn_patch, "ax"
 243        .word           661b
 244        nop
 245        nop
 246        .previous
 247
 248        /* For sun4v the ASI_DTLB_DATA_IN store and the retry
 249         * instruction get nop'd out and we get here to branch
 250         * to the sun4v tlb load code.  The registers are setup
 251         * as follows:
 252         *
 253         * %g4: vaddr
 254         * %g5: PTE
 255         * %g6: TAG
 256         *
 257         * The sun4v TLB load wants the PTE in %g3 so we fix that
 258         * up here.
 259         */
 260        ba,pt           %xcc, sun4v_dtlb_load
 261         mov            %g5, %g3
 262
 263#ifdef CONFIG_SPARSEMEM_VMEMMAP
 264kvmap_vmemmap:
 265        sub             %g4, %g5, %g5
 266        srlx            %g5, 22, %g5
 267        sethi           %hi(vmemmap_table), %g1
 268        sllx            %g5, 3, %g5
 269        or              %g1, %lo(vmemmap_table), %g1
 270        ba,pt           %xcc, kvmap_dtlb_load
 271         ldx            [%g1 + %g5], %g5
 272#endif
 273
 274kvmap_dtlb_nonlinear:
 275        /* Catch kernel NULL pointer derefs.  */
 276        sethi           %hi(PAGE_SIZE), %g5
 277        cmp             %g4, %g5
 278        bleu,pn         %xcc, kvmap_dtlb_longpath
 279         nop
 280
 281#ifdef CONFIG_SPARSEMEM_VMEMMAP
 282        /* Do not use the TSB for vmemmap.  */
 283        mov             (VMEMMAP_BASE >> 40), %g5
 284        sllx            %g5, 40, %g5
 285        cmp             %g4,%g5
 286        bgeu,pn         %xcc, kvmap_vmemmap
 287         nop
 288#endif
 289
 290        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
 291
 292kvmap_dtlb_tsbmiss:
 293        sethi           %hi(MODULES_VADDR), %g5
 294        cmp             %g4, %g5
 295        blu,pn          %xcc, kvmap_dtlb_longpath
 296         mov            (VMALLOC_END >> 40), %g5
 297        sllx            %g5, 40, %g5
 298        cmp             %g4, %g5
 299        bgeu,pn         %xcc, kvmap_dtlb_longpath
 300         nop
 301
 302kvmap_check_obp:
 303        sethi           %hi(LOW_OBP_ADDRESS), %g5
 304        cmp             %g4, %g5
 305        blu,pn          %xcc, kvmap_dtlb_vmalloc_addr
 306         mov            0x1, %g5
 307        sllx            %g5, 32, %g5
 308        cmp             %g4, %g5
 309        blu,pn          %xcc, kvmap_dtlb_obp
 310         nop
 311        ba,pt           %xcc, kvmap_dtlb_vmalloc_addr
 312         nop
 313
 314kvmap_dtlb_longpath:
 315
 316661:    rdpr    %pstate, %g5
 317        wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
 318        .section .sun4v_2insn_patch, "ax"
 319        .word   661b
 320        SET_GL(1)
 321        ldxa            [%g0] ASI_SCRATCHPAD, %g5
 322        .previous
 323
 324        rdpr    %tl, %g3
 325        cmp     %g3, 1
 326
 327661:    mov     TLB_TAG_ACCESS, %g4
 328        ldxa    [%g4] ASI_DMMU, %g5
 329        .section .sun4v_2insn_patch, "ax"
 330        .word   661b
 331        ldx     [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
 332        nop
 333        .previous
 334
 335        be,pt   %xcc, sparc64_realfault_common
 336         mov    FAULT_CODE_DTLB, %g4
 337        ba,pt   %xcc, winfix_trampoline
 338         nop
 339