LXR linux/arch/sparc/kernel/tsb.S

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/* tsb.S: Sparc64 TSB table handling.
   3 *
   4 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
   5 */
   6
   7
   8#include <asm/tsb.h>
   9#include <asm/hypervisor.h>
  10#include <asm/page.h>
  11#include <asm/cpudata.h>
  12#include <asm/mmu.h>
  13
  14        .text
  15        .align  32
  16
  17        /* Invoked from TLB miss handler, we are in the
  18         * MMU global registers and they are setup like
  19         * this:
  20         *
  21         * %g1: TSB entry pointer
  22         * %g2: available temporary
  23         * %g3: FAULT_CODE_{D,I}TLB
  24         * %g4: available temporary
  25         * %g5: available temporary
  26         * %g6: TAG TARGET
  27         * %g7: available temporary, will be loaded by us with
  28         *      the physical address base of the linux page
  29         *      tables for the current address space
  30         */
  31tsb_miss_dtlb:
  32        mov             TLB_TAG_ACCESS, %g4
  33        ldxa            [%g4] ASI_DMMU, %g4
  34        srlx            %g4, PAGE_SHIFT, %g4
  35        ba,pt           %xcc, tsb_miss_page_table_walk
  36         sllx           %g4, PAGE_SHIFT, %g4
  37
  38tsb_miss_itlb:
  39        mov             TLB_TAG_ACCESS, %g4
  40        ldxa            [%g4] ASI_IMMU, %g4
  41        srlx            %g4, PAGE_SHIFT, %g4
  42        ba,pt           %xcc, tsb_miss_page_table_walk
  43         sllx           %g4, PAGE_SHIFT, %g4
  44
  45        /* At this point we have:
  46         * %g1 --       PAGE_SIZE TSB entry address
  47         * %g3 --       FAULT_CODE_{D,I}TLB
  48         * %g4 --       missing virtual address
  49         * %g6 --       TAG TARGET (vaddr >> 22)
  50         */
  51tsb_miss_page_table_walk:
  52        TRAP_LOAD_TRAP_BLOCK(%g7, %g5)
  53
  54        /* Before committing to a full page table walk,
  55         * check the huge page TSB.
  56         */
  57#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
  58
  59661:    ldx             [%g7 + TRAP_PER_CPU_TSB_HUGE], %g5
  60        nop
  61        .section        .sun4v_2insn_patch, "ax"
  62        .word           661b
  63        mov             SCRATCHPAD_UTSBREG2, %g5
  64        ldxa            [%g5] ASI_SCRATCHPAD, %g5
  65        .previous
  66
  67        cmp             %g5, -1
  68        be,pt           %xcc, 80f
  69         nop
  70
  71        /* We need an aligned pair of registers containing 2 values
  72         * which can be easily rematerialized.  %g6 and %g7 foot the
  73         * bill just nicely.  We'll save %g6 away into %g2 for the
  74         * huge page TSB TAG comparison.
  75         *
  76         * Perform a huge page TSB lookup.
  77         */
  78        mov             %g6, %g2
  79        and             %g5, 0x7, %g6
  80        mov             512, %g7
  81        andn            %g5, 0x7, %g5
  82        sllx            %g7, %g6, %g7
  83        srlx            %g4, REAL_HPAGE_SHIFT, %g6
  84        sub             %g7, 1, %g7
  85        and             %g6, %g7, %g6
  86        sllx            %g6, 4, %g6
  87        add             %g5, %g6, %g5
  88
  89        TSB_LOAD_QUAD(%g5, %g6)
  90        cmp             %g6, %g2
  91        be,a,pt         %xcc, tsb_tlb_reload
  92         mov            %g7, %g5
  93
  94        /* No match, remember the huge page TSB entry address,
  95         * and restore %g6 and %g7.
  96         */
  97        TRAP_LOAD_TRAP_BLOCK(%g7, %g6)
  98        srlx            %g4, 22, %g6
  9980:     stx             %g5, [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP]
 100
 101#endif
 102
 103        ldx             [%g7 + TRAP_PER_CPU_PGD_PADDR], %g7
 104
 105        /* At this point we have:
 106         * %g1 --       TSB entry address
 107         * %g3 --       FAULT_CODE_{D,I}TLB
 108         * %g4 --       missing virtual address
 109         * %g6 --       TAG TARGET (vaddr >> 22)
 110         * %g7 --       page table physical address
 111         *
 112         * We know that both the base PAGE_SIZE TSB and the HPAGE_SIZE
 113         * TSB both lack a matching entry.
 114         */
 115tsb_miss_page_table_walk_sun4v_fastpath:
 116        USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
 117
 118        /* Valid PTE is now in %g5.  */
 119
 120#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 121        sethi           %uhi(_PAGE_PMD_HUGE | _PAGE_PUD_HUGE), %g7
 122        sllx            %g7, 32, %g7
 123
 124        andcc           %g5, %g7, %g0
 125        be,pt           %xcc, 60f
 126         nop
 127
 128        /* It is a huge page, use huge page TSB entry address we
 129         * calculated above.  If the huge page TSB has not been
 130         * allocated, setup a trap stack and call hugetlb_setup()
 131         * to do so, then return from the trap to replay the TLB
 132         * miss.
 133         *
 134         * This is necessary to handle the case of transparent huge
 135         * pages where we don't really have a non-atomic context
 136         * in which to allocate the hugepage TSB hash table.  When
 137         * the 'mm' faults in the hugepage for the first time, we
 138         * thus handle it here.  This also makes sure that we can
 139         * allocate the TSB hash table on the correct NUMA node.
 140         */
 141        TRAP_LOAD_TRAP_BLOCK(%g7, %g2)
 142        ldx             [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g1
 143        cmp             %g1, -1
 144        bne,pt          %xcc, 60f
 145         nop
 146
 147661:    rdpr            %pstate, %g5
 148        wrpr            %g5, PSTATE_AG | PSTATE_MG, %pstate
 149        .section        .sun4v_2insn_patch, "ax"
 150        .word           661b
 151        SET_GL(1)
 152        nop
 153        .previous
 154
 155        rdpr    %tl, %g7
 156        cmp     %g7, 1
 157        bne,pn  %xcc, winfix_trampoline
 158         mov    %g3, %g4
 159        ba,pt   %xcc, etrap
 160         rd     %pc, %g7
 161        call    hugetlb_setup
 162         add    %sp, PTREGS_OFF, %o0
 163        ba,pt   %xcc, rtrap
 164         nop
 165
 16660:
 167#endif
 168
 169        /* At this point we have:
 170         * %g1 --       TSB entry address
 171         * %g3 --       FAULT_CODE_{D,I}TLB
 172         * %g5 --       valid PTE
 173         * %g6 --       TAG TARGET (vaddr >> 22)
 174         */
 175tsb_reload:
 176        TSB_LOCK_TAG(%g1, %g2, %g7)
 177        TSB_WRITE(%g1, %g5, %g6)
 178
 179        /* Finally, load TLB and return from trap.  */
 180tsb_tlb_reload:
 181        cmp             %g3, FAULT_CODE_DTLB
 182        bne,pn          %xcc, tsb_itlb_load
 183         nop
 184
 185tsb_dtlb_load:
 186
 187661:    stxa            %g5, [%g0] ASI_DTLB_DATA_IN
 188        retry
 189        .section        .sun4v_2insn_patch, "ax"
 190        .word           661b
 191        nop
 192        nop
 193        .previous
 194
 195        /* For sun4v the ASI_DTLB_DATA_IN store and the retry
 196         * instruction get nop'd out and we get here to branch
 197         * to the sun4v tlb load code.  The registers are setup
 198         * as follows:
 199         *
 200         * %g4: vaddr
 201         * %g5: PTE
 202         * %g6: TAG
 203         *
 204         * The sun4v TLB load wants the PTE in %g3 so we fix that
 205         * up here.
 206         */
 207        ba,pt           %xcc, sun4v_dtlb_load
 208         mov            %g5, %g3
 209
 210tsb_itlb_load:
 211        /* Executable bit must be set.  */
 212661:    sethi           %hi(_PAGE_EXEC_4U), %g4
 213        andcc           %g5, %g4, %g0
 214        .section        .sun4v_2insn_patch, "ax"
 215        .word           661b
 216        andcc           %g5, _PAGE_EXEC_4V, %g0
 217        nop
 218        .previous
 219
 220        be,pn           %xcc, tsb_do_fault
 221         nop
 222
 223661:    stxa            %g5, [%g0] ASI_ITLB_DATA_IN
 224        retry
 225        .section        .sun4v_2insn_patch, "ax"
 226        .word           661b
 227        nop
 228        nop
 229        .previous
 230
 231        /* For sun4v the ASI_ITLB_DATA_IN store and the retry
 232         * instruction get nop'd out and we get here to branch
 233         * to the sun4v tlb load code.  The registers are setup
 234         * as follows:
 235         *
 236         * %g4: vaddr
 237         * %g5: PTE
 238         * %g6: TAG
 239         *
 240         * The sun4v TLB load wants the PTE in %g3 so we fix that
 241         * up here.
 242         */
 243        ba,pt           %xcc, sun4v_itlb_load
 244         mov            %g5, %g3
 245
 246        /* No valid entry in the page tables, do full fault
 247         * processing.
 248         */
 249
 250        .globl          tsb_do_fault
 251tsb_do_fault:
 252        cmp             %g3, FAULT_CODE_DTLB
 253
 254661:    rdpr            %pstate, %g5
 255        wrpr            %g5, PSTATE_AG | PSTATE_MG, %pstate
 256        .section        .sun4v_2insn_patch, "ax"
 257        .word           661b
 258        SET_GL(1)
 259        ldxa            [%g0] ASI_SCRATCHPAD, %g4
 260        .previous
 261
 262        bne,pn          %xcc, tsb_do_itlb_fault
 263         nop
 264
 265tsb_do_dtlb_fault:
 266        rdpr    %tl, %g3
 267        cmp     %g3, 1
 268
 269661:    mov     TLB_TAG_ACCESS, %g4
 270        ldxa    [%g4] ASI_DMMU, %g5
 271        .section .sun4v_2insn_patch, "ax"
 272        .word   661b
 273        ldx     [%g4 + HV_FAULT_D_ADDR_OFFSET], %g5
 274        nop
 275        .previous
 276
 277        /* Clear context ID bits.  */
 278        srlx            %g5, PAGE_SHIFT, %g5
 279        sllx            %g5, PAGE_SHIFT, %g5
 280
 281        be,pt   %xcc, sparc64_realfault_common
 282         mov    FAULT_CODE_DTLB, %g4
 283        ba,pt   %xcc, winfix_trampoline
 284         nop
 285
 286tsb_do_itlb_fault:
 287        rdpr    %tpc, %g5
 288        ba,pt   %xcc, sparc64_realfault_common
 289         mov    FAULT_CODE_ITLB, %g4
 290
 291        .globl  sparc64_realfault_common
 292sparc64_realfault_common:
 293        /* fault code in %g4, fault address in %g5, etrap will
 294         * preserve these two values in %l4 and %l5 respectively
 295         */
 296        ba,pt   %xcc, etrap                     ! Save trap state
 2971:       rd     %pc, %g7                        ! ...
 298        stb     %l4, [%g6 + TI_FAULT_CODE]      ! Save fault code
 299        stx     %l5, [%g6 + TI_FAULT_ADDR]      ! Save fault address
 300        call    do_sparc64_fault                ! Call fault handler
 301         add    %sp, PTREGS_OFF, %o0            ! Compute pt_regs arg
 302        ba,pt   %xcc, rtrap                     ! Restore cpu state
 303         nop                                    ! Delay slot (fill me)
 304
 305winfix_trampoline:
 306        rdpr    %tpc, %g3                       ! Prepare winfixup TNPC
 307        or      %g3, 0x7c, %g3                  ! Compute branch offset
 308        wrpr    %g3, %tnpc                      ! Write it into TNPC
 309        done                                    ! Trap return
 310
 311        /* Insert an entry into the TSB.
 312         *
 313         * %o0: TSB entry pointer (virt or phys address)
 314         * %o1: tag
 315         * %o2: pte
 316         */
 317        .align  32
 318        .globl  __tsb_insert
 319__tsb_insert:
 320        rdpr    %pstate, %o5
 321        wrpr    %o5, PSTATE_IE, %pstate
 322        TSB_LOCK_TAG(%o0, %g2, %g3)
 323        TSB_WRITE(%o0, %o2, %o1)
 324        wrpr    %o5, %pstate
 325        retl
 326         nop
 327        .size   __tsb_insert, .-__tsb_insert
 328
 329        /* Flush the given TSB entry if it has the matching
 330         * tag.
 331         *
 332         * %o0: TSB entry pointer (virt or phys address)
 333         * %o1: tag
 334         */
 335        .align  32
 336        .globl  tsb_flush
 337        .type   tsb_flush,#function
 338tsb_flush:
 339        sethi   %hi(TSB_TAG_LOCK_HIGH), %g2
 3401:      TSB_LOAD_TAG(%o0, %g1)
 341        srlx    %g1, 32, %o3
 342        andcc   %o3, %g2, %g0
 343        bne,pn  %icc, 1b
 344         nop
 345        cmp     %g1, %o1
 346        mov     1, %o3
 347        bne,pt  %xcc, 2f
 348         sllx   %o3, TSB_TAG_INVALID_BIT, %o3
 349        TSB_CAS_TAG(%o0, %g1, %o3)
 350        cmp     %g1, %o3
 351        bne,pn  %xcc, 1b
 352         nop
 3532:      retl
 354         nop
 355        .size   tsb_flush, .-tsb_flush
 356
 357        /* Reload MMU related context switch state at
 358         * schedule() time.
 359         *
 360         * %o0: page table physical address
 361         * %o1: TSB base config pointer
 362         * %o2: TSB huge config pointer, or NULL if none
 363         * %o3: Hypervisor TSB descriptor physical address
 364         * %o4: Secondary context to load, if non-zero
 365         *
 366         * We have to run this whole thing with interrupts
 367         * disabled so that the current cpu doesn't change
 368         * due to preemption.
 369         */
 370        .align  32
 371        .globl  __tsb_context_switch
 372        .type   __tsb_context_switch,#function
 373__tsb_context_switch:
 374        rdpr    %pstate, %g1
 375        wrpr    %g1, PSTATE_IE, %pstate
 376
 377        brz,pn  %o4, 1f
 378         mov    SECONDARY_CONTEXT, %o5
 379
 380661:    stxa    %o4, [%o5] ASI_DMMU
 381        .section .sun4v_1insn_patch, "ax"
 382        .word   661b
 383        stxa    %o4, [%o5] ASI_MMU
 384        .previous
 385        flush   %g6
 386
 3871:
 388        TRAP_LOAD_TRAP_BLOCK(%g2, %g3)
 389
 390        stx     %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR]
 391
 392        ldx     [%o1 + TSB_CONFIG_REG_VAL], %o0
 393        brz,pt  %o2, 1f
 394         mov    -1, %g3
 395
 396        ldx     [%o2 + TSB_CONFIG_REG_VAL], %g3
 397
 3981:      stx     %g3, [%g2 + TRAP_PER_CPU_TSB_HUGE]
 399
 400        sethi   %hi(tlb_type), %g2
 401        lduw    [%g2 + %lo(tlb_type)], %g2
 402        cmp     %g2, 3
 403        bne,pt  %icc, 50f
 404         nop
 405
 406        /* Hypervisor TSB switch. */
 407        mov     SCRATCHPAD_UTSBREG1, %o5
 408        stxa    %o0, [%o5] ASI_SCRATCHPAD
 409        mov     SCRATCHPAD_UTSBREG2, %o5
 410        stxa    %g3, [%o5] ASI_SCRATCHPAD
 411
 412        mov     2, %o0
 413        cmp     %g3, -1
 414        move    %xcc, 1, %o0
 415
 416        mov     HV_FAST_MMU_TSB_CTXNON0, %o5
 417        mov     %o3, %o1
 418        ta      HV_FAST_TRAP
 419
 420        /* Finish up.  */
 421        ba,pt   %xcc, 9f
 422         nop
 423
 424        /* SUN4U TSB switch.  */
 42550:     mov     TSB_REG, %o5
 426        stxa    %o0, [%o5] ASI_DMMU
 427        membar  #Sync
 428        stxa    %o0, [%o5] ASI_IMMU
 429        membar  #Sync
 430
 4312:      ldx     [%o1 + TSB_CONFIG_MAP_VADDR], %o4
 432        brz     %o4, 9f
 433         ldx    [%o1 + TSB_CONFIG_MAP_PTE], %o5
 434
 435        sethi   %hi(sparc64_highest_unlocked_tlb_ent), %g2
 436        mov     TLB_TAG_ACCESS, %g3
 437        lduw    [%g2 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2
 438        stxa    %o4, [%g3] ASI_DMMU
 439        membar  #Sync
 440        sllx    %g2, 3, %g2
 441        stxa    %o5, [%g2] ASI_DTLB_DATA_ACCESS
 442        membar  #Sync
 443
 444        brz,pt  %o2, 9f
 445         nop
 446
 447        ldx     [%o2 + TSB_CONFIG_MAP_VADDR], %o4
 448        ldx     [%o2 + TSB_CONFIG_MAP_PTE], %o5
 449        mov     TLB_TAG_ACCESS, %g3
 450        stxa    %o4, [%g3] ASI_DMMU
 451        membar  #Sync
 452        sub     %g2, (1 << 3), %g2
 453        stxa    %o5, [%g2] ASI_DTLB_DATA_ACCESS
 454        membar  #Sync
 455
 4569:
 457        wrpr    %g1, %pstate
 458
 459        retl
 460         nop
 461        .size   __tsb_context_switch, .-__tsb_context_switch
 462
 463#define TSB_PASS_BITS   ((1 << TSB_TAG_LOCK_BIT) | \
 464                         (1 << TSB_TAG_INVALID_BIT))
 465
 466        .align  32
 467        .globl  copy_tsb
 468        .type   copy_tsb,#function
 469copy_tsb:               /* %o0=old_tsb_base, %o1=old_tsb_size
 470                         * %o2=new_tsb_base, %o3=new_tsb_size
 471                         * %o4=page_size_shift
 472                         */
 473        sethi           %uhi(TSB_PASS_BITS), %g7
 474        srlx            %o3, 4, %o3
 475        add             %o0, %o1, %o1   /* end of old tsb */
 476        sllx            %g7, 32, %g7
 477        sub             %o3, 1, %o3     /* %o3 == new tsb hash mask */
 478
 479        mov             %o4, %g1        /* page_size_shift */
 480
 481661:    prefetcha       [%o0] ASI_N, #one_read
 482        .section        .tsb_phys_patch, "ax"
 483        .word           661b
 484        prefetcha       [%o0] ASI_PHYS_USE_EC, #one_read
 485        .previous
 486
 48790:     andcc           %o0, (64 - 1), %g0
 488        bne             1f
 489         add            %o0, 64, %o5
 490
 491661:    prefetcha       [%o5] ASI_N, #one_read
 492        .section        .tsb_phys_patch, "ax"
 493        .word           661b
 494        prefetcha       [%o5] ASI_PHYS_USE_EC, #one_read
 495        .previous
 496
 4971:      TSB_LOAD_QUAD(%o0, %g2)         /* %g2/%g3 == TSB entry */
 498        andcc           %g2, %g7, %g0   /* LOCK or INVALID set? */
 499        bne,pn          %xcc, 80f       /* Skip it */
 500         sllx           %g2, 22, %o4    /* TAG --> VADDR */
 501
 502        /* This can definitely be computed faster... */
 503        srlx            %o0, 4, %o5     /* Build index */
 504        and             %o5, 511, %o5   /* Mask index */
 505        sllx            %o5, %g1, %o5   /* Put into vaddr position */
 506        or              %o4, %o5, %o4   /* Full VADDR. */
 507        srlx            %o4, %g1, %o4   /* Shift down to create index */
 508        and             %o4, %o3, %o4   /* Mask with new_tsb_nents-1 */
 509        sllx            %o4, 4, %o4     /* Shift back up into tsb ent offset */
 510        TSB_STORE(%o2 + %o4, %g2)       /* Store TAG */
 511        add             %o4, 0x8, %o4   /* Advance to TTE */
 512        TSB_STORE(%o2 + %o4, %g3)       /* Store TTE */
 513
 51480:     add             %o0, 16, %o0
 515        cmp             %o0, %o1
 516        bne,pt          %xcc, 90b
 517         nop
 518
 519        retl
 520         nop
 521        .size           copy_tsb, .-copy_tsb
 522
 523        /* Set the invalid bit in all TSB entries.  */
 524        .align          32
 525        .globl          tsb_init
 526        .type           tsb_init,#function
 527tsb_init:               /* %o0 = TSB vaddr, %o1 = size in bytes */
 528        prefetch        [%o0 + 0x000], #n_writes
 529        mov             1, %g1
 530        prefetch        [%o0 + 0x040], #n_writes
 531        sllx            %g1, TSB_TAG_INVALID_BIT, %g1
 532        prefetch        [%o0 + 0x080], #n_writes
 5331:      prefetch        [%o0 + 0x0c0], #n_writes
 534        stx             %g1, [%o0 + 0x00]
 535        stx             %g1, [%o0 + 0x10]
 536        stx             %g1, [%o0 + 0x20]
 537        stx             %g1, [%o0 + 0x30]
 538        prefetch        [%o0 + 0x100], #n_writes
 539        stx             %g1, [%o0 + 0x40]
 540        stx             %g1, [%o0 + 0x50]
 541        stx             %g1, [%o0 + 0x60]
 542        stx             %g1, [%o0 + 0x70]
 543        prefetch        [%o0 + 0x140], #n_writes
 544        stx             %g1, [%o0 + 0x80]
 545        stx             %g1, [%o0 + 0x90]
 546        stx             %g1, [%o0 + 0xa0]
 547        stx             %g1, [%o0 + 0xb0]
 548        prefetch        [%o0 + 0x180], #n_writes
 549        stx             %g1, [%o0 + 0xc0]
 550        stx             %g1, [%o0 + 0xd0]
 551        stx             %g1, [%o0 + 0xe0]
 552        stx             %g1, [%o0 + 0xf0]
 553        subcc           %o1, 0x100, %o1
 554        bne,pt          %xcc, 1b
 555         add            %o0, 0x100, %o0
 556        retl
 557         nop
 558        nop
 559        nop
 560        .size           tsb_init, .-tsb_init
 561
 562        .globl          NGtsb_init
 563        .type           NGtsb_init,#function
 564NGtsb_init:
 565        rd              %asi, %g2
 566        mov             1, %g1
 567        wr              %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
 568        sllx            %g1, TSB_TAG_INVALID_BIT, %g1
 5691:      stxa            %g1, [%o0 + 0x00] %asi
 570        stxa            %g1, [%o0 + 0x10] %asi
 571        stxa            %g1, [%o0 + 0x20] %asi
 572        stxa            %g1, [%o0 + 0x30] %asi
 573        stxa            %g1, [%o0 + 0x40] %asi
 574        stxa            %g1, [%o0 + 0x50] %asi
 575        stxa            %g1, [%o0 + 0x60] %asi
 576        stxa            %g1, [%o0 + 0x70] %asi
 577        stxa            %g1, [%o0 + 0x80] %asi
 578        stxa            %g1, [%o0 + 0x90] %asi
 579        stxa            %g1, [%o0 + 0xa0] %asi
 580        stxa            %g1, [%o0 + 0xb0] %asi
 581        stxa            %g1, [%o0 + 0xc0] %asi
 582        stxa            %g1, [%o0 + 0xd0] %asi
 583        stxa            %g1, [%o0 + 0xe0] %asi
 584        stxa            %g1, [%o0 + 0xf0] %asi
 585        subcc           %o1, 0x100, %o1
 586        bne,pt          %xcc, 1b
 587         add            %o0, 0x100, %o0
 588        membar          #Sync
 589        retl
 590         wr             %g2, 0x0, %asi
 591        .size           NGtsb_init, .-NGtsb_init
 592