LXR linux/arch/sparc/mm/hypersparc.S

   1/*
   2 * hypersparc.S: High speed Hypersparc mmu/cache operations.
   3 *
   4 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
   5 */
   6
   7#include <asm/ptrace.h>
   8#include <asm/psr.h>
   9#include <asm/asm-offsets.h>
  10#include <asm/asi.h>
  11#include <asm/page.h>
  12#include <asm/pgtsrmmu.h>
  13#include <linux/init.h>
  14
  15        .text
  16        .align  4
  17
  18        .globl  hypersparc_flush_cache_all, hypersparc_flush_cache_mm
  19        .globl  hypersparc_flush_cache_range, hypersparc_flush_cache_page
  20        .globl  hypersparc_flush_page_to_ram
  21        .globl  hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
  22        .globl  hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
  23        .globl  hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
  24
  25hypersparc_flush_cache_all:
  26        WINDOW_FLUSH(%g4, %g5)
  27        sethi   %hi(vac_cache_size), %g4
  28        ld      [%g4 + %lo(vac_cache_size)], %g5
  29        sethi   %hi(vac_line_size), %g1
  30        ld      [%g1 + %lo(vac_line_size)], %g2
  311:      
  32        subcc   %g5, %g2, %g5                   ! hyper_flush_unconditional_combined
  33        bne     1b
  34         sta    %g0, [%g5] ASI_M_FLUSH_CTX
  35        retl
  36         sta    %g0, [%g0] ASI_M_FLUSH_IWHOLE   ! hyper_flush_whole_icache
  37
  38        /* We expand the window flush to get maximum performance. */
  39hypersparc_flush_cache_mm:
  40#ifndef CONFIG_SMP
  41        ld      [%o0 + AOFF_mm_context], %g1
  42        cmp     %g1, -1
  43        be      hypersparc_flush_cache_mm_out
  44#endif
  45        WINDOW_FLUSH(%g4, %g5)
  46
  47        sethi   %hi(vac_line_size), %g1
  48        ld      [%g1 + %lo(vac_line_size)], %o1
  49        sethi   %hi(vac_cache_size), %g2
  50        ld      [%g2 + %lo(vac_cache_size)], %o0
  51        add     %o1, %o1, %g1
  52        add     %o1, %g1, %g2
  53        add     %o1, %g2, %g3
  54        add     %o1, %g3, %g4
  55        add     %o1, %g4, %g5
  56        add     %o1, %g5, %o4
  57        add     %o1, %o4, %o5
  58
  59        /* BLAMMO! */
  601:
  61        subcc   %o0, %o5, %o0                           ! hyper_flush_cache_user
  62        sta     %g0, [%o0 + %g0] ASI_M_FLUSH_USER
  63        sta     %g0, [%o0 + %o1] ASI_M_FLUSH_USER
  64        sta     %g0, [%o0 + %g1] ASI_M_FLUSH_USER
  65        sta     %g0, [%o0 + %g2] ASI_M_FLUSH_USER
  66        sta     %g0, [%o0 + %g3] ASI_M_FLUSH_USER
  67        sta     %g0, [%o0 + %g4] ASI_M_FLUSH_USER
  68        sta     %g0, [%o0 + %g5] ASI_M_FLUSH_USER
  69        bne     1b
  70         sta    %g0, [%o0 + %o4] ASI_M_FLUSH_USER
  71hypersparc_flush_cache_mm_out:
  72        retl
  73         nop
  74
  75        /* The things we do for performance... */
  76hypersparc_flush_cache_range:
  77        ld      [%o0 + 0x0], %o0                /* XXX vma->vm_mm, GROSS XXX */
  78#ifndef CONFIG_SMP
  79        ld      [%o0 + AOFF_mm_context], %g1
  80        cmp     %g1, -1
  81        be      hypersparc_flush_cache_range_out
  82#endif
  83        WINDOW_FLUSH(%g4, %g5)
  84
  85        sethi   %hi(vac_line_size), %g1
  86        ld      [%g1 + %lo(vac_line_size)], %o4
  87        sethi   %hi(vac_cache_size), %g2
  88        ld      [%g2 + %lo(vac_cache_size)], %o3
  89
  90        /* Here comes the fun part... */
  91        add     %o2, (PAGE_SIZE - 1), %o2
  92        andn    %o1, (PAGE_SIZE - 1), %o1
  93        add     %o4, %o4, %o5
  94        andn    %o2, (PAGE_SIZE - 1), %o2
  95        add     %o4, %o5, %g1
  96        sub     %o2, %o1, %g4
  97        add     %o4, %g1, %g2
  98        sll     %o3, 2, %g5
  99        add     %o4, %g2, %g3
 100        cmp     %g4, %g5
 101        add     %o4, %g3, %g4
 102        blu     0f
 103         add    %o4, %g4, %g5
 104        add     %o4, %g5, %g7
 105
 106        /* Flush entire user space, believe it or not this is quicker
 107         * than page at a time flushings for range > (cache_size<<2).
 108         */
 1091:
 110        subcc   %o3, %g7, %o3
 111        sta     %g0, [%o3 + %g0] ASI_M_FLUSH_USER
 112        sta     %g0, [%o3 + %o4] ASI_M_FLUSH_USER
 113        sta     %g0, [%o3 + %o5] ASI_M_FLUSH_USER
 114        sta     %g0, [%o3 + %g1] ASI_M_FLUSH_USER
 115        sta     %g0, [%o3 + %g2] ASI_M_FLUSH_USER
 116        sta     %g0, [%o3 + %g3] ASI_M_FLUSH_USER
 117        sta     %g0, [%o3 + %g4] ASI_M_FLUSH_USER
 118        bne     1b
 119         sta    %g0, [%o3 + %g5] ASI_M_FLUSH_USER
 120        retl
 121         nop
 122
 123        /* Below our threshold, flush one page at a time. */
 1240:
 125        ld      [%o0 + AOFF_mm_context], %o0
 126        mov     SRMMU_CTX_REG, %g7
 127        lda     [%g7] ASI_M_MMUREGS, %o3
 128        sta     %o0, [%g7] ASI_M_MMUREGS
 129        add     %o2, -PAGE_SIZE, %o0
 1301:
 131        or      %o0, 0x400, %g7
 132        lda     [%g7] ASI_M_FLUSH_PROBE, %g7
 133        orcc    %g7, 0, %g0
 134        be,a    3f
 135         mov    %o0, %o2
 136        add     %o4, %g5, %g7
 1372:
 138        sub     %o2, %g7, %o2
 139        sta     %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
 140        sta     %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
 141        sta     %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
 142        sta     %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
 143        sta     %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
 144        sta     %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
 145        andcc   %o2, 0xffc, %g0
 146        sta     %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
 147        bne     2b
 148         sta    %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
 1493:
 150        cmp     %o2, %o1
 151        bne     1b
 152         add    %o2, -PAGE_SIZE, %o0
 153        mov     SRMMU_FAULT_STATUS, %g5
 154        lda     [%g5] ASI_M_MMUREGS, %g0
 155        mov     SRMMU_CTX_REG, %g7
 156        sta     %o3, [%g7] ASI_M_MMUREGS
 157hypersparc_flush_cache_range_out:
 158        retl
 159         nop
 160
 161        /* HyperSparc requires a valid mapping where we are about to flush
 162         * in order to check for a physical tag match during the flush.
 163         */
 164        /* Verified, my ass... */
 165hypersparc_flush_cache_page:
 166        ld      [%o0 + 0x0], %o0                /* XXX vma->vm_mm, GROSS XXX */
 167        ld      [%o0 + AOFF_mm_context], %g2
 168#ifndef CONFIG_SMP
 169        cmp     %g2, -1
 170        be      hypersparc_flush_cache_page_out
 171#endif
 172        WINDOW_FLUSH(%g4, %g5)
 173
 174        sethi   %hi(vac_line_size), %g1
 175        ld      [%g1 + %lo(vac_line_size)], %o4
 176        mov     SRMMU_CTX_REG, %o3
 177        andn    %o1, (PAGE_SIZE - 1), %o1
 178        lda     [%o3] ASI_M_MMUREGS, %o2
 179        sta     %g2, [%o3] ASI_M_MMUREGS
 180        or      %o1, 0x400, %o5
 181        lda     [%o5] ASI_M_FLUSH_PROBE, %g1
 182        orcc    %g0, %g1, %g0
 183        be      2f
 184         add    %o4, %o4, %o5
 185        sub     %o1, -PAGE_SIZE, %o1
 186        add     %o4, %o5, %g1
 187        add     %o4, %g1, %g2
 188        add     %o4, %g2, %g3
 189        add     %o4, %g3, %g4
 190        add     %o4, %g4, %g5
 191        add     %o4, %g5, %g7
 192
 193        /* BLAMMO! */
 1941:
 195        sub     %o1, %g7, %o1
 196        sta     %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
 197        sta     %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
 198        sta     %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
 199        sta     %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
 200        sta     %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
 201        sta     %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
 202        andcc   %o1, 0xffc, %g0
 203        sta     %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
 204        bne     1b
 205         sta    %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
 2062:
 207        mov     SRMMU_FAULT_STATUS, %g7
 208        mov     SRMMU_CTX_REG, %g4
 209        lda     [%g7] ASI_M_MMUREGS, %g0
 210        sta     %o2, [%g4] ASI_M_MMUREGS
 211hypersparc_flush_cache_page_out:
 212        retl
 213         nop
 214
 215hypersparc_flush_sig_insns:
 216        flush   %o1
 217        retl
 218         flush  %o1 + 4
 219
 220        /* HyperSparc is copy-back. */
 221hypersparc_flush_page_to_ram:
 222        sethi   %hi(vac_line_size), %g1
 223        ld      [%g1 + %lo(vac_line_size)], %o4
 224        andn    %o0, (PAGE_SIZE - 1), %o0
 225        add     %o4, %o4, %o5
 226        or      %o0, 0x400, %g7
 227        lda     [%g7] ASI_M_FLUSH_PROBE, %g5
 228        add     %o4, %o5, %g1
 229        orcc    %g5, 0, %g0
 230        be      2f
 231         add    %o4, %g1, %g2
 232        add     %o4, %g2, %g3
 233        sub     %o0, -PAGE_SIZE, %o0
 234        add     %o4, %g3, %g4
 235        add     %o4, %g4, %g5
 236        add     %o4, %g5, %g7
 237
 238        /* BLAMMO! */
 2391:
 240        sub     %o0, %g7, %o0
 241        sta     %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
 242        sta     %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
 243        sta     %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
 244        sta     %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
 245        sta     %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
 246        sta     %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
 247        andcc   %o0, 0xffc, %g0
 248        sta     %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
 249        bne     1b
 250         sta    %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
 2512:
 252        mov     SRMMU_FAULT_STATUS, %g1
 253        retl
 254         lda    [%g1] ASI_M_MMUREGS, %g0
 255
 256        /* HyperSparc is IO cache coherent. */
 257hypersparc_flush_page_for_dma:
 258        retl
 259         nop
 260
 261        /* It was noted that at boot time a TLB flush all in a delay slot
 262         * can deliver an illegal instruction to the processor if the timing
 263         * is just right...
 264         */
 265hypersparc_flush_tlb_all:
 266        mov     0x400, %g1
 267        sta     %g0, [%g1] ASI_M_FLUSH_PROBE
 268        retl
 269         nop
 270
 271hypersparc_flush_tlb_mm:
 272        mov     SRMMU_CTX_REG, %g1
 273        ld      [%o0 + AOFF_mm_context], %o1
 274        lda     [%g1] ASI_M_MMUREGS, %g5
 275#ifndef CONFIG_SMP
 276        cmp     %o1, -1
 277        be      hypersparc_flush_tlb_mm_out
 278#endif
 279         mov    0x300, %g2
 280        sta     %o1, [%g1] ASI_M_MMUREGS
 281        sta     %g0, [%g2] ASI_M_FLUSH_PROBE
 282hypersparc_flush_tlb_mm_out:
 283        retl
 284         sta    %g5, [%g1] ASI_M_MMUREGS
 285
 286hypersparc_flush_tlb_range:
 287        ld      [%o0 + 0x00], %o0       /* XXX vma->vm_mm GROSS XXX */
 288        mov     SRMMU_CTX_REG, %g1
 289        ld      [%o0 + AOFF_mm_context], %o3
 290        lda     [%g1] ASI_M_MMUREGS, %g5
 291#ifndef CONFIG_SMP
 292        cmp     %o3, -1
 293        be      hypersparc_flush_tlb_range_out
 294#endif
 295         sethi  %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4
 296        sta     %o3, [%g1] ASI_M_MMUREGS
 297        and     %o1, %o4, %o1
 298        add     %o1, 0x200, %o1
 299        sta     %g0, [%o1] ASI_M_FLUSH_PROBE
 3001:
 301        sub     %o1, %o4, %o1
 302        cmp     %o1, %o2
 303        blu,a   1b
 304         sta    %g0, [%o1] ASI_M_FLUSH_PROBE
 305hypersparc_flush_tlb_range_out:
 306        retl
 307         sta    %g5, [%g1] ASI_M_MMUREGS
 308
 309hypersparc_flush_tlb_page:
 310        ld      [%o0 + 0x00], %o0       /* XXX vma->vm_mm GROSS XXX */
 311        mov     SRMMU_CTX_REG, %g1
 312        ld      [%o0 + AOFF_mm_context], %o3
 313        andn    %o1, (PAGE_SIZE - 1), %o1
 314#ifndef CONFIG_SMP
 315        cmp     %o3, -1
 316        be      hypersparc_flush_tlb_page_out
 317#endif
 318         lda    [%g1] ASI_M_MMUREGS, %g5
 319        sta     %o3, [%g1] ASI_M_MMUREGS
 320        sta     %g0, [%o1] ASI_M_FLUSH_PROBE
 321hypersparc_flush_tlb_page_out:
 322        retl
 323         sta    %g5, [%g1] ASI_M_MMUREGS
 324
 325        __INIT
 326        
 327        /* High speed page clear/copy. */
 328hypersparc_bzero_1page:
 329/* NOTE: This routine has to be shorter than 40insns --jj */
 330        clr     %g1
 331        mov     32, %g2
 332        mov     64, %g3
 333        mov     96, %g4
 334        mov     128, %g5
 335        mov     160, %g7
 336        mov     192, %o2
 337        mov     224, %o3
 338        mov     16, %o1
 3391:
 340        stda    %g0, [%o0 + %g0] ASI_M_BFILL
 341        stda    %g0, [%o0 + %g2] ASI_M_BFILL
 342        stda    %g0, [%o0 + %g3] ASI_M_BFILL
 343        stda    %g0, [%o0 + %g4] ASI_M_BFILL
 344        stda    %g0, [%o0 + %g5] ASI_M_BFILL
 345        stda    %g0, [%o0 + %g7] ASI_M_BFILL
 346        stda    %g0, [%o0 + %o2] ASI_M_BFILL
 347        stda    %g0, [%o0 + %o3] ASI_M_BFILL
 348        subcc   %o1, 1, %o1
 349        bne     1b
 350         add    %o0, 256, %o0
 351
 352        retl
 353         nop
 354
 355hypersparc_copy_1page:
 356/* NOTE: This routine has to be shorter than 70insns --jj */
 357        sub     %o1, %o0, %o2           ! difference
 358        mov     16, %g1
 3591:
 360        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 361        add     %o0, 32, %o0
 362        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 363        add     %o0, 32, %o0
 364        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 365        add     %o0, 32, %o0
 366        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 367        add     %o0, 32, %o0
 368        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 369        add     %o0, 32, %o0
 370        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 371        add     %o0, 32, %o0
 372        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 373        add     %o0, 32, %o0
 374        sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 375        subcc   %g1, 1, %g1
 376        bne     1b
 377         add    %o0, 32, %o0
 378
 379        retl
 380         nop
 381
 382        .globl  hypersparc_setup_blockops
 383hypersparc_setup_blockops:
 384        sethi   %hi(bzero_1page), %o0
 385        or      %o0, %lo(bzero_1page), %o0
 386        sethi   %hi(hypersparc_bzero_1page), %o1
 387        or      %o1, %lo(hypersparc_bzero_1page), %o1
 388        sethi   %hi(hypersparc_copy_1page), %o2
 389        or      %o2, %lo(hypersparc_copy_1page), %o2
 390        ld      [%o1], %o4
 3911:
 392        add     %o1, 4, %o1
 393        st      %o4, [%o0]
 394        add     %o0, 4, %o0
 395        cmp     %o1, %o2
 396        bne     1b
 397         ld     [%o1], %o4
 398        sethi   %hi(__copy_1page), %o0
 399        or      %o0, %lo(__copy_1page), %o0
 400        sethi   %hi(hypersparc_setup_blockops), %o2
 401        or      %o2, %lo(hypersparc_setup_blockops), %o2
 402        ld      [%o1], %o4
 4031:
 404        add     %o1, 4, %o1
 405        st      %o4, [%o0]
 406        add     %o0, 4, %o0
 407        cmp     %o1, %o2
 408        bne     1b
 409         ld     [%o1], %o4
 410        sta     %g0, [%g0] ASI_M_FLUSH_IWHOLE
 411        retl
 412         nop
 413