LXR linux/arch/sparc/mm/ultra.S

   1/*
   2 * ultra.S: Don't expand these all over the place...
   3 *
   4 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
   5 */
   6
   7#include <asm/asi.h>
   8#include <asm/pgtable.h>
   9#include <asm/page.h>
  10#include <asm/spitfire.h>
  11#include <asm/mmu_context.h>
  12#include <asm/mmu.h>
  13#include <asm/pil.h>
  14#include <asm/head.h>
  15#include <asm/thread_info.h>
  16#include <asm/cacheflush.h>
  17#include <asm/hypervisor.h>
  18#include <asm/cpudata.h>
  19
  20        /* Basically, most of the Spitfire vs. Cheetah madness
  21         * has to do with the fact that Cheetah does not support
  22         * IMMU flushes out of the secondary context.  Someone needs
  23         * to throw a south lake birthday party for the folks
  24         * in Microelectronics who refused to fix this shit.
  25         */
  26
  27        /* This file is meant to be read efficiently by the CPU, not humans.
  28         * Staraj sie tego nikomu nie pierdolnac...
  29         */
  30        .text
  31        .align          32
  32        .globl          __flush_tlb_mm
  33__flush_tlb_mm:         /* 18 insns */
  34        /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
  35        ldxa            [%o1] ASI_DMMU, %g2
  36        cmp             %g2, %o0
  37        bne,pn          %icc, __spitfire_flush_tlb_mm_slow
  38         mov            0x50, %g3
  39        stxa            %g0, [%g3] ASI_DMMU_DEMAP
  40        stxa            %g0, [%g3] ASI_IMMU_DEMAP
  41        sethi           %hi(KERNBASE), %g3
  42        flush           %g3
  43        retl
  44         nop
  45        nop
  46        nop
  47        nop
  48        nop
  49        nop
  50        nop
  51        nop
  52        nop
  53        nop
  54
  55        .align          32
  56        .globl          __flush_tlb_pending
  57__flush_tlb_pending:    /* 26 insns */
  58        /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
  59        rdpr            %pstate, %g7
  60        sllx            %o1, 3, %o1
  61        andn            %g7, PSTATE_IE, %g2
  62        wrpr            %g2, %pstate
  63        mov             SECONDARY_CONTEXT, %o4
  64        ldxa            [%o4] ASI_DMMU, %g2
  65        stxa            %o0, [%o4] ASI_DMMU
  661:      sub             %o1, (1 << 3), %o1
  67        ldx             [%o2 + %o1], %o3
  68        andcc           %o3, 1, %g0
  69        andn            %o3, 1, %o3
  70        be,pn           %icc, 2f
  71         or             %o3, 0x10, %o3
  72        stxa            %g0, [%o3] ASI_IMMU_DEMAP
  732:      stxa            %g0, [%o3] ASI_DMMU_DEMAP
  74        membar          #Sync
  75        brnz,pt         %o1, 1b
  76         nop
  77        stxa            %g2, [%o4] ASI_DMMU
  78        sethi           %hi(KERNBASE), %o4
  79        flush           %o4
  80        retl
  81         wrpr           %g7, 0x0, %pstate
  82        nop
  83        nop
  84        nop
  85        nop
  86
  87        .align          32
  88        .globl          __flush_tlb_kernel_range
  89__flush_tlb_kernel_range:       /* 16 insns */
  90        /* %o0=start, %o1=end */
  91        cmp             %o0, %o1
  92        be,pn           %xcc, 2f
  93         sethi          %hi(PAGE_SIZE), %o4
  94        sub             %o1, %o0, %o3
  95        sub             %o3, %o4, %o3
  96        or              %o0, 0x20, %o0          ! Nucleus
  971:      stxa            %g0, [%o0 + %o3] ASI_DMMU_DEMAP
  98        stxa            %g0, [%o0 + %o3] ASI_IMMU_DEMAP
  99        membar          #Sync
 100        brnz,pt         %o3, 1b
 101         sub            %o3, %o4, %o3
 1022:      sethi           %hi(KERNBASE), %o3
 103        flush           %o3
 104        retl
 105         nop
 106        nop
 107
 108__spitfire_flush_tlb_mm_slow:
 109        rdpr            %pstate, %g1
 110        wrpr            %g1, PSTATE_IE, %pstate
 111        stxa            %o0, [%o1] ASI_DMMU
 112        stxa            %g0, [%g3] ASI_DMMU_DEMAP
 113        stxa            %g0, [%g3] ASI_IMMU_DEMAP
 114        flush           %g6
 115        stxa            %g2, [%o1] ASI_DMMU
 116        sethi           %hi(KERNBASE), %o1
 117        flush           %o1
 118        retl
 119         wrpr           %g1, 0, %pstate
 120
 121/*
 122 * The following code flushes one page_size worth.
 123 */
 124        .section .kprobes.text, "ax"
 125        .align          32
 126        .globl          __flush_icache_page
 127__flush_icache_page:    /* %o0 = phys_page */
 128        srlx            %o0, PAGE_SHIFT, %o0
 129        sethi           %uhi(PAGE_OFFSET), %g1
 130        sllx            %o0, PAGE_SHIFT, %o0
 131        sethi           %hi(PAGE_SIZE), %g2
 132        sllx            %g1, 32, %g1
 133        add             %o0, %g1, %o0
 1341:      subcc           %g2, 32, %g2
 135        bne,pt          %icc, 1b
 136         flush          %o0 + %g2
 137        retl
 138         nop
 139
 140#ifdef DCACHE_ALIASING_POSSIBLE
 141
 142#if (PAGE_SHIFT != 13)
 143#error only page shift of 13 is supported by dcache flush
 144#endif
 145
 146#define DTAG_MASK 0x3
 147
 148        /* This routine is Spitfire specific so the hardcoded
 149         * D-cache size and line-size are OK.
 150         */
 151        .align          64
 152        .globl          __flush_dcache_page
 153__flush_dcache_page:    /* %o0=kaddr, %o1=flush_icache */
 154        sethi           %uhi(PAGE_OFFSET), %g1
 155        sllx            %g1, 32, %g1
 156        sub             %o0, %g1, %o0                   ! physical address
 157        srlx            %o0, 11, %o0                    ! make D-cache TAG
 158        sethi           %hi(1 << 14), %o2               ! D-cache size
 159        sub             %o2, (1 << 5), %o2              ! D-cache line size
 1601:      ldxa            [%o2] ASI_DCACHE_TAG, %o3       ! load D-cache TAG
 161        andcc           %o3, DTAG_MASK, %g0             ! Valid?
 162        be,pn           %xcc, 2f                        ! Nope, branch
 163         andn           %o3, DTAG_MASK, %o3             ! Clear valid bits
 164        cmp             %o3, %o0                        ! TAG match?
 165        bne,pt          %xcc, 2f                        ! Nope, branch
 166         nop
 167        stxa            %g0, [%o2] ASI_DCACHE_TAG       ! Invalidate TAG
 168        membar          #Sync
 1692:      brnz,pt         %o2, 1b
 170         sub            %o2, (1 << 5), %o2              ! D-cache line size
 171
 172        /* The I-cache does not snoop local stores so we
 173         * better flush that too when necessary.
 174         */
 175        brnz,pt         %o1, __flush_icache_page
 176         sllx           %o0, 11, %o0
 177        retl
 178         nop
 179
 180#endif /* DCACHE_ALIASING_POSSIBLE */
 181
 182        .previous
 183
 184        /* Cheetah specific versions, patched at boot time. */
 185__cheetah_flush_tlb_mm: /* 19 insns */
 186        rdpr            %pstate, %g7
 187        andn            %g7, PSTATE_IE, %g2
 188        wrpr            %g2, 0x0, %pstate
 189        wrpr            %g0, 1, %tl
 190        mov             PRIMARY_CONTEXT, %o2
 191        mov             0x40, %g3
 192        ldxa            [%o2] ASI_DMMU, %g2
 193        srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o1
 194        sllx            %o1, CTX_PGSZ1_NUC_SHIFT, %o1
 195        or              %o0, %o1, %o0   /* Preserve nucleus page size fields */
 196        stxa            %o0, [%o2] ASI_DMMU
 197        stxa            %g0, [%g3] ASI_DMMU_DEMAP
 198        stxa            %g0, [%g3] ASI_IMMU_DEMAP
 199        stxa            %g2, [%o2] ASI_DMMU
 200        sethi           %hi(KERNBASE), %o2
 201        flush           %o2
 202        wrpr            %g0, 0, %tl
 203        retl
 204         wrpr           %g7, 0x0, %pstate
 205
 206__cheetah_flush_tlb_pending:    /* 27 insns */
 207        /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 208        rdpr            %pstate, %g7
 209        sllx            %o1, 3, %o1
 210        andn            %g7, PSTATE_IE, %g2
 211        wrpr            %g2, 0x0, %pstate
 212        wrpr            %g0, 1, %tl
 213        mov             PRIMARY_CONTEXT, %o4
 214        ldxa            [%o4] ASI_DMMU, %g2
 215        srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %o3
 216        sllx            %o3, CTX_PGSZ1_NUC_SHIFT, %o3
 217        or              %o0, %o3, %o0   /* Preserve nucleus page size fields */
 218        stxa            %o0, [%o4] ASI_DMMU
 2191:      sub             %o1, (1 << 3), %o1
 220        ldx             [%o2 + %o1], %o3
 221        andcc           %o3, 1, %g0
 222        be,pn           %icc, 2f
 223         andn           %o3, 1, %o3
 224        stxa            %g0, [%o3] ASI_IMMU_DEMAP
 2252:      stxa            %g0, [%o3] ASI_DMMU_DEMAP       
 226        membar          #Sync
 227        brnz,pt         %o1, 1b
 228         nop
 229        stxa            %g2, [%o4] ASI_DMMU
 230        sethi           %hi(KERNBASE), %o4
 231        flush           %o4
 232        wrpr            %g0, 0, %tl
 233        retl
 234         wrpr           %g7, 0x0, %pstate
 235
 236#ifdef DCACHE_ALIASING_POSSIBLE
 237__cheetah_flush_dcache_page: /* 11 insns */
 238        sethi           %uhi(PAGE_OFFSET), %g1
 239        sllx            %g1, 32, %g1
 240        sub             %o0, %g1, %o0
 241        sethi           %hi(PAGE_SIZE), %o4
 2421:      subcc           %o4, (1 << 5), %o4
 243        stxa            %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
 244        membar          #Sync
 245        bne,pt          %icc, 1b
 246         nop
 247        retl            /* I-cache flush never needed on Cheetah, see callers. */
 248         nop
 249#endif /* DCACHE_ALIASING_POSSIBLE */
 250
 251        /* Hypervisor specific versions, patched at boot time.  */
 252__hypervisor_tlb_tl0_error:
 253        save            %sp, -192, %sp
 254        mov             %i0, %o0
 255        call            hypervisor_tlbop_error
 256         mov            %i1, %o1
 257        ret
 258         restore
 259
 260__hypervisor_flush_tlb_mm: /* 10 insns */
 261        mov             %o0, %o2        /* ARG2: mmu context */
 262        mov             0, %o0          /* ARG0: CPU lists unimplemented */
 263        mov             0, %o1          /* ARG1: CPU lists unimplemented */
 264        mov             HV_MMU_ALL, %o3 /* ARG3: flags */
 265        mov             HV_FAST_MMU_DEMAP_CTX, %o5
 266        ta              HV_FAST_TRAP
 267        brnz,pn         %o0, __hypervisor_tlb_tl0_error
 268         mov            HV_FAST_MMU_DEMAP_CTX, %o1
 269        retl
 270         nop
 271
 272__hypervisor_flush_tlb_pending: /* 16 insns */
 273        /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 274        sllx            %o1, 3, %g1
 275        mov             %o2, %g2
 276        mov             %o0, %g3
 2771:      sub             %g1, (1 << 3), %g1
 278        ldx             [%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
 279        mov             %g3, %o1              /* ARG1: mmu context */
 280        mov             HV_MMU_ALL, %o2       /* ARG2: flags */
 281        srlx            %o0, PAGE_SHIFT, %o0
 282        sllx            %o0, PAGE_SHIFT, %o0
 283        ta              HV_MMU_UNMAP_ADDR_TRAP
 284        brnz,pn         %o0, __hypervisor_tlb_tl0_error
 285         mov            HV_MMU_UNMAP_ADDR_TRAP, %o1
 286        brnz,pt         %g1, 1b
 287         nop
 288        retl
 289         nop
 290
 291__hypervisor_flush_tlb_kernel_range: /* 16 insns */
 292        /* %o0=start, %o1=end */
 293        cmp             %o0, %o1
 294        be,pn           %xcc, 2f
 295         sethi          %hi(PAGE_SIZE), %g3
 296        mov             %o0, %g1
 297        sub             %o1, %g1, %g2
 298        sub             %g2, %g3, %g2
 2991:      add             %g1, %g2, %o0   /* ARG0: virtual address */
 300        mov             0, %o1          /* ARG1: mmu context */
 301        mov             HV_MMU_ALL, %o2 /* ARG2: flags */
 302        ta              HV_MMU_UNMAP_ADDR_TRAP
 303        brnz,pn         %o0, __hypervisor_tlb_tl0_error
 304         mov            HV_MMU_UNMAP_ADDR_TRAP, %o1
 305        brnz,pt         %g2, 1b
 306         sub            %g2, %g3, %g2
 3072:      retl
 308         nop
 309
 310#ifdef DCACHE_ALIASING_POSSIBLE
 311        /* XXX Niagara and friends have an 8K cache, so no aliasing is
 312         * XXX possible, but nothing explicit in the Hypervisor API
 313         * XXX guarantees this.
 314         */
 315__hypervisor_flush_dcache_page: /* 2 insns */
 316        retl
 317         nop
 318#endif
 319
 320tlb_patch_one:
 3211:      lduw            [%o1], %g1
 322        stw             %g1, [%o0]
 323        flush           %o0
 324        subcc           %o2, 1, %o2
 325        add             %o1, 4, %o1
 326        bne,pt          %icc, 1b
 327         add            %o0, 4, %o0
 328        retl
 329         nop
 330
 331        .globl          cheetah_patch_cachetlbops
 332cheetah_patch_cachetlbops:
 333        save            %sp, -128, %sp
 334
 335        sethi           %hi(__flush_tlb_mm), %o0
 336        or              %o0, %lo(__flush_tlb_mm), %o0
 337        sethi           %hi(__cheetah_flush_tlb_mm), %o1
 338        or              %o1, %lo(__cheetah_flush_tlb_mm), %o1
 339        call            tlb_patch_one
 340         mov            19, %o2
 341
 342        sethi           %hi(__flush_tlb_pending), %o0
 343        or              %o0, %lo(__flush_tlb_pending), %o0
 344        sethi           %hi(__cheetah_flush_tlb_pending), %o1
 345        or              %o1, %lo(__cheetah_flush_tlb_pending), %o1
 346        call            tlb_patch_one
 347         mov            27, %o2
 348
 349#ifdef DCACHE_ALIASING_POSSIBLE
 350        sethi           %hi(__flush_dcache_page), %o0
 351        or              %o0, %lo(__flush_dcache_page), %o0
 352        sethi           %hi(__cheetah_flush_dcache_page), %o1
 353        or              %o1, %lo(__cheetah_flush_dcache_page), %o1
 354        call            tlb_patch_one
 355         mov            11, %o2
 356#endif /* DCACHE_ALIASING_POSSIBLE */
 357
 358        ret
 359         restore
 360
 361#ifdef CONFIG_SMP
 362        /* These are all called by the slaves of a cross call, at
 363         * trap level 1, with interrupts fully disabled.
 364         *
 365         * Register usage:
 366         *   %g5        mm->context     (all tlb flushes)
 367         *   %g1        address arg 1   (tlb page and range flushes)
 368         *   %g7        address arg 2   (tlb range flush only)
 369         *
 370         *   %g6        scratch 1
 371         *   %g2        scratch 2
 372         *   %g3        scratch 3
 373         *   %g4        scratch 4
 374         */
 375        .align          32
 376        .globl          xcall_flush_tlb_mm
 377xcall_flush_tlb_mm:     /* 21 insns */
 378        mov             PRIMARY_CONTEXT, %g2
 379        ldxa            [%g2] ASI_DMMU, %g3
 380        srlx            %g3, CTX_PGSZ1_NUC_SHIFT, %g4
 381        sllx            %g4, CTX_PGSZ1_NUC_SHIFT, %g4
 382        or              %g5, %g4, %g5   /* Preserve nucleus page size fields */
 383        stxa            %g5, [%g2] ASI_DMMU
 384        mov             0x40, %g4
 385        stxa            %g0, [%g4] ASI_DMMU_DEMAP
 386        stxa            %g0, [%g4] ASI_IMMU_DEMAP
 387        stxa            %g3, [%g2] ASI_DMMU
 388        retry
 389        nop
 390        nop
 391        nop
 392        nop
 393        nop
 394        nop
 395        nop
 396        nop
 397        nop
 398        nop
 399
 400        .globl          xcall_flush_tlb_pending
 401xcall_flush_tlb_pending:        /* 21 insns */
 402        /* %g5=context, %g1=nr, %g7=vaddrs[] */
 403        sllx            %g1, 3, %g1
 404        mov             PRIMARY_CONTEXT, %g4
 405        ldxa            [%g4] ASI_DMMU, %g2
 406        srlx            %g2, CTX_PGSZ1_NUC_SHIFT, %g4
 407        sllx            %g4, CTX_PGSZ1_NUC_SHIFT, %g4
 408        or              %g5, %g4, %g5
 409        mov             PRIMARY_CONTEXT, %g4
 410        stxa            %g5, [%g4] ASI_DMMU
 4111:      sub             %g1, (1 << 3), %g1
 412        ldx             [%g7 + %g1], %g5
 413        andcc           %g5, 0x1, %g0
 414        be,pn           %icc, 2f
 415
 416         andn           %g5, 0x1, %g5
 417        stxa            %g0, [%g5] ASI_IMMU_DEMAP
 4182:      stxa            %g0, [%g5] ASI_DMMU_DEMAP
 419        membar          #Sync
 420        brnz,pt         %g1, 1b
 421         nop
 422        stxa            %g2, [%g4] ASI_DMMU
 423        retry
 424        nop
 425
 426        .globl          xcall_flush_tlb_kernel_range
 427xcall_flush_tlb_kernel_range:   /* 25 insns */
 428        sethi           %hi(PAGE_SIZE - 1), %g2
 429        or              %g2, %lo(PAGE_SIZE - 1), %g2
 430        andn            %g1, %g2, %g1
 431        andn            %g7, %g2, %g7
 432        sub             %g7, %g1, %g3
 433        add             %g2, 1, %g2
 434        sub             %g3, %g2, %g3
 435        or              %g1, 0x20, %g1          ! Nucleus
 4361:      stxa            %g0, [%g1 + %g3] ASI_DMMU_DEMAP
 437        stxa            %g0, [%g1 + %g3] ASI_IMMU_DEMAP
 438        membar          #Sync
 439        brnz,pt         %g3, 1b
 440         sub            %g3, %g2, %g3
 441        retry
 442        nop
 443        nop
 444        nop
 445        nop
 446        nop
 447        nop
 448        nop
 449        nop
 450        nop
 451        nop
 452        nop
 453
 454        /* This runs in a very controlled environment, so we do
 455         * not need to worry about BH races etc.
 456         */
 457        .globl          xcall_sync_tick
 458xcall_sync_tick:
 459
 460661:    rdpr            %pstate, %g2
 461        wrpr            %g2, PSTATE_IG | PSTATE_AG, %pstate
 462        .section        .sun4v_2insn_patch, "ax"
 463        .word           661b
 464        nop
 465        nop
 466        .previous
 467
 468        rdpr            %pil, %g2
 469        wrpr            %g0, PIL_NORMAL_MAX, %pil
 470        sethi           %hi(109f), %g7
 471        b,pt            %xcc, etrap_irq
 472109:     or             %g7, %lo(109b), %g7
 473#ifdef CONFIG_TRACE_IRQFLAGS
 474        call            trace_hardirqs_off
 475         nop
 476#endif
 477        call            smp_synchronize_tick_client
 478         nop
 479        b               rtrap_xcall
 480         ldx            [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 481
 482        .globl          xcall_fetch_glob_regs
 483xcall_fetch_glob_regs:
 484        sethi           %hi(global_reg_snapshot), %g1
 485        or              %g1, %lo(global_reg_snapshot), %g1
 486        __GET_CPUID(%g2)
 487        sllx            %g2, 6, %g3
 488        add             %g1, %g3, %g1
 489        rdpr            %tstate, %g7
 490        stx             %g7, [%g1 + GR_SNAP_TSTATE]
 491        rdpr            %tpc, %g7
 492        stx             %g7, [%g1 + GR_SNAP_TPC]
 493        rdpr            %tnpc, %g7
 494        stx             %g7, [%g1 + GR_SNAP_TNPC]
 495        stx             %o7, [%g1 + GR_SNAP_O7]
 496        stx             %i7, [%g1 + GR_SNAP_I7]
 497        /* Don't try this at home kids... */
 498        rdpr            %cwp, %g2
 499        sub             %g2, 1, %g7
 500        wrpr            %g7, %cwp
 501        mov             %i7, %g7
 502        wrpr            %g2, %cwp
 503        stx             %g7, [%g1 + GR_SNAP_RPC]
 504        sethi           %hi(trap_block), %g7
 505        or              %g7, %lo(trap_block), %g7
 506        sllx            %g2, TRAP_BLOCK_SZ_SHIFT, %g2
 507        add             %g7, %g2, %g7
 508        ldx             [%g7 + TRAP_PER_CPU_THREAD], %g3
 509        stx             %g3, [%g1 + GR_SNAP_THREAD]
 510        retry
 511
 512#ifdef DCACHE_ALIASING_POSSIBLE
 513        .align          32
 514        .globl          xcall_flush_dcache_page_cheetah
 515xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
 516        sethi           %hi(PAGE_SIZE), %g3
 5171:      subcc           %g3, (1 << 5), %g3
 518        stxa            %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
 519        membar          #Sync
 520        bne,pt          %icc, 1b
 521         nop
 522        retry
 523        nop
 524#endif /* DCACHE_ALIASING_POSSIBLE */
 525
 526        .globl          xcall_flush_dcache_page_spitfire
 527xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
 528                                     %g7 == kernel page virtual address
 529                                     %g5 == (page->mapping != NULL)  */
 530#ifdef DCACHE_ALIASING_POSSIBLE
 531        srlx            %g1, (13 - 2), %g1      ! Form tag comparitor
 532        sethi           %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
 533        sub             %g3, (1 << 5), %g3      ! D$ linesize == 32
 5341:      ldxa            [%g3] ASI_DCACHE_TAG, %g2
 535        andcc           %g2, 0x3, %g0
 536        be,pn           %xcc, 2f
 537         andn           %g2, 0x3, %g2
 538        cmp             %g2, %g1
 539
 540        bne,pt          %xcc, 2f
 541         nop
 542        stxa            %g0, [%g3] ASI_DCACHE_TAG
 543        membar          #Sync
 5442:      cmp             %g3, 0
 545        bne,pt          %xcc, 1b
 546         sub            %g3, (1 << 5), %g3
 547
 548        brz,pn          %g5, 2f
 549#endif /* DCACHE_ALIASING_POSSIBLE */
 550         sethi          %hi(PAGE_SIZE), %g3
 551
 5521:      flush           %g7
 553        subcc           %g3, (1 << 5), %g3
 554        bne,pt          %icc, 1b
 555         add            %g7, (1 << 5), %g7
 556
 5572:      retry
 558        nop
 559        nop
 560
 561        /* %g5: error
 562         * %g6: tlb op
 563         */
 564__hypervisor_tlb_xcall_error:
 565        mov     %g5, %g4
 566        mov     %g6, %g5
 567        ba,pt   %xcc, etrap
 568         rd     %pc, %g7
 569        mov     %l4, %o0
 570        call    hypervisor_tlbop_error_xcall
 571         mov    %l5, %o1
 572        ba,a,pt %xcc, rtrap
 573
 574        .globl          __hypervisor_xcall_flush_tlb_mm
 575__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
 576        /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
 577        mov             %o0, %g2
 578        mov             %o1, %g3
 579        mov             %o2, %g4
 580        mov             %o3, %g1
 581        mov             %o5, %g7
 582        clr             %o0             /* ARG0: CPU lists unimplemented */
 583        clr             %o1             /* ARG1: CPU lists unimplemented */
 584        mov             %g5, %o2        /* ARG2: mmu context */
 585        mov             HV_MMU_ALL, %o3 /* ARG3: flags */
 586        mov             HV_FAST_MMU_DEMAP_CTX, %o5
 587        ta              HV_FAST_TRAP
 588        mov             HV_FAST_MMU_DEMAP_CTX, %g6
 589        brnz,pn         %o0, __hypervisor_tlb_xcall_error
 590         mov            %o0, %g5
 591        mov             %g2, %o0
 592        mov             %g3, %o1
 593        mov             %g4, %o2
 594        mov             %g1, %o3
 595        mov             %g7, %o5
 596        membar          #Sync
 597        retry
 598
 599        .globl          __hypervisor_xcall_flush_tlb_pending
 600__hypervisor_xcall_flush_tlb_pending: /* 21 insns */
 601        /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */
 602        sllx            %g1, 3, %g1
 603        mov             %o0, %g2
 604        mov             %o1, %g3
 605        mov             %o2, %g4
 6061:      sub             %g1, (1 << 3), %g1
 607        ldx             [%g7 + %g1], %o0        /* ARG0: virtual address */
 608        mov             %g5, %o1                /* ARG1: mmu context */
 609        mov             HV_MMU_ALL, %o2         /* ARG2: flags */
 610        srlx            %o0, PAGE_SHIFT, %o0
 611        sllx            %o0, PAGE_SHIFT, %o0
 612        ta              HV_MMU_UNMAP_ADDR_TRAP
 613        mov             HV_MMU_UNMAP_ADDR_TRAP, %g6
 614        brnz,a,pn       %o0, __hypervisor_tlb_xcall_error
 615         mov            %o0, %g5
 616        brnz,pt         %g1, 1b
 617         nop
 618        mov             %g2, %o0
 619        mov             %g3, %o1
 620        mov             %g4, %o2
 621        membar          #Sync
 622        retry
 623
 624        .globl          __hypervisor_xcall_flush_tlb_kernel_range
 625__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
 626        /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
 627        sethi           %hi(PAGE_SIZE - 1), %g2
 628        or              %g2, %lo(PAGE_SIZE - 1), %g2
 629        andn            %g1, %g2, %g1
 630        andn            %g7, %g2, %g7
 631        sub             %g7, %g1, %g3
 632        add             %g2, 1, %g2
 633        sub             %g3, %g2, %g3
 634        mov             %o0, %g2
 635        mov             %o1, %g4
 636        mov             %o2, %g7
 6371:      add             %g1, %g3, %o0   /* ARG0: virtual address */
 638        mov             0, %o1          /* ARG1: mmu context */
 639        mov             HV_MMU_ALL, %o2 /* ARG2: flags */
 640        ta              HV_MMU_UNMAP_ADDR_TRAP
 641        mov             HV_MMU_UNMAP_ADDR_TRAP, %g6
 642        brnz,pn         %o0, __hypervisor_tlb_xcall_error
 643         mov            %o0, %g5
 644        sethi           %hi(PAGE_SIZE), %o2
 645        brnz,pt         %g3, 1b
 646         sub            %g3, %o2, %g3
 647        mov             %g2, %o0
 648        mov             %g4, %o1
 649        mov             %g7, %o2
 650        membar          #Sync
 651        retry
 652
 653        /* These just get rescheduled to PIL vectors. */
 654        .globl          xcall_call_function
 655xcall_call_function:
 656        wr              %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
 657        retry
 658
 659        .globl          xcall_call_function_single
 660xcall_call_function_single:
 661        wr              %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
 662        retry
 663
 664        .globl          xcall_receive_signal
 665xcall_receive_signal:
 666        wr              %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
 667        retry
 668
 669        .globl          xcall_capture
 670xcall_capture:
 671        wr              %g0, (1 << PIL_SMP_CAPTURE), %set_softint
 672        retry
 673
 674        .globl          xcall_new_mmu_context_version
 675xcall_new_mmu_context_version:
 676        wr              %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint
 677        retry
 678
 679#ifdef CONFIG_KGDB
 680        .globl          xcall_kgdb_capture
 681xcall_kgdb_capture:
 682        wr              %g0, (1 << PIL_KGDB_CAPTURE), %set_softint
 683        retry
 684#endif
 685
 686#endif /* CONFIG_SMP */
 687
 688
 689        .globl          hypervisor_patch_cachetlbops
 690hypervisor_patch_cachetlbops:
 691        save            %sp, -128, %sp
 692
 693        sethi           %hi(__flush_tlb_mm), %o0
 694        or              %o0, %lo(__flush_tlb_mm), %o0
 695        sethi           %hi(__hypervisor_flush_tlb_mm), %o1
 696        or              %o1, %lo(__hypervisor_flush_tlb_mm), %o1
 697        call            tlb_patch_one
 698         mov            10, %o2
 699
 700        sethi           %hi(__flush_tlb_pending), %o0
 701        or              %o0, %lo(__flush_tlb_pending), %o0
 702        sethi           %hi(__hypervisor_flush_tlb_pending), %o1
 703        or              %o1, %lo(__hypervisor_flush_tlb_pending), %o1
 704        call            tlb_patch_one
 705         mov            16, %o2
 706
 707        sethi           %hi(__flush_tlb_kernel_range), %o0
 708        or              %o0, %lo(__flush_tlb_kernel_range), %o0
 709        sethi           %hi(__hypervisor_flush_tlb_kernel_range), %o1
 710        or              %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
 711        call            tlb_patch_one
 712         mov            16, %o2
 713
 714#ifdef DCACHE_ALIASING_POSSIBLE
 715        sethi           %hi(__flush_dcache_page), %o0
 716        or              %o0, %lo(__flush_dcache_page), %o0
 717        sethi           %hi(__hypervisor_flush_dcache_page), %o1
 718        or              %o1, %lo(__hypervisor_flush_dcache_page), %o1
 719        call            tlb_patch_one
 720         mov            2, %o2
 721#endif /* DCACHE_ALIASING_POSSIBLE */
 722
 723#ifdef CONFIG_SMP
 724        sethi           %hi(xcall_flush_tlb_mm), %o0
 725        or              %o0, %lo(xcall_flush_tlb_mm), %o0
 726        sethi           %hi(__hypervisor_xcall_flush_tlb_mm), %o1
 727        or              %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
 728        call            tlb_patch_one
 729         mov            21, %o2
 730
 731        sethi           %hi(xcall_flush_tlb_pending), %o0
 732        or              %o0, %lo(xcall_flush_tlb_pending), %o0
 733        sethi           %hi(__hypervisor_xcall_flush_tlb_pending), %o1
 734        or              %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1
 735        call            tlb_patch_one
 736         mov            21, %o2
 737
 738        sethi           %hi(xcall_flush_tlb_kernel_range), %o0
 739        or              %o0, %lo(xcall_flush_tlb_kernel_range), %o0
 740        sethi           %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
 741        or              %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
 742        call            tlb_patch_one
 743         mov            25, %o2
 744#endif /* CONFIG_SMP */
 745
 746        ret
 747         restore
 748