LXR linux/arch/powerpc/mm/tlb_nohash

   1/*
   2 * This file contains low-level functions for performing various
   3 * types of TLB invalidations on various processors with no hash
   4 * table.
   5 *
   6 * This file implements the following functions for all no-hash
   7 * processors. Some aren't implemented for some variants. Some
   8 * are inline in tlbflush.h
   9 *
  10 *      - tlbil_va
  11 *      - tlbil_pid
  12 *      - tlbil_all
  13 *      - tlbivax_bcast
  14 *
  15 * Code mostly moved over from misc_32.S
  16 *
  17 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  18 *
  19 * Partially rewritten by Cort Dougan (cort@cs.nmt.edu)
  20 * Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt.
  21 *
  22 * This program is free software; you can redistribute it and/or
  23 * modify it under the terms of the GNU General Public License
  24 * as published by the Free Software Foundation; either version
  25 * 2 of the License, or (at your option) any later version.
  26 *
  27 */
  28
  29#include <asm/reg.h>
  30#include <asm/page.h>
  31#include <asm/cputable.h>
  32#include <asm/mmu.h>
  33#include <asm/ppc_asm.h>
  34#include <asm/asm-offsets.h>
  35#include <asm/processor.h>
  36#include <asm/bug.h>
  37
  38#if defined(CONFIG_40x)
  39
  40/*
  41 * 40x implementation needs only tlbil_va
  42 */
  43_GLOBAL(__tlbil_va)
  44        /* We run the search with interrupts disabled because we have to change
  45         * the PID and I don't want to preempt when that happens.
  46         */
  47        mfmsr   r5
  48        mfspr   r6,SPRN_PID
  49        wrteei  0
  50        mtspr   SPRN_PID,r4
  51        tlbsx.  r3, 0, r3
  52        mtspr   SPRN_PID,r6
  53        wrtee   r5
  54        bne     1f
  55        sync
  56        /* There are only 64 TLB entries, so r3 < 64, which means bit 25 is
  57         * clear. Since 25 is the V bit in the TLB_TAG, loading this value
  58         * will invalidate the TLB entry. */
  59        tlbwe   r3, r3, TLB_TAG
  60        isync
  611:      blr
  62
  63#elif defined(CONFIG_8xx)
  64
  65/*
  66 * Nothing to do for 8xx, everything is inline
  67 */
  68
  69#elif defined(CONFIG_44x) /* Includes 47x */
  70
  71/*
  72 * 440 implementation uses tlbsx/we for tlbil_va and a full sweep
  73 * of the TLB for everything else.
  74 */
  75_GLOBAL(__tlbil_va)
  76        mfspr   r5,SPRN_MMUCR
  77        mfmsr   r10
  78
  79        /*
  80         * We write 16 bits of STID since 47x supports that much, we
  81         * will never be passed out of bounds values on 440 (hopefully)
  82         */
  83        rlwimi  r5,r4,0,16,31
  84
  85        /* We have to run the search with interrupts disabled, otherwise
  86         * an interrupt which causes a TLB miss can clobber the MMUCR
  87         * between the mtspr and the tlbsx.
  88         *
  89         * Critical and Machine Check interrupts take care of saving
  90         * and restoring MMUCR, so only normal interrupts have to be
  91         * taken care of.
  92         */
  93        wrteei  0
  94        mtspr   SPRN_MMUCR,r5
  95        tlbsx.  r6,0,r3
  96        bne     10f
  97        sync
  98BEGIN_MMU_FTR_SECTION
  99        b       2f
 100END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
 101        /* On 440 There are only 64 TLB entries, so r3 < 64, which means bit
 102         * 22, is clear.  Since 22 is the V bit in the TLB_PAGEID, loading this
 103         * value will invalidate the TLB entry.
 104         */
 105        tlbwe   r6,r6,PPC44x_TLB_PAGEID
 106        isync
 10710:     wrtee   r10
 108        blr
 1092:
 110#ifdef CONFIG_PPC_47x
 111        oris    r7,r6,0x8000    /* specify way explicitly */
 112        clrrwi  r4,r3,12        /* get an EPN for the hashing with V = 0 */
 113        ori     r4,r4,PPC47x_TLBE_SIZE
 114        tlbwe   r4,r7,0         /* write it */
 115        isync
 116        wrtee   r10
 117        blr
 118#else /* CONFIG_PPC_47x */
 1191:      trap
 120        EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;
 121#endif /* !CONFIG_PPC_47x */
 122
 123_GLOBAL(_tlbil_all)
 124_GLOBAL(_tlbil_pid)
 125BEGIN_MMU_FTR_SECTION
 126        b       2f
 127END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
 128        li      r3,0
 129        sync
 130
 131        /* Load high watermark */
 132        lis     r4,tlb_44x_hwater@ha
 133        lwz     r5,tlb_44x_hwater@l(r4)
 134
 1351:      tlbwe   r3,r3,PPC44x_TLB_PAGEID
 136        addi    r3,r3,1
 137        cmpw    0,r3,r5
 138        ble     1b
 139
 140        isync
 141        blr
 1422:
 143#ifdef CONFIG_PPC_47x
 144        /* 476 variant. There's not simple way to do this, hopefully we'll
 145         * try to limit the amount of such full invalidates
 146         */
 147        mfmsr   r11             /* Interrupts off */
 148        wrteei  0
 149        li      r3,-1           /* Current set */
 150        lis     r10,tlb_47x_boltmap@h
 151        ori     r10,r10,tlb_47x_boltmap@l
 152        lis     r7,0x8000       /* Specify way explicitly */
 153
 154        b       9f              /* For each set */
 155
 1561:      li      r9,4            /* Number of ways */
 157        li      r4,0            /* Current way */
 158        li      r6,0            /* Default entry value 0 */
 159        andi.   r0,r8,1         /* Check if way 0 is bolted */
 160        mtctr   r9              /* Load way counter */
 161        bne-    3f              /* Bolted, skip loading it */
 162
 1632:      /* For each way */
 164        or      r5,r3,r4        /* Make way|index for tlbre */
 165        rlwimi  r5,r5,16,8,15   /* Copy index into position */
 166        tlbre   r6,r5,0         /* Read entry */
 1673:      addis   r4,r4,0x2000    /* Next way */
 168        andi.   r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */
 169        beq     4f              /* Nope, skip it */
 170        rlwimi  r7,r5,0,1,2     /* Insert way number */
 171        rlwinm  r6,r6,0,21,19   /* Clear V */
 172        tlbwe   r6,r7,0         /* Write it */
 1734:      bdnz    2b              /* Loop for each way */
 174        srwi    r8,r8,1         /* Next boltmap bit */
 1759:      cmpwi   cr1,r3,255      /* Last set done ? */
 176        addi    r3,r3,1         /* Next set */
 177        beq     cr1,1f          /* End of loop */
 178        andi.   r0,r3,0x1f      /* Need to load a new boltmap word ? */
 179        bne     1b              /* No, loop */
 180        lwz     r8,0(r10)       /* Load boltmap entry */
 181        addi    r10,r10,4       /* Next word */
 182        b       1b              /* Then loop */
 1831:      isync                   /* Sync shadows */
 184        wrtee   r11
 185#else /* CONFIG_PPC_47x */
 1861:      trap
 187        EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;
 188#endif /* !CONFIG_PPC_47x */
 189        blr
 190
 191#ifdef CONFIG_PPC_47x
 192
 193/*
 194 * _tlbivax_bcast is only on 47x. We don't bother doing a runtime
 195 * check though, it will blow up soon enough if we mistakenly try
 196 * to use it on a 440.
 197 */
 198_GLOBAL(_tlbivax_bcast)
 199        mfspr   r5,SPRN_MMUCR
 200        mfmsr   r10
 201        rlwimi  r5,r4,0,16,31
 202        wrteei  0
 203        mtspr   SPRN_MMUCR,r5
 204        isync
 205        PPC_TLBIVAX(0, R3)
 206        isync
 207        eieio
 208        tlbsync
 209BEGIN_FTR_SECTION
 210        b       1f
 211END_FTR_SECTION_IFSET(CPU_FTR_476_DD2)
 212        sync
 213        wrtee   r10
 214        blr
 215/*
 216 * DD2 HW could hang if in instruction fetch happens before msync completes.
 217 * Touch enough instruction cache lines to ensure cache hits
 218 */
 2191:      mflr    r9
 220        bl      2f
 2212:      mflr    r6
 222        li      r7,32
 223        PPC_ICBT(0,R6,R7)               /* touch next cache line */
 224        add     r6,r6,r7
 225        PPC_ICBT(0,R6,R7)               /* touch next cache line */
 226        add     r6,r6,r7
 227        PPC_ICBT(0,R6,R7)               /* touch next cache line */
 228        sync
 229        nop
 230        nop
 231        nop
 232        nop
 233        nop
 234        nop
 235        nop
 236        nop
 237        mtlr    r9
 238        wrtee   r10
 239        blr
 240#endif /* CONFIG_PPC_47x */
 241
 242#elif defined(CONFIG_FSL_BOOKE)
 243/*
 244 * FSL BookE implementations.
 245 *
 246 * Since feature sections are using _SECTION_ELSE we need
 247 * to have the larger code path before the _SECTION_ELSE
 248 */
 249
 250/*
 251 * Flush MMU TLB on the local processor
 252 */
 253_GLOBAL(_tlbil_all)
 254BEGIN_MMU_FTR_SECTION
 255        li      r3,(MMUCSR0_TLBFI)@l
 256        mtspr   SPRN_MMUCSR0, r3
 2571:
 258        mfspr   r3,SPRN_MMUCSR0
 259        andi.   r3,r3,MMUCSR0_TLBFI@l
 260        bne     1b
 261MMU_FTR_SECTION_ELSE
 262        PPC_TLBILX_ALL(0,R0)
 263ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
 264        msync
 265        isync
 266        blr
 267
 268_GLOBAL(_tlbil_pid)
 269BEGIN_MMU_FTR_SECTION
 270        slwi    r3,r3,16
 271        mfmsr   r10
 272        wrteei  0
 273        mfspr   r4,SPRN_MAS6    /* save MAS6 */
 274        mtspr   SPRN_MAS6,r3
 275        PPC_TLBILX_PID(0,R0)
 276        mtspr   SPRN_MAS6,r4    /* restore MAS6 */
 277        wrtee   r10
 278MMU_FTR_SECTION_ELSE
 279        li      r3,(MMUCSR0_TLBFI)@l
 280        mtspr   SPRN_MMUCSR0, r3
 2811:
 282        mfspr   r3,SPRN_MMUCSR0
 283        andi.   r3,r3,MMUCSR0_TLBFI@l
 284        bne     1b
 285ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX)
 286        msync
 287        isync
 288        blr
 289
 290/*
 291 * Flush MMU TLB for a particular address, but only on the local processor
 292 * (no broadcast)
 293 */
 294_GLOBAL(__tlbil_va)
 295        mfmsr   r10
 296        wrteei  0
 297        slwi    r4,r4,16
 298        ori     r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l
 299        mtspr   SPRN_MAS6,r4            /* assume AS=0 for now */
 300BEGIN_MMU_FTR_SECTION
 301        tlbsx   0,r3
 302        mfspr   r4,SPRN_MAS1            /* check valid */
 303        andis.  r3,r4,MAS1_VALID@h
 304        beq     1f
 305        rlwinm  r4,r4,0,1,31
 306        mtspr   SPRN_MAS1,r4
 307        tlbwe
 308MMU_FTR_SECTION_ELSE
 309        PPC_TLBILX_VA(0,R3)
 310ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
 311        msync
 312        isync
 3131:      wrtee   r10
 314        blr
 315#elif defined(CONFIG_PPC_BOOK3E)
 316/*
 317 * New Book3E (>= 2.06) implementation
 318 *
 319 * Note: We may be able to get away without the interrupt masking stuff
 320 * if we save/restore MAS6 on exceptions that might modify it
 321 */
 322_GLOBAL(_tlbil_pid)
 323        slwi    r4,r3,MAS6_SPID_SHIFT
 324        mfmsr   r10
 325        wrteei  0
 326        mtspr   SPRN_MAS6,r4
 327        PPC_TLBILX_PID(0,R0)
 328        wrtee   r10
 329        msync
 330        isync
 331        blr
 332
 333_GLOBAL(_tlbil_pid_noind)
 334        slwi    r4,r3,MAS6_SPID_SHIFT
 335        mfmsr   r10
 336        ori     r4,r4,MAS6_SIND
 337        wrteei  0
 338        mtspr   SPRN_MAS6,r4
 339        PPC_TLBILX_PID(0,R0)
 340        wrtee   r10
 341        msync
 342        isync
 343        blr
 344
 345_GLOBAL(_tlbil_all)
 346        PPC_TLBILX_ALL(0,R0)
 347        msync
 348        isync
 349        blr
 350
 351_GLOBAL(_tlbil_va)
 352        mfmsr   r10
 353        wrteei  0
 354        cmpwi   cr0,r6,0
 355        slwi    r4,r4,MAS6_SPID_SHIFT
 356        rlwimi  r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
 357        beq     1f
 358        rlwimi  r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
 3591:      mtspr   SPRN_MAS6,r4            /* assume AS=0 for now */
 360        PPC_TLBILX_VA(0,R3)
 361        msync
 362        isync
 363        wrtee   r10
 364        blr
 365
 366_GLOBAL(_tlbivax_bcast)
 367        mfmsr   r10
 368        wrteei  0
 369        cmpwi   cr0,r6,0
 370        slwi    r4,r4,MAS6_SPID_SHIFT
 371        rlwimi  r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
 372        beq     1f
 373        rlwimi  r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
 3741:      mtspr   SPRN_MAS6,r4            /* assume AS=0 for now */
 375        PPC_TLBIVAX(0,R3)
 376        eieio
 377        tlbsync
 378        sync
 379        wrtee   r10
 380        blr
 381
 382_GLOBAL(set_context)
 383#ifdef CONFIG_BDI_SWITCH
 384        /* Context switch the PTE pointer for the Abatron BDI2000.
 385         * The PGDIR is the second parameter.
 386         */
 387        lis     r5, abatron_pteptrs@h
 388        ori     r5, r5, abatron_pteptrs@l
 389        stw     r4, 0x4(r5)
 390#endif
 391        mtspr   SPRN_PID,r3
 392        isync                   /* Force context change */
 393        blr
 394#else
 395#error Unsupported processor type !
 396#endif
 397
 398#if defined(CONFIG_PPC_FSL_BOOK3E)
 399/*
 400 * extern void loadcam_entry(unsigned int index)
 401 *
 402 * Load TLBCAM[index] entry in to the L2 CAM MMU
 403 * Must preserve r7, r8, r9, and r10
 404 */
 405_GLOBAL(loadcam_entry)
 406        mflr    r5
 407        LOAD_REG_ADDR_PIC(r4, TLBCAM)
 408        mtlr    r5
 409        mulli   r5,r3,TLBCAM_SIZE
 410        add     r3,r5,r4
 411        lwz     r4,TLBCAM_MAS0(r3)
 412        mtspr   SPRN_MAS0,r4
 413        lwz     r4,TLBCAM_MAS1(r3)
 414        mtspr   SPRN_MAS1,r4
 415        PPC_LL  r4,TLBCAM_MAS2(r3)
 416        mtspr   SPRN_MAS2,r4
 417        lwz     r4,TLBCAM_MAS3(r3)
 418        mtspr   SPRN_MAS3,r4
 419BEGIN_MMU_FTR_SECTION
 420        lwz     r4,TLBCAM_MAS7(r3)
 421        mtspr   SPRN_MAS7,r4
 422END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS)
 423        isync
 424        tlbwe
 425        isync
 426        blr
 427
 428/*
 429 * Load multiple TLB entries at once, using an alternate-space
 430 * trampoline so that we don't have to care about whether the same
 431 * TLB entry maps us before and after.
 432 *
 433 * r3 = first entry to write
 434 * r4 = number of entries to write
 435 * r5 = temporary tlb entry
 436 */
 437_GLOBAL(loadcam_multi)
 438        mflr    r8
 439
 440        /*
 441         * Set up temporary TLB entry that is the same as what we're
 442         * running from, but in AS=1.
 443         */
 444        bl      1f
 4451:      mflr    r6
 446        tlbsx   0,r8
 447        mfspr   r6,SPRN_MAS1
 448        ori     r6,r6,MAS1_TS
 449        mtspr   SPRN_MAS1,r6
 450        mfspr   r6,SPRN_MAS0
 451        rlwimi  r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
 452        mr      r7,r5
 453        mtspr   SPRN_MAS0,r6
 454        isync
 455        tlbwe
 456        isync
 457
 458        /* Switch to AS=1 */
 459        mfmsr   r6
 460        ori     r6,r6,MSR_IS|MSR_DS
 461        mtmsr   r6
 462        isync
 463
 464        mr      r9,r3
 465        add     r10,r3,r4
 4662:      bl      loadcam_entry
 467        addi    r9,r9,1
 468        cmpw    r9,r10
 469        mr      r3,r9
 470        blt     2b
 471
 472        /* Return to AS=0 and clear the temporary entry */
 473        mfmsr   r6
 474        rlwinm. r6,r6,0,~(MSR_IS|MSR_DS)
 475        mtmsr   r6
 476        isync
 477
 478        li      r6,0
 479        mtspr   SPRN_MAS1,r6
 480        rlwinm  r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
 481        oris    r6,r6,MAS0_TLBSEL(1)@h
 482        mtspr   SPRN_MAS0,r6
 483        isync
 484        tlbwe
 485        isync
 486
 487        mtlr    r8
 488        blr
 489#endif
 490