LXR linux/arch/powerpc/mm/nohash/tlb

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 * This file contains low-level functions for performing various
   4 * types of TLB invalidations on various processors with no hash
   5 * table.
   6 *
   7 * This file implements the following functions for all no-hash
   8 * processors. Some aren't implemented for some variants. Some
   9 * are inline in tlbflush.h
  10 *
  11 *      - tlbil_va
  12 *      - tlbil_pid
  13 *      - tlbil_all
  14 *      - tlbivax_bcast
  15 *
  16 * Code mostly moved over from misc_32.S
  17 *
  18 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  19 *
  20 * Partially rewritten by Cort Dougan (cort@cs.nmt.edu)
  21 * Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt.
  22 */
  23
  24#include <asm/reg.h>
  25#include <asm/page.h>
  26#include <asm/cputable.h>
  27#include <asm/mmu.h>
  28#include <asm/ppc_asm.h>
  29#include <asm/asm-offsets.h>
  30#include <asm/processor.h>
  31#include <asm/bug.h>
  32#include <asm/asm-compat.h>
  33#include <asm/feature-fixups.h>
  34
  35#if defined(CONFIG_40x)
  36
  37/*
  38 * 40x implementation needs only tlbil_va
  39 */
  40_GLOBAL(__tlbil_va)
  41        /* We run the search with interrupts disabled because we have to change
  42         * the PID and I don't want to preempt when that happens.
  43         */
  44        mfmsr   r5
  45        mfspr   r6,SPRN_PID
  46        wrteei  0
  47        mtspr   SPRN_PID,r4
  48        tlbsx.  r3, 0, r3
  49        mtspr   SPRN_PID,r6
  50        wrtee   r5
  51        bne     1f
  52        sync
  53        /* There are only 64 TLB entries, so r3 < 64, which means bit 25 is
  54         * clear. Since 25 is the V bit in the TLB_TAG, loading this value
  55         * will invalidate the TLB entry. */
  56        tlbwe   r3, r3, TLB_TAG
  57        isync
  581:      blr
  59
  60#elif defined(CONFIG_PPC_8xx)
  61
  62/*
  63 * Nothing to do for 8xx, everything is inline
  64 */
  65
  66#elif defined(CONFIG_44x) /* Includes 47x */
  67
  68/*
  69 * 440 implementation uses tlbsx/we for tlbil_va and a full sweep
  70 * of the TLB for everything else.
  71 */
  72_GLOBAL(__tlbil_va)
  73        mfspr   r5,SPRN_MMUCR
  74        mfmsr   r10
  75
  76        /*
  77         * We write 16 bits of STID since 47x supports that much, we
  78         * will never be passed out of bounds values on 440 (hopefully)
  79         */
  80        rlwimi  r5,r4,0,16,31
  81
  82        /* We have to run the search with interrupts disabled, otherwise
  83         * an interrupt which causes a TLB miss can clobber the MMUCR
  84         * between the mtspr and the tlbsx.
  85         *
  86         * Critical and Machine Check interrupts take care of saving
  87         * and restoring MMUCR, so only normal interrupts have to be
  88         * taken care of.
  89         */
  90        wrteei  0
  91        mtspr   SPRN_MMUCR,r5
  92        tlbsx.  r6,0,r3
  93        bne     10f
  94        sync
  95BEGIN_MMU_FTR_SECTION
  96        b       2f
  97END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
  98        /* On 440 There are only 64 TLB entries, so r3 < 64, which means bit
  99         * 22, is clear.  Since 22 is the V bit in the TLB_PAGEID, loading this
 100         * value will invalidate the TLB entry.
 101         */
 102        tlbwe   r6,r6,PPC44x_TLB_PAGEID
 103        isync
 10410:     wrtee   r10
 105        blr
 1062:
 107#ifdef CONFIG_PPC_47x
 108        oris    r7,r6,0x8000    /* specify way explicitly */
 109        clrrwi  r4,r3,12        /* get an EPN for the hashing with V = 0 */
 110        ori     r4,r4,PPC47x_TLBE_SIZE
 111        tlbwe   r4,r7,0         /* write it */
 112        isync
 113        wrtee   r10
 114        blr
 115#else /* CONFIG_PPC_47x */
 1161:      trap
 117        EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;
 118#endif /* !CONFIG_PPC_47x */
 119
 120_GLOBAL(_tlbil_all)
 121_GLOBAL(_tlbil_pid)
 122BEGIN_MMU_FTR_SECTION
 123        b       2f
 124END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
 125        li      r3,0
 126        sync
 127
 128        /* Load high watermark */
 129        lis     r4,tlb_44x_hwater@ha
 130        lwz     r5,tlb_44x_hwater@l(r4)
 131
 1321:      tlbwe   r3,r3,PPC44x_TLB_PAGEID
 133        addi    r3,r3,1
 134        cmpw    0,r3,r5
 135        ble     1b
 136
 137        isync
 138        blr
 1392:
 140#ifdef CONFIG_PPC_47x
 141        /* 476 variant. There's not simple way to do this, hopefully we'll
 142         * try to limit the amount of such full invalidates
 143         */
 144        mfmsr   r11             /* Interrupts off */
 145        wrteei  0
 146        li      r3,-1           /* Current set */
 147        lis     r10,tlb_47x_boltmap@h
 148        ori     r10,r10,tlb_47x_boltmap@l
 149        lis     r7,0x8000       /* Specify way explicitly */
 150
 151        b       9f              /* For each set */
 152
 1531:      li      r9,4            /* Number of ways */
 154        li      r4,0            /* Current way */
 155        li      r6,0            /* Default entry value 0 */
 156        andi.   r0,r8,1         /* Check if way 0 is bolted */
 157        mtctr   r9              /* Load way counter */
 158        bne-    3f              /* Bolted, skip loading it */
 159
 1602:      /* For each way */
 161        or      r5,r3,r4        /* Make way|index for tlbre */
 162        rlwimi  r5,r5,16,8,15   /* Copy index into position */
 163        tlbre   r6,r5,0         /* Read entry */
 1643:      addis   r4,r4,0x2000    /* Next way */
 165        andi.   r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */
 166        beq     4f              /* Nope, skip it */
 167        rlwimi  r7,r5,0,1,2     /* Insert way number */
 168        rlwinm  r6,r6,0,21,19   /* Clear V */
 169        tlbwe   r6,r7,0         /* Write it */
 1704:      bdnz    2b              /* Loop for each way */
 171        srwi    r8,r8,1         /* Next boltmap bit */
 1729:      cmpwi   cr1,r3,255      /* Last set done ? */
 173        addi    r3,r3,1         /* Next set */
 174        beq     cr1,1f          /* End of loop */
 175        andi.   r0,r3,0x1f      /* Need to load a new boltmap word ? */
 176        bne     1b              /* No, loop */
 177        lwz     r8,0(r10)       /* Load boltmap entry */
 178        addi    r10,r10,4       /* Next word */
 179        b       1b              /* Then loop */
 1801:      isync                   /* Sync shadows */
 181        wrtee   r11
 182#else /* CONFIG_PPC_47x */
 1831:      trap
 184        EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;
 185#endif /* !CONFIG_PPC_47x */
 186        blr
 187
 188#ifdef CONFIG_PPC_47x
 189
 190/*
 191 * _tlbivax_bcast is only on 47x. We don't bother doing a runtime
 192 * check though, it will blow up soon enough if we mistakenly try
 193 * to use it on a 440.
 194 */
 195_GLOBAL(_tlbivax_bcast)
 196        mfspr   r5,SPRN_MMUCR
 197        mfmsr   r10
 198        rlwimi  r5,r4,0,16,31
 199        wrteei  0
 200        mtspr   SPRN_MMUCR,r5
 201        isync
 202        PPC_TLBIVAX(0, R3)
 203        isync
 204        eieio
 205        tlbsync
 206BEGIN_FTR_SECTION
 207        b       1f
 208END_FTR_SECTION_IFSET(CPU_FTR_476_DD2)
 209        sync
 210        wrtee   r10
 211        blr
 212/*
 213 * DD2 HW could hang if in instruction fetch happens before msync completes.
 214 * Touch enough instruction cache lines to ensure cache hits
 215 */
 2161:      mflr    r9
 217        bl      2f
 2182:      mflr    r6
 219        li      r7,32
 220        PPC_ICBT(0,R6,R7)               /* touch next cache line */
 221        add     r6,r6,r7
 222        PPC_ICBT(0,R6,R7)               /* touch next cache line */
 223        add     r6,r6,r7
 224        PPC_ICBT(0,R6,R7)               /* touch next cache line */
 225        sync
 226        nop
 227        nop
 228        nop
 229        nop
 230        nop
 231        nop
 232        nop
 233        nop
 234        mtlr    r9
 235        wrtee   r10
 236        blr
 237#endif /* CONFIG_PPC_47x */
 238
 239#elif defined(CONFIG_FSL_BOOKE)
 240/*
 241 * FSL BookE implementations.
 242 *
 243 * Since feature sections are using _SECTION_ELSE we need
 244 * to have the larger code path before the _SECTION_ELSE
 245 */
 246
 247/*
 248 * Flush MMU TLB on the local processor
 249 */
 250_GLOBAL(_tlbil_all)
 251BEGIN_MMU_FTR_SECTION
 252        li      r3,(MMUCSR0_TLBFI)@l
 253        mtspr   SPRN_MMUCSR0, r3
 2541:
 255        mfspr   r3,SPRN_MMUCSR0
 256        andi.   r3,r3,MMUCSR0_TLBFI@l
 257        bne     1b
 258MMU_FTR_SECTION_ELSE
 259        PPC_TLBILX_ALL(0,R0)
 260ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
 261        msync
 262        isync
 263        blr
 264
 265_GLOBAL(_tlbil_pid)
 266BEGIN_MMU_FTR_SECTION
 267        slwi    r3,r3,16
 268        mfmsr   r10
 269        wrteei  0
 270        mfspr   r4,SPRN_MAS6    /* save MAS6 */
 271        mtspr   SPRN_MAS6,r3
 272        PPC_TLBILX_PID(0,R0)
 273        mtspr   SPRN_MAS6,r4    /* restore MAS6 */
 274        wrtee   r10
 275MMU_FTR_SECTION_ELSE
 276        li      r3,(MMUCSR0_TLBFI)@l
 277        mtspr   SPRN_MMUCSR0, r3
 2781:
 279        mfspr   r3,SPRN_MMUCSR0
 280        andi.   r3,r3,MMUCSR0_TLBFI@l
 281        bne     1b
 282ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX)
 283        msync
 284        isync
 285        blr
 286
 287/*
 288 * Flush MMU TLB for a particular address, but only on the local processor
 289 * (no broadcast)
 290 */
 291_GLOBAL(__tlbil_va)
 292        mfmsr   r10
 293        wrteei  0
 294        slwi    r4,r4,16
 295        ori     r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l
 296        mtspr   SPRN_MAS6,r4            /* assume AS=0 for now */
 297BEGIN_MMU_FTR_SECTION
 298        tlbsx   0,r3
 299        mfspr   r4,SPRN_MAS1            /* check valid */
 300        andis.  r3,r4,MAS1_VALID@h
 301        beq     1f
 302        rlwinm  r4,r4,0,1,31
 303        mtspr   SPRN_MAS1,r4
 304        tlbwe
 305MMU_FTR_SECTION_ELSE
 306        PPC_TLBILX_VA(0,R3)
 307ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
 308        msync
 309        isync
 3101:      wrtee   r10
 311        blr
 312#elif defined(CONFIG_PPC_BOOK3E)
 313/*
 314 * New Book3E (>= 2.06) implementation
 315 *
 316 * Note: We may be able to get away without the interrupt masking stuff
 317 * if we save/restore MAS6 on exceptions that might modify it
 318 */
 319_GLOBAL(_tlbil_pid)
 320        slwi    r4,r3,MAS6_SPID_SHIFT
 321        mfmsr   r10
 322        wrteei  0
 323        mtspr   SPRN_MAS6,r4
 324        PPC_TLBILX_PID(0,R0)
 325        wrtee   r10
 326        msync
 327        isync
 328        blr
 329
 330_GLOBAL(_tlbil_pid_noind)
 331        slwi    r4,r3,MAS6_SPID_SHIFT
 332        mfmsr   r10
 333        ori     r4,r4,MAS6_SIND
 334        wrteei  0
 335        mtspr   SPRN_MAS6,r4
 336        PPC_TLBILX_PID(0,R0)
 337        wrtee   r10
 338        msync
 339        isync
 340        blr
 341
 342_GLOBAL(_tlbil_all)
 343        PPC_TLBILX_ALL(0,R0)
 344        msync
 345        isync
 346        blr
 347
 348_GLOBAL(_tlbil_va)
 349        mfmsr   r10
 350        wrteei  0
 351        cmpwi   cr0,r6,0
 352        slwi    r4,r4,MAS6_SPID_SHIFT
 353        rlwimi  r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
 354        beq     1f
 355        rlwimi  r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
 3561:      mtspr   SPRN_MAS6,r4            /* assume AS=0 for now */
 357        PPC_TLBILX_VA(0,R3)
 358        msync
 359        isync
 360        wrtee   r10
 361        blr
 362
 363_GLOBAL(_tlbivax_bcast)
 364        mfmsr   r10
 365        wrteei  0
 366        cmpwi   cr0,r6,0
 367        slwi    r4,r4,MAS6_SPID_SHIFT
 368        rlwimi  r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
 369        beq     1f
 370        rlwimi  r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
 3711:      mtspr   SPRN_MAS6,r4            /* assume AS=0 for now */
 372        PPC_TLBIVAX(0,R3)
 373        eieio
 374        tlbsync
 375        sync
 376        wrtee   r10
 377        blr
 378
 379_GLOBAL(set_context)
 380#ifdef CONFIG_BDI_SWITCH
 381        /* Context switch the PTE pointer for the Abatron BDI2000.
 382         * The PGDIR is the second parameter.
 383         */
 384        lis     r5, abatron_pteptrs@h
 385        ori     r5, r5, abatron_pteptrs@l
 386        stw     r4, 0x4(r5)
 387#endif
 388        mtspr   SPRN_PID,r3
 389        isync                   /* Force context change */
 390        blr
 391#else
 392#error Unsupported processor type !
 393#endif
 394
 395#if defined(CONFIG_PPC_FSL_BOOK3E)
 396/*
 397 * extern void loadcam_entry(unsigned int index)
 398 *
 399 * Load TLBCAM[index] entry in to the L2 CAM MMU
 400 * Must preserve r7, r8, r9, r10 and r11
 401 */
 402_GLOBAL(loadcam_entry)
 403        mflr    r5
 404        LOAD_REG_ADDR_PIC(r4, TLBCAM)
 405        mtlr    r5
 406        mulli   r5,r3,TLBCAM_SIZE
 407        add     r3,r5,r4
 408        lwz     r4,TLBCAM_MAS0(r3)
 409        mtspr   SPRN_MAS0,r4
 410        lwz     r4,TLBCAM_MAS1(r3)
 411        mtspr   SPRN_MAS1,r4
 412        PPC_LL  r4,TLBCAM_MAS2(r3)
 413        mtspr   SPRN_MAS2,r4
 414        lwz     r4,TLBCAM_MAS3(r3)
 415        mtspr   SPRN_MAS3,r4
 416BEGIN_MMU_FTR_SECTION
 417        lwz     r4,TLBCAM_MAS7(r3)
 418        mtspr   SPRN_MAS7,r4
 419END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS)
 420        isync
 421        tlbwe
 422        isync
 423        blr
 424
 425/*
 426 * Load multiple TLB entries at once, using an alternate-space
 427 * trampoline so that we don't have to care about whether the same
 428 * TLB entry maps us before and after.
 429 *
 430 * r3 = first entry to write
 431 * r4 = number of entries to write
 432 * r5 = temporary tlb entry
 433 */
 434_GLOBAL(loadcam_multi)
 435        mflr    r8
 436        /* Don't switch to AS=1 if already there */
 437        mfmsr   r11
 438        andi.   r11,r11,MSR_IS
 439        bne     10f
 440
 441        /*
 442         * Set up temporary TLB entry that is the same as what we're
 443         * running from, but in AS=1.
 444         */
 445        bl      1f
 4461:      mflr    r6
 447        tlbsx   0,r8
 448        mfspr   r6,SPRN_MAS1
 449        ori     r6,r6,MAS1_TS
 450        mtspr   SPRN_MAS1,r6
 451        mfspr   r6,SPRN_MAS0
 452        rlwimi  r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
 453        mr      r7,r5
 454        mtspr   SPRN_MAS0,r6
 455        isync
 456        tlbwe
 457        isync
 458
 459        /* Switch to AS=1 */
 460        mfmsr   r6
 461        ori     r6,r6,MSR_IS|MSR_DS
 462        mtmsr   r6
 463        isync
 464
 46510:
 466        mr      r9,r3
 467        add     r10,r3,r4
 4682:      bl      loadcam_entry
 469        addi    r9,r9,1
 470        cmpw    r9,r10
 471        mr      r3,r9
 472        blt     2b
 473
 474        /* Don't return to AS=0 if we were in AS=1 at function start */
 475        andi.   r11,r11,MSR_IS
 476        bne     3f
 477
 478        /* Return to AS=0 and clear the temporary entry */
 479        mfmsr   r6
 480        rlwinm. r6,r6,0,~(MSR_IS|MSR_DS)
 481        mtmsr   r6
 482        isync
 483
 484        li      r6,0
 485        mtspr   SPRN_MAS1,r6
 486        rlwinm  r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
 487        oris    r6,r6,MAS0_TLBSEL(1)@h
 488        mtspr   SPRN_MAS0,r6
 489        isync
 490        tlbwe
 491        isync
 492
 4933:
 494        mtlr    r8
 495        blr
 496#endif
 497