LXR linux/arch/powerpc/platforms/powermac/cache.S

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 * This file contains low-level cache management functions
   4 * used for sleep and CPU speed changes on Apple machines.
   5 * (In fact the only thing that is Apple-specific is that we assume
   6 * that we can read from ROM at physical address 0xfff00000.)
   7 *
   8 *    Copyright (C) 2004 Paul Mackerras (paulus@samba.org) and
   9 *                       Benjamin Herrenschmidt (benh@kernel.crashing.org)
  10 */
  11
  12#include <asm/processor.h>
  13#include <asm/ppc_asm.h>
  14#include <asm/cputable.h>
  15#include <asm/feature-fixups.h>
  16
  17/*
  18 * Flush and disable all data caches (dL1, L2, L3). This is used
  19 * when going to sleep, when doing a PMU based cpufreq transition,
  20 * or when "offlining" a CPU on SMP machines. This code is over
  21 * paranoid, but I've had enough issues with various CPU revs and
  22 * bugs that I decided it was worth being over cautious
  23 */
  24
  25_GLOBAL(flush_disable_caches)
  26#ifndef CONFIG_PPC_BOOK3S_32
  27        blr
  28#else
  29BEGIN_FTR_SECTION
  30        b       flush_disable_745x
  31END_FTR_SECTION_IFSET(CPU_FTR_SPEC7450)
  32BEGIN_FTR_SECTION
  33        b       flush_disable_75x
  34END_FTR_SECTION_IFSET(CPU_FTR_L2CR)
  35        b       __flush_disable_L1
  36
  37/* This is the code for G3 and 74[01]0 */
  38flush_disable_75x:
  39        mflr    r10
  40
  41        /* Turn off EE and DR in MSR */
  42        mfmsr   r11
  43        rlwinm  r0,r11,0,~MSR_EE
  44        rlwinm  r0,r0,0,~MSR_DR
  45        sync
  46        mtmsr   r0
  47        isync
  48
  49        /* Stop DST streams */
  50BEGIN_FTR_SECTION
  51        DSSALL
  52        sync
  53END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  54
  55        /* Stop DPM */
  56        mfspr   r8,SPRN_HID0            /* Save SPRN_HID0 in r8 */
  57        rlwinm  r4,r8,0,12,10           /* Turn off HID0[DPM] */
  58        sync
  59        mtspr   SPRN_HID0,r4            /* Disable DPM */
  60        sync
  61
  62        /* Disp-flush L1. We have a weird problem here that I never
  63         * totally figured out. On 750FX, using the ROM for the flush
  64         * results in a non-working flush. We use that workaround for
  65         * now until I finally understand what's going on. --BenH
  66         */
  67
  68        /* ROM base by default */
  69        lis     r4,0xfff0
  70        mfpvr   r3
  71        srwi    r3,r3,16
  72        cmplwi  cr0,r3,0x7000
  73        bne+    1f
  74        /* RAM base on 750FX */
  75        li      r4,0
  761:      li      r4,0x4000
  77        mtctr   r4
  781:      lwz     r0,0(r4)
  79        addi    r4,r4,32
  80        bdnz    1b
  81        sync
  82        isync
  83
  84        /* Disable / invalidate / enable L1 data */
  85        mfspr   r3,SPRN_HID0
  86        rlwinm  r3,r3,0,~(HID0_DCE | HID0_ICE)
  87        mtspr   SPRN_HID0,r3
  88        sync
  89        isync
  90        ori     r3,r3,(HID0_DCE|HID0_DCI|HID0_ICE|HID0_ICFI)
  91        sync
  92        isync
  93        mtspr   SPRN_HID0,r3
  94        xori    r3,r3,(HID0_DCI|HID0_ICFI)
  95        mtspr   SPRN_HID0,r3
  96        sync
  97
  98        /* Get the current enable bit of the L2CR into r4 */
  99        mfspr   r5,SPRN_L2CR
 100        /* Set to data-only (pre-745x bit) */
 101        oris    r3,r5,L2CR_L2DO@h
 102        b       2f
 103        /* When disabling L2, code must be in L1 */
 104        .balign 32
 1051:      mtspr   SPRN_L2CR,r3
 1063:      sync
 107        isync
 108        b       1f
 1092:      b       3f
 1103:      sync
 111        isync
 112        b       1b
 1131:      /* disp-flush L2. The interesting thing here is that the L2 can be
 114         * up to 2Mb ... so using the ROM, we'll end up wrapping back to memory
 115         * but that is probbaly fine. We disp-flush over 4Mb to be safe
 116         */
 117        lis     r4,2
 118        mtctr   r4
 119        lis     r4,0xfff0
 1201:      lwz     r0,0(r4)
 121        addi    r4,r4,32
 122        bdnz    1b
 123        sync
 124        isync
 125        lis     r4,2
 126        mtctr   r4
 127        lis     r4,0xfff0
 1281:      dcbf    0,r4
 129        addi    r4,r4,32
 130        bdnz    1b
 131        sync
 132        isync
 133
 134        /* now disable L2 */
 135        rlwinm  r5,r5,0,~L2CR_L2E
 136        b       2f
 137        /* When disabling L2, code must be in L1 */
 138        .balign 32
 1391:      mtspr   SPRN_L2CR,r5
 1403:      sync
 141        isync
 142        b       1f
 1432:      b       3f
 1443:      sync
 145        isync
 146        b       1b
 1471:      sync
 148        isync
 149        /* Invalidate L2. This is pre-745x, we clear the L2I bit ourselves */
 150        oris    r4,r5,L2CR_L2I@h
 151        mtspr   SPRN_L2CR,r4
 152        sync
 153        isync
 154
 155        /* Wait for the invalidation to complete */
 1561:      mfspr   r3,SPRN_L2CR
 157        rlwinm. r0,r3,0,31,31
 158        bne     1b
 159
 160        /* Clear L2I */
 161        xoris   r4,r4,L2CR_L2I@h
 162        sync
 163        mtspr   SPRN_L2CR,r4
 164        sync
 165
 166        /* now disable the L1 data cache */
 167        mfspr   r0,SPRN_HID0
 168        rlwinm  r0,r0,0,~(HID0_DCE|HID0_ICE)
 169        mtspr   SPRN_HID0,r0
 170        sync
 171        isync
 172
 173        /* Restore HID0[DPM] to whatever it was before */
 174        sync
 175        mfspr   r0,SPRN_HID0
 176        rlwimi  r0,r8,0,11,11           /* Turn back HID0[DPM] */
 177        mtspr   SPRN_HID0,r0
 178        sync
 179
 180        /* restore DR and EE */
 181        sync
 182        mtmsr   r11
 183        isync
 184
 185        mtlr    r10
 186        blr
 187_ASM_NOKPROBE_SYMBOL(flush_disable_75x)
 188
 189/* This code is for 745x processors */
 190flush_disable_745x:
 191        /* Turn off EE and DR in MSR */
 192        mfmsr   r11
 193        rlwinm  r0,r11,0,~MSR_EE
 194        rlwinm  r0,r0,0,~MSR_DR
 195        sync
 196        mtmsr   r0
 197        isync
 198
 199        /* Stop prefetch streams */
 200        DSSALL
 201        sync
 202
 203        /* Disable L2 prefetching */
 204        mfspr   r0,SPRN_MSSCR0
 205        rlwinm  r0,r0,0,0,29
 206        mtspr   SPRN_MSSCR0,r0
 207        sync
 208        isync
 209        lis     r4,0
 210        dcbf    0,r4
 211        dcbf    0,r4
 212        dcbf    0,r4
 213        dcbf    0,r4
 214        dcbf    0,r4
 215        dcbf    0,r4
 216        dcbf    0,r4
 217        dcbf    0,r4
 218
 219        /* Due to a bug with the HW flush on some CPU revs, we occasionally
 220         * experience data corruption. I'm adding a displacement flush along
 221         * with a dcbf loop over a few Mb to "help". The problem isn't totally
 222         * fixed by this in theory, but at least, in practice, I couldn't reproduce
 223         * it even with a big hammer...
 224         */
 225
 226        lis     r4,0x0002
 227        mtctr   r4
 228        li      r4,0
 2291:
 230        lwz     r0,0(r4)
 231        addi    r4,r4,32                /* Go to start of next cache line */
 232        bdnz    1b
 233        isync
 234
 235        /* Now, flush the first 4MB of memory */
 236        lis     r4,0x0002
 237        mtctr   r4
 238        li      r4,0
 239        sync
 2401:
 241        dcbf    0,r4
 242        addi    r4,r4,32                /* Go to start of next cache line */
 243        bdnz    1b
 244
 245        /* Flush and disable the L1 data cache */
 246        mfspr   r6,SPRN_LDSTCR
 247        lis     r3,0xfff0       /* read from ROM for displacement flush */
 248        li      r4,0xfe         /* start with only way 0 unlocked */
 249        li      r5,128          /* 128 lines in each way */
 2501:      mtctr   r5
 251        rlwimi  r6,r4,0,24,31
 252        mtspr   SPRN_LDSTCR,r6
 253        sync
 254        isync
 2552:      lwz     r0,0(r3)        /* touch each cache line */
 256        addi    r3,r3,32
 257        bdnz    2b
 258        rlwinm  r4,r4,1,24,30   /* move on to the next way */
 259        ori     r4,r4,1
 260        cmpwi   r4,0xff         /* all done? */
 261        bne     1b
 262        /* now unlock the L1 data cache */
 263        li      r4,0
 264        rlwimi  r6,r4,0,24,31
 265        sync
 266        mtspr   SPRN_LDSTCR,r6
 267        sync
 268        isync
 269
 270        /* Flush the L2 cache using the hardware assist */
 271        mfspr   r3,SPRN_L2CR
 272        cmpwi   r3,0            /* check if it is enabled first */
 273        bge     4f
 274        oris    r0,r3,(L2CR_L2IO_745x|L2CR_L2DO_745x)@h
 275        b       2f
 276        /* When disabling/locking L2, code must be in L1 */
 277        .balign 32
 2781:      mtspr   SPRN_L2CR,r0    /* lock the L2 cache */
 2793:      sync
 280        isync
 281        b       1f
 2822:      b       3f
 2833:      sync
 284        isync
 285        b       1b
 2861:      sync
 287        isync
 288        ori     r0,r3,L2CR_L2HWF_745x
 289        sync
 290        mtspr   SPRN_L2CR,r0    /* set the hardware flush bit */
 2913:      mfspr   r0,SPRN_L2CR    /* wait for it to go to 0 */
 292        andi.   r0,r0,L2CR_L2HWF_745x
 293        bne     3b
 294        sync
 295        rlwinm  r3,r3,0,~L2CR_L2E
 296        b       2f
 297        /* When disabling L2, code must be in L1 */
 298        .balign 32
 2991:      mtspr   SPRN_L2CR,r3    /* disable the L2 cache */
 3003:      sync
 301        isync
 302        b       1f
 3032:      b       3f
 3043:      sync
 305        isync
 306        b       1b
 3071:      sync
 308        isync
 309        oris    r4,r3,L2CR_L2I@h
 310        mtspr   SPRN_L2CR,r4
 311        sync
 312        isync
 3131:      mfspr   r4,SPRN_L2CR
 314        andis.  r0,r4,L2CR_L2I@h
 315        bne     1b
 316        sync
 317
 318BEGIN_FTR_SECTION
 319        /* Flush the L3 cache using the hardware assist */
 3204:      mfspr   r3,SPRN_L3CR
 321        cmpwi   r3,0            /* check if it is enabled */
 322        bge     6f
 323        oris    r0,r3,L3CR_L3IO@h
 324        ori     r0,r0,L3CR_L3DO
 325        sync
 326        mtspr   SPRN_L3CR,r0    /* lock the L3 cache */
 327        sync
 328        isync
 329        ori     r0,r0,L3CR_L3HWF
 330        sync
 331        mtspr   SPRN_L3CR,r0    /* set the hardware flush bit */
 3325:      mfspr   r0,SPRN_L3CR    /* wait for it to go to zero */
 333        andi.   r0,r0,L3CR_L3HWF
 334        bne     5b
 335        rlwinm  r3,r3,0,~L3CR_L3E
 336        sync
 337        mtspr   SPRN_L3CR,r3    /* disable the L3 cache */
 338        sync
 339        ori     r4,r3,L3CR_L3I
 340        mtspr   SPRN_L3CR,r4
 3411:      mfspr   r4,SPRN_L3CR
 342        andi.   r0,r4,L3CR_L3I
 343        bne     1b
 344        sync
 345END_FTR_SECTION_IFSET(CPU_FTR_L3CR)
 346
 3476:      mfspr   r0,SPRN_HID0    /* now disable the L1 data cache */
 348        rlwinm  r0,r0,0,~HID0_DCE
 349        mtspr   SPRN_HID0,r0
 350        sync
 351        isync
 352        mtmsr   r11             /* restore DR and EE */
 353        isync
 354        blr
 355_ASM_NOKPROBE_SYMBOL(flush_disable_745x)
 356#endif  /* CONFIG_PPC_BOOK3S_32 */
 357