LXR linux/arch/powerpc/kernel/misc

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 * This file contains miscellaneous low-level functions.
   4 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   5 *
   6 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
   7 * and Paul Mackerras.
   8 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
   9 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
  10 */
  11
  12#include <linux/sys.h>
  13#include <asm/unistd.h>
  14#include <asm/errno.h>
  15#include <asm/processor.h>
  16#include <asm/page.h>
  17#include <asm/cache.h>
  18#include <asm/ppc_asm.h>
  19#include <asm/asm-offsets.h>
  20#include <asm/cputable.h>
  21#include <asm/thread_info.h>
  22#include <asm/kexec.h>
  23#include <asm/ptrace.h>
  24#include <asm/mmu.h>
  25#include <asm/export.h>
  26#include <asm/feature-fixups.h>
  27
  28        .text
  29
  30_GLOBAL(call_do_softirq)
  31        mflr    r0
  32        std     r0,16(r1)
  33        stdu    r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
  34        mr      r1,r3
  35        bl      __do_softirq
  36        ld      r1,0(r1)
  37        ld      r0,16(r1)
  38        mtlr    r0
  39        blr
  40
  41_GLOBAL(call_do_irq)
  42        mflr    r0
  43        std     r0,16(r1)
  44        stdu    r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
  45        mr      r1,r4
  46        bl      __do_irq
  47        ld      r1,0(r1)
  48        ld      r0,16(r1)
  49        mtlr    r0
  50        blr
  51
  52        .section        ".toc","aw"
  53PPC64_CACHES:
  54        .tc             ppc64_caches[TC],ppc64_caches
  55        .section        ".text"
  56
  57/*
  58 * Write any modified data cache blocks out to memory
  59 * and invalidate the corresponding instruction cache blocks.
  60 *
  61 * flush_icache_range(unsigned long start, unsigned long stop)
  62 *
  63 *   flush all bytes from start through stop-1 inclusive
  64 */
  65
  66_GLOBAL_TOC(flush_icache_range)
  67BEGIN_FTR_SECTION
  68        PURGE_PREFETCHED_INS
  69        blr
  70END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
  71/*
  72 * Flush the data cache to memory 
  73 * 
  74 * Different systems have different cache line sizes
  75 * and in some cases i-cache and d-cache line sizes differ from
  76 * each other.
  77 */
  78        ld      r10,PPC64_CACHES@toc(r2)
  79        lwz     r7,DCACHEL1BLOCKSIZE(r10)/* Get cache block size */
  80        addi    r5,r7,-1
  81        andc    r6,r3,r5                /* round low to line bdy */
  82        subf    r8,r6,r4                /* compute length */
  83        add     r8,r8,r5                /* ensure we get enough */
  84        lwz     r9,DCACHEL1LOGBLOCKSIZE(r10)    /* Get log-2 of cache block size */
  85        srw.    r8,r8,r9                /* compute line count */
  86        beqlr                           /* nothing to do? */
  87        mtctr   r8
  881:      dcbst   0,r6
  89        add     r6,r6,r7
  90        bdnz    1b
  91        sync
  92
  93/* Now invalidate the instruction cache */
  94        
  95        lwz     r7,ICACHEL1BLOCKSIZE(r10)       /* Get Icache block size */
  96        addi    r5,r7,-1
  97        andc    r6,r3,r5                /* round low to line bdy */
  98        subf    r8,r6,r4                /* compute length */
  99        add     r8,r8,r5
 100        lwz     r9,ICACHEL1LOGBLOCKSIZE(r10)    /* Get log-2 of Icache block size */
 101        srw.    r8,r8,r9                /* compute line count */
 102        beqlr                           /* nothing to do? */
 103        mtctr   r8
 1042:      icbi    0,r6
 105        add     r6,r6,r7
 106        bdnz    2b
 107        isync
 108        blr
 109_ASM_NOKPROBE_SYMBOL(flush_icache_range)
 110EXPORT_SYMBOL(flush_icache_range)
 111
 112/*
 113 * Flush a particular page from the data cache to RAM.
 114 * Note: this is necessary because the instruction cache does *not*
 115 * snoop from the data cache.
 116 *
 117 *      void __flush_dcache_icache(void *page)
 118 */
 119_GLOBAL(__flush_dcache_icache)
 120/*
 121 * Flush the data cache to memory 
 122 * 
 123 * Different systems have different cache line sizes
 124 */
 125
 126BEGIN_FTR_SECTION
 127        PURGE_PREFETCHED_INS
 128        blr
 129END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
 130
 131/* Flush the dcache */
 132        ld      r7,PPC64_CACHES@toc(r2)
 133        clrrdi  r3,r3,PAGE_SHIFT                    /* Page align */
 134        lwz     r4,DCACHEL1BLOCKSPERPAGE(r7)    /* Get # dcache blocks per page */
 135        lwz     r5,DCACHEL1BLOCKSIZE(r7)        /* Get dcache block size */
 136        mr      r6,r3
 137        mtctr   r4
 1380:      dcbst   0,r6
 139        add     r6,r6,r5
 140        bdnz    0b
 141        sync
 142
 143/* Now invalidate the icache */ 
 144
 145        lwz     r4,ICACHEL1BLOCKSPERPAGE(r7)    /* Get # icache blocks per page */
 146        lwz     r5,ICACHEL1BLOCKSIZE(r7)        /* Get icache block size */
 147        mtctr   r4
 1481:      icbi    0,r3
 149        add     r3,r3,r5
 150        bdnz    1b
 151        isync
 152        blr
 153
 154_GLOBAL(__bswapdi2)
 155EXPORT_SYMBOL(__bswapdi2)
 156        srdi    r8,r3,32
 157        rlwinm  r7,r3,8,0xffffffff
 158        rlwimi  r7,r3,24,0,7
 159        rlwinm  r9,r8,8,0xffffffff
 160        rlwimi  r7,r3,24,16,23
 161        rlwimi  r9,r8,24,0,7
 162        rlwimi  r9,r8,24,16,23
 163        sldi    r7,r7,32
 164        or      r3,r7,r9
 165        blr
 166
 167
 168#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
 169_GLOBAL(rmci_on)
 170        sync
 171        isync
 172        li      r3,0x100
 173        rldicl  r3,r3,32,0
 174        mfspr   r5,SPRN_HID4
 175        or      r5,r5,r3
 176        sync
 177        mtspr   SPRN_HID4,r5
 178        isync
 179        slbia
 180        isync
 181        sync
 182        blr
 183
 184_GLOBAL(rmci_off)
 185        sync
 186        isync
 187        li      r3,0x100
 188        rldicl  r3,r3,32,0
 189        mfspr   r5,SPRN_HID4
 190        andc    r5,r5,r3
 191        sync
 192        mtspr   SPRN_HID4,r5
 193        isync
 194        slbia
 195        isync
 196        sync
 197        blr
 198#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
 199
 200#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
 201
 202/*
 203 * Do an IO access in real mode
 204 */
 205_GLOBAL(real_readb)
 206        mfmsr   r7
 207        ori     r0,r7,MSR_DR
 208        xori    r0,r0,MSR_DR
 209        sync
 210        mtmsrd  r0
 211        sync
 212        isync
 213        mfspr   r6,SPRN_HID4
 214        rldicl  r5,r6,32,0
 215        ori     r5,r5,0x100
 216        rldicl  r5,r5,32,0
 217        sync
 218        mtspr   SPRN_HID4,r5
 219        isync
 220        slbia
 221        isync
 222        lbz     r3,0(r3)
 223        sync
 224        mtspr   SPRN_HID4,r6
 225        isync
 226        slbia
 227        isync
 228        mtmsrd  r7
 229        sync
 230        isync
 231        blr
 232
 233        /*
 234 * Do an IO access in real mode
 235 */
 236_GLOBAL(real_writeb)
 237        mfmsr   r7
 238        ori     r0,r7,MSR_DR
 239        xori    r0,r0,MSR_DR
 240        sync
 241        mtmsrd  r0
 242        sync
 243        isync
 244        mfspr   r6,SPRN_HID4
 245        rldicl  r5,r6,32,0
 246        ori     r5,r5,0x100
 247        rldicl  r5,r5,32,0
 248        sync
 249        mtspr   SPRN_HID4,r5
 250        isync
 251        slbia
 252        isync
 253        stb     r3,0(r4)
 254        sync
 255        mtspr   SPRN_HID4,r6
 256        isync
 257        slbia
 258        isync
 259        mtmsrd  r7
 260        sync
 261        isync
 262        blr
 263#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
 264
 265#ifdef CONFIG_PPC_PASEMI
 266
 267_GLOBAL(real_205_readb)
 268        mfmsr   r7
 269        ori     r0,r7,MSR_DR
 270        xori    r0,r0,MSR_DR
 271        sync
 272        mtmsrd  r0
 273        sync
 274        isync
 275        LBZCIX(R3,R0,R3)
 276        isync
 277        mtmsrd  r7
 278        sync
 279        isync
 280        blr
 281
 282_GLOBAL(real_205_writeb)
 283        mfmsr   r7
 284        ori     r0,r7,MSR_DR
 285        xori    r0,r0,MSR_DR
 286        sync
 287        mtmsrd  r0
 288        sync
 289        isync
 290        STBCIX(R3,R0,R4)
 291        isync
 292        mtmsrd  r7
 293        sync
 294        isync
 295        blr
 296
 297#endif /* CONFIG_PPC_PASEMI */
 298
 299
 300#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
 301/*
 302 * SCOM access functions for 970 (FX only for now)
 303 *
 304 * unsigned long scom970_read(unsigned int address);
 305 * void scom970_write(unsigned int address, unsigned long value);
 306 *
 307 * The address passed in is the 24 bits register address. This code
 308 * is 970 specific and will not check the status bits, so you should
 309 * know what you are doing.
 310 */
 311_GLOBAL(scom970_read)
 312        /* interrupts off */
 313        mfmsr   r4
 314        ori     r0,r4,MSR_EE
 315        xori    r0,r0,MSR_EE
 316        mtmsrd  r0,1
 317
 318        /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
 319         * (including parity). On current CPUs they must be 0'd,
 320         * and finally or in RW bit
 321         */
 322        rlwinm  r3,r3,8,0,15
 323        ori     r3,r3,0x8000
 324
 325        /* do the actual scom read */
 326        sync
 327        mtspr   SPRN_SCOMC,r3
 328        isync
 329        mfspr   r3,SPRN_SCOMD
 330        isync
 331        mfspr   r0,SPRN_SCOMC
 332        isync
 333
 334        /* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah
 335         * that's the best we can do). Not implemented yet as we don't use
 336         * the scom on any of the bogus CPUs yet, but may have to be done
 337         * ultimately
 338         */
 339
 340        /* restore interrupts */
 341        mtmsrd  r4,1
 342        blr
 343
 344
 345_GLOBAL(scom970_write)
 346        /* interrupts off */
 347        mfmsr   r5
 348        ori     r0,r5,MSR_EE
 349        xori    r0,r0,MSR_EE
 350        mtmsrd  r0,1
 351
 352        /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
 353         * (including parity). On current CPUs they must be 0'd.
 354         */
 355
 356        rlwinm  r3,r3,8,0,15
 357
 358        sync
 359        mtspr   SPRN_SCOMD,r4      /* write data */
 360        isync
 361        mtspr   SPRN_SCOMC,r3      /* write command */
 362        isync
 363        mfspr   3,SPRN_SCOMC
 364        isync
 365
 366        /* restore interrupts */
 367        mtmsrd  r5,1
 368        blr
 369#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */
 370
 371/* kexec_wait(phys_cpu)
 372 *
 373 * wait for the flag to change, indicating this kernel is going away but
 374 * the slave code for the next one is at addresses 0 to 100.
 375 *
 376 * This is used by all slaves, even those that did not find a matching
 377 * paca in the secondary startup code.
 378 *
 379 * Physical (hardware) cpu id should be in r3.
 380 */
 381_GLOBAL(kexec_wait)
 382        bl      1f
 3831:      mflr    r5
 384        addi    r5,r5,kexec_flag-1b
 385
 38699:     HMT_LOW
 387#ifdef CONFIG_KEXEC_CORE        /* use no memory without kexec */
 388        lwz     r4,0(r5)
 389        cmpwi   0,r4,0
 390        beq     99b
 391#ifdef CONFIG_PPC_BOOK3S_64
 392        li      r10,0x60
 393        mfmsr   r11
 394        clrrdi  r11,r11,1       /* Clear MSR_LE */
 395        mtsrr0  r10
 396        mtsrr1  r11
 397        rfid
 398#else
 399        /* Create TLB entry in book3e_secondary_core_init */
 400        li      r4,0
 401        ba      0x60
 402#endif
 403#endif
 404
 405/* this can be in text because we won't change it until we are
 406 * running in real anyways
 407 */
 408kexec_flag:
 409        .long   0
 410
 411
 412#ifdef CONFIG_KEXEC_CORE
 413#ifdef CONFIG_PPC_BOOK3E
 414/*
 415 * BOOK3E has no real MMU mode, so we have to setup the initial TLB
 416 * for a core to identity map v:0 to p:0.  This current implementation
 417 * assumes that 1G is enough for kexec.
 418 */
 419kexec_create_tlb:
 420        /*
 421         * Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
 422         * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
 423         */
 424        PPC_TLBILX_ALL(0,R0)
 425        sync
 426        isync
 427
 428        mfspr   r10,SPRN_TLB1CFG
 429        andi.   r10,r10,TLBnCFG_N_ENTRY /* Extract # entries */
 430        subi    r10,r10,1       /* Last entry: no conflict with kernel text */
 431        lis     r9,MAS0_TLBSEL(1)@h
 432        rlwimi  r9,r10,16,4,15          /* Setup MAS0 = TLBSEL | ESEL(r9) */
 433
 434/* Set up a temp identity mapping v:0 to p:0 and return to it. */
 435#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC)
 436#define M_IF_NEEDED     MAS2_M
 437#else
 438#define M_IF_NEEDED     0
 439#endif
 440        mtspr   SPRN_MAS0,r9
 441
 442        lis     r9,(MAS1_VALID|MAS1_IPROT)@h
 443        ori     r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
 444        mtspr   SPRN_MAS1,r9
 445
 446        LOAD_REG_IMMEDIATE(r9, 0x0 | M_IF_NEEDED)
 447        mtspr   SPRN_MAS2,r9
 448
 449        LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX)
 450        mtspr   SPRN_MAS3,r9
 451        li      r9,0
 452        mtspr   SPRN_MAS7,r9
 453
 454        tlbwe
 455        isync
 456        blr
 457#endif
 458
 459/* kexec_smp_wait(void)
 460 *
 461 * call with interrupts off
 462 * note: this is a terminal routine, it does not save lr
 463 *
 464 * get phys id from paca
 465 * switch to real mode
 466 * mark the paca as no longer used
 467 * join other cpus in kexec_wait(phys_id)
 468 */
 469_GLOBAL(kexec_smp_wait)
 470        lhz     r3,PACAHWCPUID(r13)
 471        bl      real_mode
 472
 473        li      r4,KEXEC_STATE_REAL_MODE
 474        stb     r4,PACAKEXECSTATE(r13)
 475        SYNC
 476
 477        b       kexec_wait
 478
 479/*
 480 * switch to real mode (turn mmu off)
 481 * we use the early kernel trick that the hardware ignores bits
 482 * 0 and 1 (big endian) of the effective address in real mode
 483 *
 484 * don't overwrite r3 here, it is live for kexec_wait above.
 485 */
 486real_mode:      /* assume normal blr return */
 487#ifdef CONFIG_PPC_BOOK3E
 488        /* Create an identity mapping. */
 489        b       kexec_create_tlb
 490#else
 4911:      li      r9,MSR_RI
 492        li      r10,MSR_DR|MSR_IR
 493        mflr    r11             /* return address to SRR0 */
 494        mfmsr   r12
 495        andc    r9,r12,r9
 496        andc    r10,r12,r10
 497
 498        mtmsrd  r9,1
 499        mtspr   SPRN_SRR1,r10
 500        mtspr   SPRN_SRR0,r11
 501        rfid
 502#endif
 503
 504/*
 505 * kexec_sequence(newstack, start, image, control, clear_all(),
 506                  copy_with_mmu_off)
 507 *
 508 * does the grungy work with stack switching and real mode switches
 509 * also does simple calls to other code
 510 */
 511
 512_GLOBAL(kexec_sequence)
 513        mflr    r0
 514        std     r0,16(r1)
 515
 516        /* switch stacks to newstack -- &kexec_stack.stack */
 517        stdu    r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
 518        mr      r1,r3
 519
 520        li      r0,0
 521        std     r0,16(r1)
 522
 523BEGIN_FTR_SECTION
 524        /*
 525         * This is the best time to turn AMR/IAMR off.
 526         * key 0 is used in radix for supervisor<->user
 527         * protection, but on hash key 0 is reserved
 528         * ideally we want to enter with a clean state.
 529         * NOTE, we rely on r0 being 0 from above.
 530         */
 531        mtspr   SPRN_IAMR,r0
 532BEGIN_FTR_SECTION_NESTED(42)
 533        mtspr   SPRN_AMOR,r0
 534END_FTR_SECTION_NESTED_IFSET(CPU_FTR_HVMODE, 42)
 535END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 536
 537        /* save regs for local vars on new stack.
 538         * yes, we won't go back, but ...
 539         */
 540        std     r31,-8(r1)
 541        std     r30,-16(r1)
 542        std     r29,-24(r1)
 543        std     r28,-32(r1)
 544        std     r27,-40(r1)
 545        std     r26,-48(r1)
 546        std     r25,-56(r1)
 547
 548        stdu    r1,-STACK_FRAME_OVERHEAD-64(r1)
 549
 550        /* save args into preserved regs */
 551        mr      r31,r3                  /* newstack (both) */
 552        mr      r30,r4                  /* start (real) */
 553        mr      r29,r5                  /* image (virt) */
 554        mr      r28,r6                  /* control, unused */
 555        mr      r27,r7                  /* clear_all() fn desc */
 556        mr      r26,r8                  /* copy_with_mmu_off */
 557        lhz     r25,PACAHWCPUID(r13)    /* get our phys cpu from paca */
 558
 559        /* disable interrupts, we are overwriting kernel data next */
 560#ifdef CONFIG_PPC_BOOK3E
 561        wrteei  0
 562#else
 563        mfmsr   r3
 564        rlwinm  r3,r3,0,17,15
 565        mtmsrd  r3,1
 566#endif
 567
 568        /* We need to turn the MMU off unless we are in hash mode
 569         * under a hypervisor
 570         */
 571        cmpdi   r26,0
 572        beq     1f
 573        bl      real_mode
 5741:
 575        /* copy dest pages, flush whole dest image */
 576        mr      r3,r29
 577        bl      kexec_copy_flush        /* (image) */
 578
 579        /* turn off mmu now if not done earlier */
 580        cmpdi   r26,0
 581        bne     1f
 582        bl      real_mode
 583
 584        /* copy  0x100 bytes starting at start to 0 */
 5851:      li      r3,0
 586        mr      r4,r30          /* start, aka phys mem offset */
 587        li      r5,0x100
 588        li      r6,0
 589        bl      copy_and_flush  /* (dest, src, copy limit, start offset) */
 5901:      /* assume normal blr return */
 591
 592        /* release other cpus to the new kernel secondary start at 0x60 */
 593        mflr    r5
 594        li      r6,1
 595        stw     r6,kexec_flag-1b(5)
 596
 597        cmpdi   r27,0
 598        beq     1f
 599
 600        /* clear out hardware hash page table and tlb */
 601#ifdef PPC64_ELF_ABI_v1
 602        ld      r12,0(r27)              /* deref function descriptor */
 603#else
 604        mr      r12,r27
 605#endif
 606        mtctr   r12
 607        bctrl                           /* mmu_hash_ops.hpte_clear_all(void); */
 608
 609/*
 610 *   kexec image calling is:
 611 *      the first 0x100 bytes of the entry point are copied to 0
 612 *
 613 *      all slaves branch to slave = 0x60 (absolute)
 614 *              slave(phys_cpu_id);
 615 *
 616 *      master goes to start = entry point
 617 *              start(phys_cpu_id, start, 0);
 618 *
 619 *
 620 *   a wrapper is needed to call existing kernels, here is an approximate
 621 *   description of one method:
 622 *
 623 * v2: (2.6.10)
 624 *   start will be near the boot_block (maybe 0x100 bytes before it?)
 625 *   it will have a 0x60, which will b to boot_block, where it will wait
 626 *   and 0 will store phys into struct boot-block and load r3 from there,
 627 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
 628 *
 629 * v1: (2.6.9)
 630 *    boot block will have all cpus scanning device tree to see if they
 631 *    are the boot cpu ?????
 632 *    other device tree differences (prop sizes, va vs pa, etc)...
 633 */
 6341:      mr      r3,r25  # my phys cpu
 635        mr      r4,r30  # start, aka phys mem offset
 636        mtlr    4
 637        li      r5,0
 638        blr     /* image->start(physid, image->start, 0); */
 639#endif /* CONFIG_KEXEC_CORE */
 640