LXR linux/arch/powerpc/mm/hash_low

   1/*
   2 * ppc64 MMU hashtable management routines
   3 *
   4 * (c) Copyright IBM Corp. 2003, 2005
   5 *
   6 * Maintained by: Benjamin Herrenschmidt
   7 *                <benh@kernel.crashing.org>
   8 *
   9 * This file is covered by the GNU Public Licence v2 as
  10 * described in the kernel's COPYING file.
  11 */
  12
  13#include <asm/reg.h>
  14#include <asm/pgtable.h>
  15#include <asm/mmu.h>
  16#include <asm/page.h>
  17#include <asm/types.h>
  18#include <asm/ppc_asm.h>
  19#include <asm/asm-offsets.h>
  20#include <asm/cputable.h>
  21
  22        .text
  23
  24/*
  25 * Stackframe:
  26 *              
  27 *         +-> Back chain                       (SP + 256)
  28 *         |   General register save area       (SP + 112)
  29 *         |   Parameter save area              (SP + 48)
  30 *         |   TOC save area                    (SP + 40)
  31 *         |   link editor doubleword           (SP + 32)
  32 *         |   compiler doubleword              (SP + 24)
  33 *         |   LR save area                     (SP + 16)
  34 *         |   CR save area                     (SP + 8)
  35 * SP ---> +-- Back chain                       (SP + 0)
  36 */
  37
  38#ifndef CONFIG_PPC_64K_PAGES
  39
  40/*****************************************************************************
  41 *                                                                           *
  42 *           4K SW & 4K HW pages implementation                              *
  43 *                                                                           *
  44 *****************************************************************************/
  45
  46
  47/*
  48 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  49 *               pte_t *ptep, unsigned long trap, unsigned long flags,
  50 *               int ssize)
  51 *
  52 * Adds a 4K page to the hash table in a segment of 4K pages only
  53 */
  54
  55_GLOBAL(__hash_page_4K)
  56        mflr    r0
  57        std     r0,16(r1)
  58        stdu    r1,-STACKFRAMESIZE(r1)
  59        /* Save all params that we need after a function call */
  60        std     r6,STK_PARAM(R6)(r1)
  61        std     r8,STK_PARAM(R8)(r1)
  62        std     r9,STK_PARAM(R9)(r1)
  63        
  64        /* Save non-volatile registers.
  65         * r31 will hold "old PTE"
  66         * r30 is "new PTE"
  67         * r29 is vpn
  68         * r28 is a hash value
  69         * r27 is hashtab mask (maybe dynamic patched instead ?)
  70         */
  71        std     r27,STK_REG(R27)(r1)
  72        std     r28,STK_REG(R28)(r1)
  73        std     r29,STK_REG(R29)(r1)
  74        std     r30,STK_REG(R30)(r1)
  75        std     r31,STK_REG(R31)(r1)
  76        
  77        /* Step 1:
  78         *
  79         * Check permissions, atomically mark the linux PTE busy
  80         * and hashed.
  81         */ 
  821:
  83        ldarx   r31,0,r6
  84        /* Check access rights (access & ~(pte_val(*ptep))) */
  85        andc.   r0,r4,r31
  86        bne-    htab_wrong_access
  87        /* Check if PTE is busy */
  88        andi.   r0,r31,_PAGE_BUSY
  89        /* If so, just bail out and refault if needed. Someone else
  90         * is changing this PTE anyway and might hash it.
  91         */
  92        bne-    htab_bail_ok
  93
  94        /* Prepare new PTE value (turn access RW into DIRTY, then
  95         * add BUSY,HASHPTE and ACCESSED)
  96         */
  97        rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  98        or      r30,r30,r31
  99        ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 100        /* Write the linux PTE atomically (setting busy) */
 101        stdcx.  r30,0,r6
 102        bne-    1b
 103        isync
 104
 105        /* Step 2:
 106         *
 107         * Insert/Update the HPTE in the hash table. At this point,
 108         * r4 (access) is re-useable, we use it for the new HPTE flags
 109         */
 110
 111BEGIN_FTR_SECTION
 112        cmpdi   r9,0                    /* check segment size */
 113        bne     3f
 114END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 115        /* Calc vpn and put it in r29 */
 116        sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 117        rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 118        or      r29,r28,r29
 119        /*
 120         * Calculate hash value for primary slot and store it in r28
 121         * r3 = va, r5 = vsid
 122         * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
 123         */
 124        rldicl  r0,r3,64-12,48
 125        xor     r28,r5,r0               /* hash */
 126        b       4f
 127
 1283:      /* Calc vpn and put it in r29 */
 129        sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 130        rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 131        or      r29,r28,r29
 132
 133        /*
 134         * calculate hash value for primary slot and
 135         * store it in r28 for 1T segment
 136         * r3 = va, r5 = vsid
 137         */
 138        sldi    r28,r5,25               /* vsid << 25 */
 139        /* r0 =  (va >> 12) & ((1ul << (40 - 12)) -1) */
 140        rldicl  r0,r3,64-12,36
 141        xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 142        xor     r28,r28,r0              /* hash */
 143
 144        /* Convert linux PTE bits into HW equivalents */
 1454:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 146        xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 147        rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 148        rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 149        and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 150        andc    r0,r30,r0               /* r0 = pte & ~r0 */
 151        rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 152        /*
 153         * Always add "C" bit for perf. Memory coherence is always enabled
 154         */
 155        ori     r3,r3,HPTE_R_C | HPTE_R_M
 156
 157        /* We eventually do the icache sync here (maybe inline that
 158         * code rather than call a C function...) 
 159         */
 160BEGIN_FTR_SECTION
 161        mr      r4,r30
 162        mr      r5,r7
 163        bl      hash_page_do_lazy_icache
 164END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 165
 166        /* At this point, r3 contains new PP bits, save them in
 167         * place of "access" in the param area (sic)
 168         */
 169        std     r3,STK_PARAM(R4)(r1)
 170
 171        /* Get htab_hash_mask */
 172        ld      r4,htab_hash_mask@got(2)
 173        ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 174
 175        /* Check if we may already be in the hashtable, in this case, we
 176         * go to out-of-line code to try to modify the HPTE
 177         */
 178        andi.   r0,r31,_PAGE_HASHPTE
 179        bne     htab_modify_pte
 180
 181htab_insert_pte:
 182        /* Clear hpte bits in new pte (we also clear BUSY btw) and
 183         * add _PAGE_HASHPTE
 184         */
 185        lis     r0,_PAGE_HPTEFLAGS@h
 186        ori     r0,r0,_PAGE_HPTEFLAGS@l
 187        andc    r30,r30,r0
 188        ori     r30,r30,_PAGE_HASHPTE
 189
 190        /* physical address r5 */
 191        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 192        sldi    r5,r5,PAGE_SHIFT
 193
 194        /* Calculate primary group hash */
 195        and     r0,r28,r27
 196        rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 197
 198        /* Call ppc_md.hpte_insert */
 199        ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 200        mr      r4,r29                  /* Retrieve vpn */
 201        li      r7,0                    /* !bolted, !secondary */
 202        li      r8,MMU_PAGE_4K          /* page size */
 203        li      r9,MMU_PAGE_4K          /* actual page size */
 204        ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 205.globl htab_call_hpte_insert1
 206htab_call_hpte_insert1:
 207        bl      .                       /* Patched by htab_finish_init() */
 208        cmpdi   0,r3,0
 209        bge     htab_pte_insert_ok      /* Insertion successful */
 210        cmpdi   0,r3,-2                 /* Critical failure */
 211        beq-    htab_pte_insert_failure
 212
 213        /* Now try secondary slot */
 214        
 215        /* physical address r5 */
 216        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 217        sldi    r5,r5,PAGE_SHIFT
 218
 219        /* Calculate secondary group hash */
 220        andc    r0,r27,r28
 221        rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 222        
 223        /* Call ppc_md.hpte_insert */
 224        ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 225        mr      r4,r29                  /* Retrieve vpn */
 226        li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 227        li      r8,MMU_PAGE_4K          /* page size */
 228        li      r9,MMU_PAGE_4K          /* actual page size */
 229        ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 230.globl htab_call_hpte_insert2
 231htab_call_hpte_insert2:
 232        bl      .                       /* Patched by htab_finish_init() */
 233        cmpdi   0,r3,0
 234        bge+    htab_pte_insert_ok      /* Insertion successful */
 235        cmpdi   0,r3,-2                 /* Critical failure */
 236        beq-    htab_pte_insert_failure
 237
 238        /* Both are full, we need to evict something */
 239        mftb    r0
 240        /* Pick a random group based on TB */
 241        andi.   r0,r0,1
 242        mr      r5,r28
 243        bne     2f
 244        not     r5,r5
 2452:      and     r0,r5,r27
 246        rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */   
 247        /* Call ppc_md.hpte_remove */
 248.globl htab_call_hpte_remove
 249htab_call_hpte_remove:
 250        bl      .                       /* Patched by htab_finish_init() */
 251
 252        /* Try all again */
 253        b       htab_insert_pte 
 254
 255htab_bail_ok:
 256        li      r3,0
 257        b       htab_bail
 258
 259htab_pte_insert_ok:
 260        /* Insert slot number & secondary bit in PTE */
 261        rldimi  r30,r3,12,63-15
 262                
 263        /* Write out the PTE with a normal write
 264         * (maybe add eieio may be good still ?)
 265         */
 266htab_write_out_pte:
 267        ld      r6,STK_PARAM(R6)(r1)
 268        std     r30,0(r6)
 269        li      r3, 0
 270htab_bail:
 271        ld      r27,STK_REG(R27)(r1)
 272        ld      r28,STK_REG(R28)(r1)
 273        ld      r29,STK_REG(R29)(r1)
 274        ld      r30,STK_REG(R30)(r1)
 275        ld      r31,STK_REG(R31)(r1)
 276        addi    r1,r1,STACKFRAMESIZE
 277        ld      r0,16(r1)
 278        mtlr    r0
 279        blr
 280
 281htab_modify_pte:
 282        /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 283        mr      r4,r3
 284        rlwinm  r3,r31,32-12,29,31
 285
 286        /* Secondary group ? if yes, get a inverted hash value */
 287        mr      r5,r28
 288        andi.   r0,r31,_PAGE_SECONDARY
 289        beq     1f
 290        not     r5,r5
 2911:
 292        /* Calculate proper slot value for ppc_md.hpte_updatepp */
 293        and     r0,r5,r27
 294        rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 295        add     r3,r0,r3        /* add slot idx */
 296
 297        /* Call ppc_md.hpte_updatepp */
 298        mr      r5,r29                  /* vpn */
 299        li      r6,MMU_PAGE_4K          /* base page size */
 300        li      r7,MMU_PAGE_4K          /* actual page size */
 301        ld      r8,STK_PARAM(R9)(r1)    /* segment size */
 302        ld      r9,STK_PARAM(R8)(r1)    /* get "flags" param */
 303.globl htab_call_hpte_updatepp
 304htab_call_hpte_updatepp:
 305        bl      .                       /* Patched by htab_finish_init() */
 306
 307        /* if we failed because typically the HPTE wasn't really here
 308         * we try an insertion. 
 309         */
 310        cmpdi   0,r3,-1
 311        beq-    htab_insert_pte
 312
 313        /* Clear the BUSY bit and Write out the PTE */
 314        li      r0,_PAGE_BUSY
 315        andc    r30,r30,r0
 316        b       htab_write_out_pte
 317
 318htab_wrong_access:
 319        /* Bail out clearing reservation */
 320        stdcx.  r31,0,r6
 321        li      r3,1
 322        b       htab_bail
 323
 324htab_pte_insert_failure:
 325        /* Bail out restoring old PTE */
 326        ld      r6,STK_PARAM(R6)(r1)
 327        std     r31,0(r6)
 328        li      r3,-1
 329        b       htab_bail
 330
 331
 332#else /* CONFIG_PPC_64K_PAGES */
 333
 334
 335/*****************************************************************************
 336 *                                                                           *
 337 *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 338 *                                                                           *
 339 *****************************************************************************/
 340
 341/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 342 *               pte_t *ptep, unsigned long trap, unsigned local flags,
 343 *               int ssize, int subpg_prot)
 344 */
 345
 346/*
 347 * For now, we do NOT implement Admixed pages
 348 */
 349_GLOBAL(__hash_page_4K)
 350        mflr    r0
 351        std     r0,16(r1)
 352        stdu    r1,-STACKFRAMESIZE(r1)
 353        /* Save all params that we need after a function call */
 354        std     r6,STK_PARAM(R6)(r1)
 355        std     r8,STK_PARAM(R8)(r1)
 356        std     r9,STK_PARAM(R9)(r1)
 357
 358        /* Save non-volatile registers.
 359         * r31 will hold "old PTE"
 360         * r30 is "new PTE"
 361         * r29 is vpn
 362         * r28 is a hash value
 363         * r27 is hashtab mask (maybe dynamic patched instead ?)
 364         * r26 is the hidx mask
 365         * r25 is the index in combo page
 366         */
 367        std     r25,STK_REG(R25)(r1)
 368        std     r26,STK_REG(R26)(r1)
 369        std     r27,STK_REG(R27)(r1)
 370        std     r28,STK_REG(R28)(r1)
 371        std     r29,STK_REG(R29)(r1)
 372        std     r30,STK_REG(R30)(r1)
 373        std     r31,STK_REG(R31)(r1)
 374
 375        /* Step 1:
 376         *
 377         * Check permissions, atomically mark the linux PTE busy
 378         * and hashed.
 379         */
 3801:
 381        ldarx   r31,0,r6
 382        /* Check access rights (access & ~(pte_val(*ptep))) */
 383        andc.   r0,r4,r31
 384        bne-    htab_wrong_access
 385        /* Check if PTE is busy */
 386        andi.   r0,r31,_PAGE_BUSY
 387        /* If so, just bail out and refault if needed. Someone else
 388         * is changing this PTE anyway and might hash it.
 389         */
 390        bne-    htab_bail_ok
 391        /* Prepare new PTE value (turn access RW into DIRTY, then
 392         * add BUSY and ACCESSED)
 393         */
 394        rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 395        or      r30,r30,r31
 396        ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 397        oris    r30,r30,_PAGE_COMBO@h
 398        /* Write the linux PTE atomically (setting busy) */
 399        stdcx.  r30,0,r6
 400        bne-    1b
 401        isync
 402
 403        /* Step 2:
 404         *
 405         * Insert/Update the HPTE in the hash table. At this point,
 406         * r4 (access) is re-useable, we use it for the new HPTE flags
 407         */
 408
 409        /* Load the hidx index */
 410        rldicl  r25,r3,64-12,60
 411
 412BEGIN_FTR_SECTION
 413        cmpdi   r9,0                    /* check segment size */
 414        bne     3f
 415END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 416        /* Calc vpn and put it in r29 */
 417        sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 418        /*
 419         * clrldi r3,r3,64 - SID_SHIFT -->  ea & 0xfffffff
 420         * srdi  r28,r3,VPN_SHIFT
 421         */
 422        rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 423        or      r29,r28,r29
 424        /*
 425         * Calculate hash value for primary slot and store it in r28
 426         * r3 = va, r5 = vsid
 427         * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
 428         */
 429        rldicl  r0,r3,64-12,48
 430        xor     r28,r5,r0               /* hash */
 431        b       4f
 432
 4333:      /* Calc vpn and put it in r29 */
 434        sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 435        /*
 436         * clrldi r3,r3,64 - SID_SHIFT_1T -->  ea & 0xffffffffff
 437         * srdi r28,r3,VPN_SHIFT
 438         */
 439        rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 440        or      r29,r28,r29
 441
 442        /*
 443         * Calculate hash value for primary slot and
 444         * store it in r28  for 1T segment
 445         * r3 = va, r5 = vsid
 446         */
 447        sldi    r28,r5,25               /* vsid << 25 */
 448        /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */
 449        rldicl  r0,r3,64-12,36
 450        xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 451        xor     r28,r28,r0              /* hash */
 452
 453        /* Convert linux PTE bits into HW equivalents */
 4544:
 455#ifdef CONFIG_PPC_SUBPAGE_PROT
 456        andc    r10,r30,r10
 457        andi.   r3,r10,0x1fe            /* Get basic set of flags */
 458        rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 459#else
 460        andi.   r3,r30,0x1fe            /* Get basic set of flags */
 461        rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 462#endif
 463        xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 464        rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 465        and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 466        andc    r0,r3,r0                /* r0 = pte & ~r0 */
 467        rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 468        /*
 469         * Always add "C" bit for perf. Memory coherence is always enabled
 470         */
 471        ori     r3,r3,HPTE_R_C | HPTE_R_M
 472
 473        /* We eventually do the icache sync here (maybe inline that
 474         * code rather than call a C function...)
 475         */
 476BEGIN_FTR_SECTION
 477        mr      r4,r30
 478        mr      r5,r7
 479        bl      hash_page_do_lazy_icache
 480END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 481
 482        /* At this point, r3 contains new PP bits, save them in
 483         * place of "access" in the param area (sic)
 484         */
 485        std     r3,STK_PARAM(R4)(r1)
 486
 487        /* Get htab_hash_mask */
 488        ld      r4,htab_hash_mask@got(2)
 489        ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 490
 491        /* Check if we may already be in the hashtable, in this case, we
 492         * go to out-of-line code to try to modify the HPTE. We look for
 493         * the bit at (1 >> (index + 32))
 494         */
 495        rldicl. r0,r31,64-12,48
 496        li      r26,0                   /* Default hidx */
 497        beq     htab_insert_pte
 498
 499        /*
 500         * Check if the pte was already inserted into the hash table
 501         * as a 64k HW page, and invalidate the 64k HPTE if so.
 502         */
 503        andis.  r0,r31,_PAGE_COMBO@h
 504        beq     htab_inval_old_hpte
 505
 506        ld      r6,STK_PARAM(R6)(r1)
 507        ori     r26,r6,PTE_PAGE_HIDX_OFFSET /* Load the hidx mask. */
 508        ld      r26,0(r26)
 509        addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 510        rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 511        bne     htab_modify_pte
 512
 513htab_insert_pte:
 514        /* real page number in r5, PTE RPN value + index */
 515        andis.  r0,r31,_PAGE_4K_PFN@h
 516        srdi    r5,r31,PTE_RPN_SHIFT
 517        bne-    htab_special_pfn
 518        sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 519        add     r5,r5,r25
 520htab_special_pfn:
 521        sldi    r5,r5,HW_PAGE_SHIFT
 522
 523        /* Calculate primary group hash */
 524        and     r0,r28,r27
 525        rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 526
 527        /* Call ppc_md.hpte_insert */
 528        ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 529        mr      r4,r29                  /* Retrieve vpn */
 530        li      r7,0                    /* !bolted, !secondary */
 531        li      r8,MMU_PAGE_4K          /* page size */
 532        li      r9,MMU_PAGE_4K          /* actual page size */
 533        ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 534.globl htab_call_hpte_insert1
 535htab_call_hpte_insert1:
 536        bl      .                       /* patched by htab_finish_init() */
 537        cmpdi   0,r3,0
 538        bge     htab_pte_insert_ok      /* Insertion successful */
 539        cmpdi   0,r3,-2                 /* Critical failure */
 540        beq-    htab_pte_insert_failure
 541
 542        /* Now try secondary slot */
 543
 544        /* real page number in r5, PTE RPN value + index */
 545        andis.  r0,r31,_PAGE_4K_PFN@h
 546        srdi    r5,r31,PTE_RPN_SHIFT
 547        bne-    3f
 548        sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 549        add     r5,r5,r25
 5503:      sldi    r5,r5,HW_PAGE_SHIFT
 551
 552        /* Calculate secondary group hash */
 553        andc    r0,r27,r28
 554        rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 555
 556        /* Call ppc_md.hpte_insert */
 557        ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 558        mr      r4,r29                  /* Retrieve vpn */
 559        li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 560        li      r8,MMU_PAGE_4K          /* page size */
 561        li      r9,MMU_PAGE_4K          /* actual page size */
 562        ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 563.globl htab_call_hpte_insert2
 564htab_call_hpte_insert2:
 565        bl      .                       /* patched by htab_finish_init() */
 566        cmpdi   0,r3,0
 567        bge+    htab_pte_insert_ok      /* Insertion successful */
 568        cmpdi   0,r3,-2                 /* Critical failure */
 569        beq-    htab_pte_insert_failure
 570
 571        /* Both are full, we need to evict something */
 572        mftb    r0
 573        /* Pick a random group based on TB */
 574        andi.   r0,r0,1
 575        mr      r5,r28
 576        bne     2f
 577        not     r5,r5
 5782:      and     r0,r5,r27
 579        rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 580        /* Call ppc_md.hpte_remove */
 581.globl htab_call_hpte_remove
 582htab_call_hpte_remove:
 583        bl      .                       /* patched by htab_finish_init() */
 584
 585        /* Try all again */
 586        b       htab_insert_pte
 587
 588        /*
 589         * Call out to C code to invalidate an 64k HW HPTE that is
 590         * useless now that the segment has been switched to 4k pages.
 591         */
 592htab_inval_old_hpte:
 593        mr      r3,r29                  /* vpn */
 594        mr      r4,r31                  /* PTE.pte */
 595        li      r5,0                    /* PTE.hidx */
 596        li      r6,MMU_PAGE_64K         /* psize */
 597        ld      r7,STK_PARAM(R9)(r1)    /* ssize */
 598        ld      r8,STK_PARAM(R8)(r1)    /* flags */
 599        bl      flush_hash_page
 600        /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 601        lis     r0,_PAGE_HPTE_SUB@h
 602        ori     r0,r0,_PAGE_HPTE_SUB@l
 603        andc    r30,r30,r0
 604        b       htab_insert_pte
 605        
 606htab_bail_ok:
 607        li      r3,0
 608        b       htab_bail
 609
 610htab_pte_insert_ok:
 611        /* Insert slot number & secondary bit in PTE second half,
 612         * clear _PAGE_BUSY and set approriate HPTE slot bit
 613         */
 614        ld      r6,STK_PARAM(R6)(r1)
 615        li      r0,_PAGE_BUSY
 616        andc    r30,r30,r0
 617        /* HPTE SUB bit */
 618        li      r0,1
 619        subfic  r5,r25,27               /* Must match bit position in */
 620        sld     r0,r0,r5                /* pgtable.h */
 621        or      r30,r30,r0
 622        /* hindx */
 623        sldi    r5,r25,2
 624        sld     r3,r3,r5
 625        li      r4,0xf
 626        sld     r4,r4,r5
 627        andc    r26,r26,r4
 628        or      r26,r26,r3
 629        ori     r5,r6,PTE_PAGE_HIDX_OFFSET
 630        std     r26,0(r5)
 631        lwsync
 632        std     r30,0(r6)
 633        li      r3, 0
 634htab_bail:
 635        ld      r25,STK_REG(R25)(r1)
 636        ld      r26,STK_REG(R26)(r1)
 637        ld      r27,STK_REG(R27)(r1)
 638        ld      r28,STK_REG(R28)(r1)
 639        ld      r29,STK_REG(R29)(r1)
 640        ld      r30,STK_REG(R30)(r1)
 641        ld      r31,STK_REG(R31)(r1)
 642        addi    r1,r1,STACKFRAMESIZE
 643        ld      r0,16(r1)
 644        mtlr    r0
 645        blr
 646
 647htab_modify_pte:
 648        /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 649        mr      r4,r3
 650        sldi    r5,r25,2
 651        srd     r3,r26,r5
 652
 653        /* Secondary group ? if yes, get a inverted hash value */
 654        mr      r5,r28
 655        andi.   r0,r3,0x8 /* page secondary ? */
 656        beq     1f
 657        not     r5,r5
 6581:      andi.   r3,r3,0x7 /* extract idx alone */
 659
 660        /* Calculate proper slot value for ppc_md.hpte_updatepp */
 661        and     r0,r5,r27
 662        rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 663        add     r3,r0,r3        /* add slot idx */
 664
 665        /* Call ppc_md.hpte_updatepp */
 666        mr      r5,r29                  /* vpn */
 667        li      r6,MMU_PAGE_4K          /* base page size */
 668        li      r7,MMU_PAGE_4K          /* actual page size */
 669        ld      r8,STK_PARAM(R9)(r1)    /* segment size */
 670        ld      r9,STK_PARAM(R8)(r1)    /* get "flags" param */
 671.globl htab_call_hpte_updatepp
 672htab_call_hpte_updatepp:
 673        bl      .                       /* patched by htab_finish_init() */
 674
 675        /* if we failed because typically the HPTE wasn't really here
 676         * we try an insertion.
 677         */
 678        cmpdi   0,r3,-1
 679        beq-    htab_insert_pte
 680
 681        /* Clear the BUSY bit and Write out the PTE */
 682        li      r0,_PAGE_BUSY
 683        andc    r30,r30,r0
 684        ld      r6,STK_PARAM(R6)(r1)
 685        std     r30,0(r6)
 686        li      r3,0
 687        b       htab_bail
 688
 689htab_wrong_access:
 690        /* Bail out clearing reservation */
 691        stdcx.  r31,0,r6
 692        li      r3,1
 693        b       htab_bail
 694
 695htab_pte_insert_failure:
 696        /* Bail out restoring old PTE */
 697        ld      r6,STK_PARAM(R6)(r1)
 698        std     r31,0(r6)
 699        li      r3,-1
 700        b       htab_bail
 701
 702#endif /* CONFIG_PPC_64K_PAGES */
 703
 704#ifdef CONFIG_PPC_HAS_HASH_64K
 705
 706/*****************************************************************************
 707 *                                                                           *
 708 *           64K SW & 64K HW in a 64K segment pages implementation           *
 709 *                                                                           *
 710 *****************************************************************************/
 711
 712_GLOBAL(__hash_page_64K)
 713        mflr    r0
 714        std     r0,16(r1)
 715        stdu    r1,-STACKFRAMESIZE(r1)
 716        /* Save all params that we need after a function call */
 717        std     r6,STK_PARAM(R6)(r1)
 718        std     r8,STK_PARAM(R8)(r1)
 719        std     r9,STK_PARAM(R9)(r1)
 720
 721        /* Save non-volatile registers.
 722         * r31 will hold "old PTE"
 723         * r30 is "new PTE"
 724         * r29 is vpn
 725         * r28 is a hash value
 726         * r27 is hashtab mask (maybe dynamic patched instead ?)
 727         */
 728        std     r27,STK_REG(R27)(r1)
 729        std     r28,STK_REG(R28)(r1)
 730        std     r29,STK_REG(R29)(r1)
 731        std     r30,STK_REG(R30)(r1)
 732        std     r31,STK_REG(R31)(r1)
 733
 734        /* Step 1:
 735         *
 736         * Check permissions, atomically mark the linux PTE busy
 737         * and hashed.
 738         */
 7391:
 740        ldarx   r31,0,r6
 741        /* Check access rights (access & ~(pte_val(*ptep))) */
 742        andc.   r0,r4,r31
 743        bne-    ht64_wrong_access
 744        /* Check if PTE is busy */
 745        andi.   r0,r31,_PAGE_BUSY
 746        /* If so, just bail out and refault if needed. Someone else
 747         * is changing this PTE anyway and might hash it.
 748         */
 749        bne-    ht64_bail_ok
 750BEGIN_FTR_SECTION
 751        /* Check if PTE has the cache-inhibit bit set */
 752        andi.   r0,r31,_PAGE_NO_CACHE
 753        /* If so, bail out and refault as a 4k page */
 754        bne-    ht64_bail_ok
 755END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
 756        /* Prepare new PTE value (turn access RW into DIRTY, then
 757         * add BUSY and ACCESSED)
 758         */
 759        rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 760        or      r30,r30,r31
 761        ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 762        /* Write the linux PTE atomically (setting busy) */
 763        stdcx.  r30,0,r6
 764        bne-    1b
 765        isync
 766
 767        /* Step 2:
 768         *
 769         * Insert/Update the HPTE in the hash table. At this point,
 770         * r4 (access) is re-useable, we use it for the new HPTE flags
 771         */
 772
 773BEGIN_FTR_SECTION
 774        cmpdi   r9,0                    /* check segment size */
 775        bne     3f
 776END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 777        /* Calc vpn and put it in r29 */
 778        sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 779        rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 780        or      r29,r28,r29
 781
 782        /* Calculate hash value for primary slot and store it in r28
 783         * r3 = va, r5 = vsid
 784         * r0 = (va >> 16) & ((1ul << (28 - 16)) -1)
 785         */
 786        rldicl  r0,r3,64-16,52
 787        xor     r28,r5,r0               /* hash */
 788        b       4f
 789
 7903:      /* Calc vpn and put it in r29 */
 791        sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 792        rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 793        or      r29,r28,r29
 794        /*
 795         * calculate hash value for primary slot and
 796         * store it in r28 for 1T segment
 797         * r3 = va, r5 = vsid
 798         */
 799        sldi    r28,r5,25               /* vsid << 25 */
 800        /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */
 801        rldicl  r0,r3,64-16,40
 802        xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 803        xor     r28,r28,r0              /* hash */
 804
 805        /* Convert linux PTE bits into HW equivalents */
 8064:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 807        xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 808        rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 809        rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 810        and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 811        andc    r0,r30,r0               /* r0 = pte & ~r0 */
 812        rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 813        /*
 814         * Always add "C" bit for perf. Memory coherence is always enabled
 815         */
 816        ori     r3,r3,HPTE_R_C | HPTE_R_M
 817
 818        /* We eventually do the icache sync here (maybe inline that
 819         * code rather than call a C function...)
 820         */
 821BEGIN_FTR_SECTION
 822        mr      r4,r30
 823        mr      r5,r7
 824        bl      hash_page_do_lazy_icache
 825END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 826
 827        /* At this point, r3 contains new PP bits, save them in
 828         * place of "access" in the param area (sic)
 829         */
 830        std     r3,STK_PARAM(R4)(r1)
 831
 832        /* Get htab_hash_mask */
 833        ld      r4,htab_hash_mask@got(2)
 834        ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 835
 836        /* Check if we may already be in the hashtable, in this case, we
 837         * go to out-of-line code to try to modify the HPTE
 838         */
 839        rldicl. r0,r31,64-12,48
 840        bne     ht64_modify_pte
 841
 842ht64_insert_pte:
 843        /* Clear hpte bits in new pte (we also clear BUSY btw) and
 844         * add _PAGE_HPTE_SUB0
 845         */
 846        lis     r0,_PAGE_HPTEFLAGS@h
 847        ori     r0,r0,_PAGE_HPTEFLAGS@l
 848        andc    r30,r30,r0
 849#ifdef CONFIG_PPC_64K_PAGES
 850        oris    r30,r30,_PAGE_HPTE_SUB0@h
 851#else
 852        ori     r30,r30,_PAGE_HASHPTE
 853#endif
 854        /* Phyical address in r5 */
 855        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 856        sldi    r5,r5,PAGE_SHIFT
 857
 858        /* Calculate primary group hash */
 859        and     r0,r28,r27
 860        rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 861
 862        /* Call ppc_md.hpte_insert */
 863        ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 864        mr      r4,r29                  /* Retrieve vpn */
 865        li      r7,0                    /* !bolted, !secondary */
 866        li      r8,MMU_PAGE_64K
 867        li      r9,MMU_PAGE_64K         /* actual page size */
 868        ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 869.globl ht64_call_hpte_insert1
 870ht64_call_hpte_insert1:
 871        bl      .                       /* patched by htab_finish_init() */
 872        cmpdi   0,r3,0
 873        bge     ht64_pte_insert_ok      /* Insertion successful */
 874        cmpdi   0,r3,-2                 /* Critical failure */
 875        beq-    ht64_pte_insert_failure
 876
 877        /* Now try secondary slot */
 878
 879        /* Phyical address in r5 */
 880        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 881        sldi    r5,r5,PAGE_SHIFT
 882
 883        /* Calculate secondary group hash */
 884        andc    r0,r27,r28
 885        rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 886
 887        /* Call ppc_md.hpte_insert */
 888        ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 889        mr      r4,r29                  /* Retrieve vpn */
 890        li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 891        li      r8,MMU_PAGE_64K
 892        li      r9,MMU_PAGE_64K         /* actual page size */
 893        ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 894.globl ht64_call_hpte_insert2
 895ht64_call_hpte_insert2:
 896        bl      .                       /* patched by htab_finish_init() */
 897        cmpdi   0,r3,0
 898        bge+    ht64_pte_insert_ok      /* Insertion successful */
 899        cmpdi   0,r3,-2                 /* Critical failure */
 900        beq-    ht64_pte_insert_failure
 901
 902        /* Both are full, we need to evict something */
 903        mftb    r0
 904        /* Pick a random group based on TB */
 905        andi.   r0,r0,1
 906        mr      r5,r28
 907        bne     2f
 908        not     r5,r5
 9092:      and     r0,r5,r27
 910        rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 911        /* Call ppc_md.hpte_remove */
 912.globl ht64_call_hpte_remove
 913ht64_call_hpte_remove:
 914        bl      .                       /* patched by htab_finish_init() */
 915
 916        /* Try all again */
 917        b       ht64_insert_pte
 918
 919ht64_bail_ok:
 920        li      r3,0
 921        b       ht64_bail
 922
 923ht64_pte_insert_ok:
 924        /* Insert slot number & secondary bit in PTE */
 925        rldimi  r30,r3,12,63-15
 926
 927        /* Write out the PTE with a normal write
 928         * (maybe add eieio may be good still ?)
 929         */
 930ht64_write_out_pte:
 931        ld      r6,STK_PARAM(R6)(r1)
 932        std     r30,0(r6)
 933        li      r3, 0
 934ht64_bail:
 935        ld      r27,STK_REG(R27)(r1)
 936        ld      r28,STK_REG(R28)(r1)
 937        ld      r29,STK_REG(R29)(r1)
 938        ld      r30,STK_REG(R30)(r1)
 939        ld      r31,STK_REG(R31)(r1)
 940        addi    r1,r1,STACKFRAMESIZE
 941        ld      r0,16(r1)
 942        mtlr    r0
 943        blr
 944
 945ht64_modify_pte:
 946        /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 947        mr      r4,r3
 948        rlwinm  r3,r31,32-12,29,31
 949
 950        /* Secondary group ? if yes, get a inverted hash value */
 951        mr      r5,r28
 952        andi.   r0,r31,_PAGE_F_SECOND
 953        beq     1f
 954        not     r5,r5
 9551:
 956        /* Calculate proper slot value for ppc_md.hpte_updatepp */
 957        and     r0,r5,r27
 958        rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 959        add     r3,r0,r3        /* add slot idx */
 960
 961        /* Call ppc_md.hpte_updatepp */
 962        mr      r5,r29                  /* vpn */
 963        li      r6,MMU_PAGE_64K         /* base page size */
 964        li      r7,MMU_PAGE_64K         /* actual page size */
 965        ld      r8,STK_PARAM(R9)(r1)    /* segment size */
 966        ld      r9,STK_PARAM(R8)(r1)    /* get "flags" param */
 967.globl ht64_call_hpte_updatepp
 968ht64_call_hpte_updatepp:
 969        bl      .                       /* patched by htab_finish_init() */
 970
 971        /* if we failed because typically the HPTE wasn't really here
 972         * we try an insertion.
 973         */
 974        cmpdi   0,r3,-1
 975        beq-    ht64_insert_pte
 976
 977        /* Clear the BUSY bit and Write out the PTE */
 978        li      r0,_PAGE_BUSY
 979        andc    r30,r30,r0
 980        b       ht64_write_out_pte
 981
 982ht64_wrong_access:
 983        /* Bail out clearing reservation */
 984        stdcx.  r31,0,r6
 985        li      r3,1
 986        b       ht64_bail
 987
 988ht64_pte_insert_failure:
 989        /* Bail out restoring old PTE */
 990        ld      r6,STK_PARAM(R6)(r1)
 991        std     r31,0(r6)
 992        li      r3,-1
 993        b       ht64_bail
 994
 995
 996#endif /* CONFIG_PPC_HAS_HASH_64K */
 997
 998
 999/*****************************************************************************
1000 *                                                                           *

1001 *           Huge pages implementation is in hugetlbpage.c                   *
1002 *                                                                           *
1003 *****************************************************************************/
1004