LXR linux/arch/powerpc/mm/hash_low

   1/*
   2 * ppc64 MMU hashtable management routines
   3 *
   4 * (c) Copyright IBM Corp. 2003, 2005
   5 *
   6 * Maintained by: Benjamin Herrenschmidt
   7 *                <benh@kernel.crashing.org>
   8 *
   9 * This file is covered by the GNU Public Licence v2 as
  10 * described in the kernel's COPYING file.
  11 */
  12
  13#include <asm/reg.h>
  14#include <asm/pgtable.h>
  15#include <asm/mmu.h>
  16#include <asm/page.h>
  17#include <asm/types.h>
  18#include <asm/ppc_asm.h>
  19#include <asm/asm-offsets.h>
  20#include <asm/cputable.h>
  21
  22        .text
  23
  24/*
  25 * Stackframe:
  26 *              
  27 *         +-> Back chain                       (SP + 256)
  28 *         |   General register save area       (SP + 112)
  29 *         |   Parameter save area              (SP + 48)
  30 *         |   TOC save area                    (SP + 40)
  31 *         |   link editor doubleword           (SP + 32)
  32 *         |   compiler doubleword              (SP + 24)
  33 *         |   LR save area                     (SP + 16)
  34 *         |   CR save area                     (SP + 8)
  35 * SP ---> +-- Back chain                       (SP + 0)
  36 */
  37#define STACKFRAMESIZE  256
  38
  39/* Save parameters offsets */
  40#define STK_PARM(i)     (STACKFRAMESIZE + 48 + ((i)-3)*8)
  41
  42/* Save non-volatile offsets */
  43#define STK_REG(i)      (112 + ((i)-14)*8)
  44
  45
  46#ifndef CONFIG_PPC_64K_PAGES
  47
  48/*****************************************************************************
  49 *                                                                           *
  50 *           4K SW & 4K HW pages implementation                              *
  51 *                                                                           *
  52 *****************************************************************************/
  53
  54
  55/*
  56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  57 *               pte_t *ptep, unsigned long trap, int local, int ssize)
  58 *
  59 * Adds a 4K page to the hash table in a segment of 4K pages only
  60 */
  61
  62_GLOBAL(__hash_page_4K)
  63        mflr    r0
  64        std     r0,16(r1)
  65        stdu    r1,-STACKFRAMESIZE(r1)
  66        /* Save all params that we need after a function call */
  67        std     r6,STK_PARM(r6)(r1)
  68        std     r8,STK_PARM(r8)(r1)
  69        std     r9,STK_PARM(r9)(r1)
  70        
  71        /* Save non-volatile registers.
  72         * r31 will hold "old PTE"
  73         * r30 is "new PTE"
  74         * r29 is "va"
  75         * r28 is a hash value
  76         * r27 is hashtab mask (maybe dynamic patched instead ?)
  77         */
  78        std     r27,STK_REG(r27)(r1)
  79        std     r28,STK_REG(r28)(r1)
  80        std     r29,STK_REG(r29)(r1)
  81        std     r30,STK_REG(r30)(r1)
  82        std     r31,STK_REG(r31)(r1)
  83        
  84        /* Step 1:
  85         *
  86         * Check permissions, atomically mark the linux PTE busy
  87         * and hashed.
  88         */ 
  891:
  90        ldarx   r31,0,r6
  91        /* Check access rights (access & ~(pte_val(*ptep))) */
  92        andc.   r0,r4,r31
  93        bne-    htab_wrong_access
  94        /* Check if PTE is busy */
  95        andi.   r0,r31,_PAGE_BUSY
  96        /* If so, just bail out and refault if needed. Someone else
  97         * is changing this PTE anyway and might hash it.
  98         */
  99        bne-    htab_bail_ok
 100
 101        /* Prepare new PTE value (turn access RW into DIRTY, then
 102         * add BUSY,HASHPTE and ACCESSED)
 103         */
 104        rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 105        or      r30,r30,r31
 106        ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 107        /* Write the linux PTE atomically (setting busy) */
 108        stdcx.  r30,0,r6
 109        bne-    1b
 110        isync
 111
 112        /* Step 2:
 113         *
 114         * Insert/Update the HPTE in the hash table. At this point,
 115         * r4 (access) is re-useable, we use it for the new HPTE flags
 116         */
 117
 118BEGIN_FTR_SECTION
 119        cmpdi   r9,0                    /* check segment size */
 120        bne     3f
 121END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 122        /* Calc va and put it in r29 */
 123        rldicr  r29,r5,28,63-28
 124        rldicl  r3,r3,0,36
 125        or      r29,r3,r29
 126
 127        /* Calculate hash value for primary slot and store it in r28 */
 128        rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 129        rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 130        xor     r28,r5,r0
 131        b       4f
 132
 1333:      /* Calc VA and hash in r29 and r28 for 1T segment */
 134        sldi    r29,r5,40               /* vsid << 40 */
 135        clrldi  r3,r3,24                /* ea & 0xffffffffff */
 136        rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 137        clrldi  r5,r5,40                /* vsid & 0xffffff */
 138        rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 139        xor     r28,r28,r5
 140        or      r29,r3,r29              /* VA */
 141        xor     r28,r28,r0              /* hash */
 142
 143        /* Convert linux PTE bits into HW equivalents */
 1444:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 145        xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 146        rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 147        rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 148        and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 149        andc    r0,r30,r0               /* r0 = pte & ~r0 */
 150        rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 151        ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 152
 153        /* We eventually do the icache sync here (maybe inline that
 154         * code rather than call a C function...) 
 155         */
 156BEGIN_FTR_SECTION
 157        mr      r4,r30
 158        mr      r5,r7
 159        bl      .hash_page_do_lazy_icache
 160END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 161
 162        /* At this point, r3 contains new PP bits, save them in
 163         * place of "access" in the param area (sic)
 164         */
 165        std     r3,STK_PARM(r4)(r1)
 166
 167        /* Get htab_hash_mask */
 168        ld      r4,htab_hash_mask@got(2)
 169        ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 170
 171        /* Check if we may already be in the hashtable, in this case, we
 172         * go to out-of-line code to try to modify the HPTE
 173         */
 174        andi.   r0,r31,_PAGE_HASHPTE
 175        bne     htab_modify_pte
 176
 177htab_insert_pte:
 178        /* Clear hpte bits in new pte (we also clear BUSY btw) and
 179         * add _PAGE_HASHPTE
 180         */
 181        lis     r0,_PAGE_HPTEFLAGS@h
 182        ori     r0,r0,_PAGE_HPTEFLAGS@l
 183        andc    r30,r30,r0
 184        ori     r30,r30,_PAGE_HASHPTE
 185
 186        /* physical address r5 */
 187        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 188        sldi    r5,r5,PAGE_SHIFT
 189
 190        /* Calculate primary group hash */
 191        and     r0,r28,r27
 192        rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 193
 194        /* Call ppc_md.hpte_insert */
 195        ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 196        mr      r4,r29                  /* Retreive va */
 197        li      r7,0                    /* !bolted, !secondary */
 198        li      r8,MMU_PAGE_4K          /* page size */
 199        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 200_GLOBAL(htab_call_hpte_insert1)
 201        bl      .                       /* Patched by htab_finish_init() */
 202        cmpdi   0,r3,0
 203        bge     htab_pte_insert_ok      /* Insertion successful */
 204        cmpdi   0,r3,-2                 /* Critical failure */
 205        beq-    htab_pte_insert_failure
 206
 207        /* Now try secondary slot */
 208        
 209        /* physical address r5 */
 210        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 211        sldi    r5,r5,PAGE_SHIFT
 212
 213        /* Calculate secondary group hash */
 214        andc    r0,r27,r28
 215        rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 216        
 217        /* Call ppc_md.hpte_insert */
 218        ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 219        mr      r4,r29                  /* Retreive va */
 220        li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 221        li      r8,MMU_PAGE_4K          /* page size */
 222        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 223_GLOBAL(htab_call_hpte_insert2)
 224        bl      .                       /* Patched by htab_finish_init() */
 225        cmpdi   0,r3,0
 226        bge+    htab_pte_insert_ok      /* Insertion successful */
 227        cmpdi   0,r3,-2                 /* Critical failure */
 228        beq-    htab_pte_insert_failure
 229
 230        /* Both are full, we need to evict something */
 231        mftb    r0
 232        /* Pick a random group based on TB */
 233        andi.   r0,r0,1
 234        mr      r5,r28
 235        bne     2f
 236        not     r5,r5
 2372:      and     r0,r5,r27
 238        rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */   
 239        /* Call ppc_md.hpte_remove */
 240_GLOBAL(htab_call_hpte_remove)
 241        bl      .                       /* Patched by htab_finish_init() */
 242
 243        /* Try all again */
 244        b       htab_insert_pte 
 245
 246htab_bail_ok:
 247        li      r3,0
 248        b       htab_bail
 249
 250htab_pte_insert_ok:
 251        /* Insert slot number & secondary bit in PTE */
 252        rldimi  r30,r3,12,63-15
 253                
 254        /* Write out the PTE with a normal write
 255         * (maybe add eieio may be good still ?)
 256         */
 257htab_write_out_pte:
 258        ld      r6,STK_PARM(r6)(r1)
 259        std     r30,0(r6)
 260        li      r3, 0
 261htab_bail:
 262        ld      r27,STK_REG(r27)(r1)
 263        ld      r28,STK_REG(r28)(r1)
 264        ld      r29,STK_REG(r29)(r1)
 265        ld      r30,STK_REG(r30)(r1)
 266        ld      r31,STK_REG(r31)(r1)
 267        addi    r1,r1,STACKFRAMESIZE
 268        ld      r0,16(r1)
 269        mtlr    r0
 270        blr
 271
 272htab_modify_pte:
 273        /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 274        mr      r4,r3
 275        rlwinm  r3,r31,32-12,29,31
 276
 277        /* Secondary group ? if yes, get a inverted hash value */
 278        mr      r5,r28
 279        andi.   r0,r31,_PAGE_SECONDARY
 280        beq     1f
 281        not     r5,r5
 2821:
 283        /* Calculate proper slot value for ppc_md.hpte_updatepp */
 284        and     r0,r5,r27
 285        rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 286        add     r3,r0,r3        /* add slot idx */
 287
 288        /* Call ppc_md.hpte_updatepp */
 289        mr      r5,r29                  /* va */
 290        li      r6,MMU_PAGE_4K          /* page size */
 291        ld      r7,STK_PARM(r9)(r1)     /* segment size */
 292        ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 293_GLOBAL(htab_call_hpte_updatepp)
 294        bl      .                       /* Patched by htab_finish_init() */
 295
 296        /* if we failed because typically the HPTE wasn't really here
 297         * we try an insertion. 
 298         */
 299        cmpdi   0,r3,-1
 300        beq-    htab_insert_pte
 301
 302        /* Clear the BUSY bit and Write out the PTE */
 303        li      r0,_PAGE_BUSY
 304        andc    r30,r30,r0
 305        b       htab_write_out_pte
 306
 307htab_wrong_access:
 308        /* Bail out clearing reservation */
 309        stdcx.  r31,0,r6
 310        li      r3,1
 311        b       htab_bail
 312
 313htab_pte_insert_failure:
 314        /* Bail out restoring old PTE */
 315        ld      r6,STK_PARM(r6)(r1)
 316        std     r31,0(r6)
 317        li      r3,-1
 318        b       htab_bail
 319
 320
 321#else /* CONFIG_PPC_64K_PAGES */
 322
 323
 324/*****************************************************************************
 325 *                                                                           *
 326 *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 327 *                                                                           *
 328 *****************************************************************************/
 329
 330/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 331 *               pte_t *ptep, unsigned long trap, int local, int ssize,
 332 *               int subpg_prot)
 333 */
 334
 335/*
 336 * For now, we do NOT implement Admixed pages
 337 */
 338_GLOBAL(__hash_page_4K)
 339        mflr    r0
 340        std     r0,16(r1)
 341        stdu    r1,-STACKFRAMESIZE(r1)
 342        /* Save all params that we need after a function call */
 343        std     r6,STK_PARM(r6)(r1)
 344        std     r8,STK_PARM(r8)(r1)
 345        std     r9,STK_PARM(r9)(r1)
 346
 347        /* Save non-volatile registers.
 348         * r31 will hold "old PTE"
 349         * r30 is "new PTE"
 350         * r29 is "va"
 351         * r28 is a hash value
 352         * r27 is hashtab mask (maybe dynamic patched instead ?)
 353         * r26 is the hidx mask
 354         * r25 is the index in combo page
 355         */
 356        std     r25,STK_REG(r25)(r1)
 357        std     r26,STK_REG(r26)(r1)
 358        std     r27,STK_REG(r27)(r1)
 359        std     r28,STK_REG(r28)(r1)
 360        std     r29,STK_REG(r29)(r1)
 361        std     r30,STK_REG(r30)(r1)
 362        std     r31,STK_REG(r31)(r1)
 363
 364        /* Step 1:
 365         *
 366         * Check permissions, atomically mark the linux PTE busy
 367         * and hashed.
 368         */
 3691:
 370        ldarx   r31,0,r6
 371        /* Check access rights (access & ~(pte_val(*ptep))) */
 372        andc.   r0,r4,r31
 373        bne-    htab_wrong_access
 374        /* Check if PTE is busy */
 375        andi.   r0,r31,_PAGE_BUSY
 376        /* If so, just bail out and refault if needed. Someone else
 377         * is changing this PTE anyway and might hash it.
 378         */
 379        bne-    htab_bail_ok
 380        /* Prepare new PTE value (turn access RW into DIRTY, then
 381         * add BUSY and ACCESSED)
 382         */
 383        rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 384        or      r30,r30,r31
 385        ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 386        oris    r30,r30,_PAGE_COMBO@h
 387        /* Write the linux PTE atomically (setting busy) */
 388        stdcx.  r30,0,r6
 389        bne-    1b
 390        isync
 391
 392        /* Step 2:
 393         *
 394         * Insert/Update the HPTE in the hash table. At this point,
 395         * r4 (access) is re-useable, we use it for the new HPTE flags
 396         */
 397
 398        /* Load the hidx index */
 399        rldicl  r25,r3,64-12,60
 400
 401BEGIN_FTR_SECTION
 402        cmpdi   r9,0                    /* check segment size */
 403        bne     3f
 404END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 405        /* Calc va and put it in r29 */
 406        rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
 407        rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
 408        or      r29,r3,r29              /* r29 = va */
 409
 410        /* Calculate hash value for primary slot and store it in r28 */
 411        rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 412        rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
 413        xor     r28,r5,r0
 414        b       4f
 415
 4163:      /* Calc VA and hash in r29 and r28 for 1T segment */
 417        sldi    r29,r5,40               /* vsid << 40 */
 418        clrldi  r3,r3,24                /* ea & 0xffffffffff */
 419        rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 420        clrldi  r5,r5,40                /* vsid & 0xffffff */
 421        rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
 422        xor     r28,r28,r5
 423        or      r29,r3,r29              /* VA */
 424        xor     r28,r28,r0              /* hash */
 425
 426        /* Convert linux PTE bits into HW equivalents */
 4274:
 428#ifdef CONFIG_PPC_SUBPAGE_PROT
 429        andc    r10,r30,r10
 430        andi.   r3,r10,0x1fe            /* Get basic set of flags */
 431        rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 432#else
 433        andi.   r3,r30,0x1fe            /* Get basic set of flags */
 434        rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 435#endif
 436        xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 437        rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 438        and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 439        andc    r0,r3,r0                /* r0 = pte & ~r0 */
 440        rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 441        ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 442
 443        /* We eventually do the icache sync here (maybe inline that
 444         * code rather than call a C function...)
 445         */
 446BEGIN_FTR_SECTION
 447        mr      r4,r30
 448        mr      r5,r7
 449        bl      .hash_page_do_lazy_icache
 450END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 451
 452        /* At this point, r3 contains new PP bits, save them in
 453         * place of "access" in the param area (sic)
 454         */
 455        std     r3,STK_PARM(r4)(r1)
 456
 457        /* Get htab_hash_mask */
 458        ld      r4,htab_hash_mask@got(2)
 459        ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 460
 461        /* Check if we may already be in the hashtable, in this case, we
 462         * go to out-of-line code to try to modify the HPTE. We look for
 463         * the bit at (1 >> (index + 32))
 464         */
 465        rldicl. r0,r31,64-12,48
 466        li      r26,0                   /* Default hidx */
 467        beq     htab_insert_pte
 468
 469        /*
 470         * Check if the pte was already inserted into the hash table
 471         * as a 64k HW page, and invalidate the 64k HPTE if so.
 472         */
 473        andis.  r0,r31,_PAGE_COMBO@h
 474        beq     htab_inval_old_hpte
 475
 476        ld      r6,STK_PARM(r6)(r1)
 477        ori     r26,r6,0x8000           /* Load the hidx mask */
 478        ld      r26,0(r26)
 479        addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 480        rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 481        bne     htab_modify_pte
 482
 483htab_insert_pte:
 484        /* real page number in r5, PTE RPN value + index */
 485        andis.  r0,r31,_PAGE_4K_PFN@h
 486        srdi    r5,r31,PTE_RPN_SHIFT
 487        bne-    htab_special_pfn
 488        sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 489        add     r5,r5,r25
 490htab_special_pfn:
 491        sldi    r5,r5,HW_PAGE_SHIFT
 492
 493        /* Calculate primary group hash */
 494        and     r0,r28,r27
 495        rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 496
 497        /* Call ppc_md.hpte_insert */
 498        ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 499        mr      r4,r29                  /* Retreive va */
 500        li      r7,0                    /* !bolted, !secondary */
 501        li      r8,MMU_PAGE_4K          /* page size */
 502        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 503_GLOBAL(htab_call_hpte_insert1)
 504        bl      .                       /* patched by htab_finish_init() */
 505        cmpdi   0,r3,0
 506        bge     htab_pte_insert_ok      /* Insertion successful */
 507        cmpdi   0,r3,-2                 /* Critical failure */
 508        beq-    htab_pte_insert_failure
 509
 510        /* Now try secondary slot */
 511
 512        /* real page number in r5, PTE RPN value + index */
 513        andis.  r0,r31,_PAGE_4K_PFN@h
 514        srdi    r5,r31,PTE_RPN_SHIFT
 515        bne-    3f
 516        sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
 517        add     r5,r5,r25
 5183:      sldi    r5,r5,HW_PAGE_SHIFT
 519
 520        /* Calculate secondary group hash */
 521        andc    r0,r27,r28
 522        rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 523
 524        /* Call ppc_md.hpte_insert */
 525        ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 526        mr      r4,r29                  /* Retreive va */
 527        li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 528        li      r8,MMU_PAGE_4K          /* page size */
 529        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 530_GLOBAL(htab_call_hpte_insert2)
 531        bl      .                       /* patched by htab_finish_init() */
 532        cmpdi   0,r3,0
 533        bge+    htab_pte_insert_ok      /* Insertion successful */
 534        cmpdi   0,r3,-2                 /* Critical failure */
 535        beq-    htab_pte_insert_failure
 536
 537        /* Both are full, we need to evict something */
 538        mftb    r0
 539        /* Pick a random group based on TB */
 540        andi.   r0,r0,1
 541        mr      r5,r28
 542        bne     2f
 543        not     r5,r5
 5442:      and     r0,r5,r27
 545        rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 546        /* Call ppc_md.hpte_remove */
 547_GLOBAL(htab_call_hpte_remove)
 548        bl      .                       /* patched by htab_finish_init() */
 549
 550        /* Try all again */
 551        b       htab_insert_pte
 552
 553        /*
 554         * Call out to C code to invalidate an 64k HW HPTE that is
 555         * useless now that the segment has been switched to 4k pages.
 556         */
 557htab_inval_old_hpte:
 558        mr      r3,r29                  /* virtual addr */
 559        mr      r4,r31                  /* PTE.pte */
 560        li      r5,0                    /* PTE.hidx */
 561        li      r6,MMU_PAGE_64K         /* psize */
 562        ld      r7,STK_PARM(r9)(r1)     /* ssize */
 563        ld      r8,STK_PARM(r8)(r1)     /* local */
 564        bl      .flush_hash_page
 565        /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 566        lis     r0,_PAGE_HPTE_SUB@h
 567        ori     r0,r0,_PAGE_HPTE_SUB@l
 568        andc    r30,r30,r0
 569        b       htab_insert_pte
 570        
 571htab_bail_ok:
 572        li      r3,0
 573        b       htab_bail
 574
 575htab_pte_insert_ok:
 576        /* Insert slot number & secondary bit in PTE second half,
 577         * clear _PAGE_BUSY and set approriate HPTE slot bit
 578         */
 579        ld      r6,STK_PARM(r6)(r1)
 580        li      r0,_PAGE_BUSY
 581        andc    r30,r30,r0
 582        /* HPTE SUB bit */
 583        li      r0,1
 584        subfic  r5,r25,27               /* Must match bit position in */
 585        sld     r0,r0,r5                /* pgtable.h */
 586        or      r30,r30,r0
 587        /* hindx */
 588        sldi    r5,r25,2
 589        sld     r3,r3,r5
 590        li      r4,0xf
 591        sld     r4,r4,r5
 592        andc    r26,r26,r4
 593        or      r26,r26,r3
 594        ori     r5,r6,0x8000
 595        std     r26,0(r5)
 596        lwsync
 597        std     r30,0(r6)
 598        li      r3, 0
 599htab_bail:
 600        ld      r25,STK_REG(r25)(r1)
 601        ld      r26,STK_REG(r26)(r1)
 602        ld      r27,STK_REG(r27)(r1)
 603        ld      r28,STK_REG(r28)(r1)
 604        ld      r29,STK_REG(r29)(r1)
 605        ld      r30,STK_REG(r30)(r1)
 606        ld      r31,STK_REG(r31)(r1)
 607        addi    r1,r1,STACKFRAMESIZE
 608        ld      r0,16(r1)
 609        mtlr    r0
 610        blr
 611
 612htab_modify_pte:
 613        /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 614        mr      r4,r3
 615        sldi    r5,r25,2
 616        srd     r3,r26,r5
 617
 618        /* Secondary group ? if yes, get a inverted hash value */
 619        mr      r5,r28
 620        andi.   r0,r3,0x8 /* page secondary ? */
 621        beq     1f
 622        not     r5,r5
 6231:      andi.   r3,r3,0x7 /* extract idx alone */
 624
 625        /* Calculate proper slot value for ppc_md.hpte_updatepp */
 626        and     r0,r5,r27
 627        rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 628        add     r3,r0,r3        /* add slot idx */
 629
 630        /* Call ppc_md.hpte_updatepp */
 631        mr      r5,r29                  /* va */
 632        li      r6,MMU_PAGE_4K          /* page size */
 633        ld      r7,STK_PARM(r9)(r1)     /* segment size */
 634        ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 635_GLOBAL(htab_call_hpte_updatepp)
 636        bl      .                       /* patched by htab_finish_init() */
 637
 638        /* if we failed because typically the HPTE wasn't really here
 639         * we try an insertion.
 640         */
 641        cmpdi   0,r3,-1
 642        beq-    htab_insert_pte
 643
 644        /* Clear the BUSY bit and Write out the PTE */
 645        li      r0,_PAGE_BUSY
 646        andc    r30,r30,r0
 647        ld      r6,STK_PARM(r6)(r1)
 648        std     r30,0(r6)
 649        li      r3,0
 650        b       htab_bail
 651
 652htab_wrong_access:
 653        /* Bail out clearing reservation */
 654        stdcx.  r31,0,r6
 655        li      r3,1
 656        b       htab_bail
 657
 658htab_pte_insert_failure:
 659        /* Bail out restoring old PTE */
 660        ld      r6,STK_PARM(r6)(r1)
 661        std     r31,0(r6)
 662        li      r3,-1
 663        b       htab_bail
 664
 665#endif /* CONFIG_PPC_64K_PAGES */
 666
 667#ifdef CONFIG_PPC_HAS_HASH_64K
 668
 669/*****************************************************************************
 670 *                                                                           *
 671 *           64K SW & 64K HW in a 64K segment pages implementation           *
 672 *                                                                           *
 673 *****************************************************************************/
 674
 675_GLOBAL(__hash_page_64K)
 676        mflr    r0
 677        std     r0,16(r1)
 678        stdu    r1,-STACKFRAMESIZE(r1)
 679        /* Save all params that we need after a function call */
 680        std     r6,STK_PARM(r6)(r1)
 681        std     r8,STK_PARM(r8)(r1)
 682        std     r9,STK_PARM(r9)(r1)
 683
 684        /* Save non-volatile registers.
 685         * r31 will hold "old PTE"
 686         * r30 is "new PTE"
 687         * r29 is "va"
 688         * r28 is a hash value
 689         * r27 is hashtab mask (maybe dynamic patched instead ?)
 690         */
 691        std     r27,STK_REG(r27)(r1)
 692        std     r28,STK_REG(r28)(r1)
 693        std     r29,STK_REG(r29)(r1)
 694        std     r30,STK_REG(r30)(r1)
 695        std     r31,STK_REG(r31)(r1)
 696
 697        /* Step 1:
 698         *
 699         * Check permissions, atomically mark the linux PTE busy
 700         * and hashed.
 701         */
 7021:
 703        ldarx   r31,0,r6
 704        /* Check access rights (access & ~(pte_val(*ptep))) */
 705        andc.   r0,r4,r31
 706        bne-    ht64_wrong_access
 707        /* Check if PTE is busy */
 708        andi.   r0,r31,_PAGE_BUSY
 709        /* If so, just bail out and refault if needed. Someone else
 710         * is changing this PTE anyway and might hash it.
 711         */
 712        bne-    ht64_bail_ok
 713BEGIN_FTR_SECTION
 714        /* Check if PTE has the cache-inhibit bit set */
 715        andi.   r0,r31,_PAGE_NO_CACHE
 716        /* If so, bail out and refault as a 4k page */
 717        bne-    ht64_bail_ok
 718END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
 719        /* Prepare new PTE value (turn access RW into DIRTY, then
 720         * add BUSY and ACCESSED)
 721         */
 722        rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 723        or      r30,r30,r31
 724        ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 725        /* Write the linux PTE atomically (setting busy) */
 726        stdcx.  r30,0,r6
 727        bne-    1b
 728        isync
 729
 730        /* Step 2:
 731         *
 732         * Insert/Update the HPTE in the hash table. At this point,
 733         * r4 (access) is re-useable, we use it for the new HPTE flags
 734         */
 735
 736BEGIN_FTR_SECTION
 737        cmpdi   r9,0                    /* check segment size */
 738        bne     3f
 739END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 740        /* Calc va and put it in r29 */
 741        rldicr  r29,r5,28,63-28
 742        rldicl  r3,r3,0,36
 743        or      r29,r3,r29
 744
 745        /* Calculate hash value for primary slot and store it in r28 */
 746        rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
 747        rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
 748        xor     r28,r5,r0
 749        b       4f
 750
 7513:      /* Calc VA and hash in r29 and r28 for 1T segment */
 752        sldi    r29,r5,40               /* vsid << 40 */
 753        clrldi  r3,r3,24                /* ea & 0xffffffffff */
 754        rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
 755        clrldi  r5,r5,40                /* vsid & 0xffffff */
 756        rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
 757        xor     r28,r28,r5
 758        or      r29,r3,r29              /* VA */
 759        xor     r28,r28,r0              /* hash */
 760
 761        /* Convert linux PTE bits into HW equivalents */
 7624:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 763        xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 764        rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 765        rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 766        and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 767        andc    r0,r30,r0               /* r0 = pte & ~r0 */
 768        rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 769        ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
 770
 771        /* We eventually do the icache sync here (maybe inline that
 772         * code rather than call a C function...)
 773         */
 774BEGIN_FTR_SECTION
 775        mr      r4,r30
 776        mr      r5,r7
 777        bl      .hash_page_do_lazy_icache
 778END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 779
 780        /* At this point, r3 contains new PP bits, save them in
 781         * place of "access" in the param area (sic)
 782         */
 783        std     r3,STK_PARM(r4)(r1)
 784
 785        /* Get htab_hash_mask */
 786        ld      r4,htab_hash_mask@got(2)
 787        ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 788
 789        /* Check if we may already be in the hashtable, in this case, we
 790         * go to out-of-line code to try to modify the HPTE
 791         */
 792        rldicl. r0,r31,64-12,48
 793        bne     ht64_modify_pte
 794
 795ht64_insert_pte:
 796        /* Clear hpte bits in new pte (we also clear BUSY btw) and
 797         * add _PAGE_HPTE_SUB0
 798         */
 799        lis     r0,_PAGE_HPTEFLAGS@h
 800        ori     r0,r0,_PAGE_HPTEFLAGS@l
 801        andc    r30,r30,r0
 802#ifdef CONFIG_PPC_64K_PAGES
 803        oris    r30,r30,_PAGE_HPTE_SUB0@h
 804#else
 805        ori     r30,r30,_PAGE_HASHPTE
 806#endif
 807        /* Phyical address in r5 */
 808        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 809        sldi    r5,r5,PAGE_SHIFT
 810
 811        /* Calculate primary group hash */
 812        and     r0,r28,r27
 813        rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 814
 815        /* Call ppc_md.hpte_insert */
 816        ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 817        mr      r4,r29                  /* Retreive va */
 818        li      r7,0                    /* !bolted, !secondary */
 819        li      r8,MMU_PAGE_64K
 820        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 821_GLOBAL(ht64_call_hpte_insert1)
 822        bl      .                       /* patched by htab_finish_init() */
 823        cmpdi   0,r3,0
 824        bge     ht64_pte_insert_ok      /* Insertion successful */
 825        cmpdi   0,r3,-2                 /* Critical failure */
 826        beq-    ht64_pte_insert_failure
 827
 828        /* Now try secondary slot */
 829
 830        /* Phyical address in r5 */
 831        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 832        sldi    r5,r5,PAGE_SHIFT
 833
 834        /* Calculate secondary group hash */
 835        andc    r0,r27,r28
 836        rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 837
 838        /* Call ppc_md.hpte_insert */
 839        ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
 840        mr      r4,r29                  /* Retreive va */
 841        li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 842        li      r8,MMU_PAGE_64K
 843        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 844_GLOBAL(ht64_call_hpte_insert2)
 845        bl      .                       /* patched by htab_finish_init() */
 846        cmpdi   0,r3,0
 847        bge+    ht64_pte_insert_ok      /* Insertion successful */
 848        cmpdi   0,r3,-2                 /* Critical failure */
 849        beq-    ht64_pte_insert_failure
 850
 851        /* Both are full, we need to evict something */
 852        mftb    r0
 853        /* Pick a random group based on TB */
 854        andi.   r0,r0,1
 855        mr      r5,r28
 856        bne     2f
 857        not     r5,r5
 8582:      and     r0,r5,r27
 859        rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 860        /* Call ppc_md.hpte_remove */
 861_GLOBAL(ht64_call_hpte_remove)
 862        bl      .                       /* patched by htab_finish_init() */
 863
 864        /* Try all again */
 865        b       ht64_insert_pte
 866
 867ht64_bail_ok:
 868        li      r3,0
 869        b       ht64_bail
 870
 871ht64_pte_insert_ok:
 872        /* Insert slot number & secondary bit in PTE */
 873        rldimi  r30,r3,12,63-15
 874
 875        /* Write out the PTE with a normal write
 876         * (maybe add eieio may be good still ?)
 877         */
 878ht64_write_out_pte:
 879        ld      r6,STK_PARM(r6)(r1)
 880        std     r30,0(r6)
 881        li      r3, 0
 882ht64_bail:
 883        ld      r27,STK_REG(r27)(r1)
 884        ld      r28,STK_REG(r28)(r1)
 885        ld      r29,STK_REG(r29)(r1)
 886        ld      r30,STK_REG(r30)(r1)
 887        ld      r31,STK_REG(r31)(r1)
 888        addi    r1,r1,STACKFRAMESIZE
 889        ld      r0,16(r1)
 890        mtlr    r0
 891        blr
 892
 893ht64_modify_pte:
 894        /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 895        mr      r4,r3
 896        rlwinm  r3,r31,32-12,29,31
 897
 898        /* Secondary group ? if yes, get a inverted hash value */
 899        mr      r5,r28
 900        andi.   r0,r31,_PAGE_F_SECOND
 901        beq     1f
 902        not     r5,r5
 9031:
 904        /* Calculate proper slot value for ppc_md.hpte_updatepp */
 905        and     r0,r5,r27
 906        rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 907        add     r3,r0,r3        /* add slot idx */
 908
 909        /* Call ppc_md.hpte_updatepp */
 910        mr      r5,r29                  /* va */
 911        li      r6,MMU_PAGE_64K
 912        ld      r7,STK_PARM(r9)(r1)     /* segment size */
 913        ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 914_GLOBAL(ht64_call_hpte_updatepp)
 915        bl      .                       /* patched by htab_finish_init() */
 916
 917        /* if we failed because typically the HPTE wasn't really here
 918         * we try an insertion.
 919         */
 920        cmpdi   0,r3,-1
 921        beq-    ht64_insert_pte
 922
 923        /* Clear the BUSY bit and Write out the PTE */
 924        li      r0,_PAGE_BUSY
 925        andc    r30,r30,r0
 926        b       ht64_write_out_pte
 927
 928ht64_wrong_access:
 929        /* Bail out clearing reservation */
 930        stdcx.  r31,0,r6
 931        li      r3,1
 932        b       ht64_bail
 933
 934ht64_pte_insert_failure:
 935        /* Bail out restoring old PTE */
 936        ld      r6,STK_PARM(r6)(r1)
 937        std     r31,0(r6)
 938        li      r3,-1
 939        b       ht64_bail
 940
 941
 942#endif /* CONFIG_PPC_HAS_HASH_64K */
 943
 944
 945/*****************************************************************************
 946 *                                                                           *
 947 *           Huge pages implementation is in hugetlbpage.c                   *
 948 *                                                                           *
 949 *****************************************************************************/
 950