LXR linux/arch/powerpc/mm/hash_native

   1/*
   2 * native hashtable management.
   3 *
   4 * SMP scalability work:
   5 *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
   6 * 
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License
   9 * as published by the Free Software Foundation; either version
  10 * 2 of the License, or (at your option) any later version.
  11 */
  12
  13#undef DEBUG_LOW
  14
  15#include <linux/spinlock.h>
  16#include <linux/bitops.h>
  17#include <linux/threads.h>
  18#include <linux/smp.h>
  19
  20#include <asm/abs_addr.h>
  21#include <asm/machdep.h>
  22#include <asm/mmu.h>
  23#include <asm/mmu_context.h>
  24#include <asm/pgtable.h>
  25#include <asm/tlbflush.h>
  26#include <asm/tlb.h>
  27#include <asm/cputable.h>
  28#include <asm/udbg.h>
  29#include <asm/kexec.h>
  30#include <asm/ppc-opcode.h>
  31
  32#ifdef DEBUG_LOW
  33#define DBG_LOW(fmt...) udbg_printf(fmt)
  34#else
  35#define DBG_LOW(fmt...)
  36#endif
  37
  38#define HPTE_LOCK_BIT 3
  39
  40static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
  41
  42static inline void __tlbie(unsigned long va, int psize, int ssize)
  43{
  44        unsigned int penc;
  45
  46        /* clear top 16 bits, non SLS segment */
  47        va &= ~(0xffffULL << 48);
  48
  49        switch (psize) {
  50        case MMU_PAGE_4K:
  51                va &= ~0xffful;
  52                va |= ssize << 8;
  53                asm volatile(ASM_MMU_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0),
  54                                               %2)
  55                             : : "r" (va), "r"(0), "i" (MMU_FTR_TLBIE_206)
  56                             : "memory");
  57                break;
  58        default:
  59                penc = mmu_psize_defs[psize].penc;
  60                va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
  61                va |= penc << 12;
  62                va |= ssize << 8;
  63                va |= 1; /* L */
  64                asm volatile(ASM_MMU_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0),
  65                                               %2)
  66                             : : "r" (va), "r"(0), "i" (MMU_FTR_TLBIE_206)
  67                             : "memory");
  68                break;
  69        }
  70}
  71
  72static inline void __tlbiel(unsigned long va, int psize, int ssize)
  73{
  74        unsigned int penc;
  75
  76        /* clear top 16 bits, non SLS segment */
  77        va &= ~(0xffffULL << 48);
  78
  79        switch (psize) {
  80        case MMU_PAGE_4K:
  81                va &= ~0xffful;
  82                va |= ssize << 8;
  83                asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
  84                             : : "r"(va) : "memory");
  85                break;
  86        default:
  87                penc = mmu_psize_defs[psize].penc;
  88                va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
  89                va |= penc << 12;
  90                va |= ssize << 8;
  91                va |= 1; /* L */
  92                asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
  93                             : : "r"(va) : "memory");
  94                break;
  95        }
  96
  97}
  98
  99static inline void tlbie(unsigned long va, int psize, int ssize, int local)
 100{
 101        unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL);
 102        int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
 103
 104        if (use_local)
 105                use_local = mmu_psize_defs[psize].tlbiel;
 106        if (lock_tlbie && !use_local)
 107                raw_spin_lock(&native_tlbie_lock);
 108        asm volatile("ptesync": : :"memory");
 109        if (use_local) {
 110                __tlbiel(va, psize, ssize);
 111                asm volatile("ptesync": : :"memory");
 112        } else {
 113                __tlbie(va, psize, ssize);
 114                asm volatile("eieio; tlbsync; ptesync": : :"memory");
 115        }
 116        if (lock_tlbie && !use_local)
 117                raw_spin_unlock(&native_tlbie_lock);
 118}
 119
 120static inline void native_lock_hpte(struct hash_pte *hptep)
 121{
 122        unsigned long *word = &hptep->v;
 123
 124        while (1) {
 125                if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
 126                        break;
 127                while(test_bit(HPTE_LOCK_BIT, word))
 128                        cpu_relax();
 129        }
 130}
 131
 132static inline void native_unlock_hpte(struct hash_pte *hptep)
 133{
 134        unsigned long *word = &hptep->v;
 135
 136        clear_bit_unlock(HPTE_LOCK_BIT, word);
 137}
 138
 139static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
 140                        unsigned long pa, unsigned long rflags,
 141                        unsigned long vflags, int psize, int ssize)
 142{
 143        struct hash_pte *hptep = htab_address + hpte_group;
 144        unsigned long hpte_v, hpte_r;
 145        int i;
 146
 147        if (!(vflags & HPTE_V_BOLTED)) {
 148                DBG_LOW("    insert(group=%lx, va=%016lx, pa=%016lx,"
 149                        " rflags=%lx, vflags=%lx, psize=%d)\n",
 150                        hpte_group, va, pa, rflags, vflags, psize);
 151        }
 152
 153        for (i = 0; i < HPTES_PER_GROUP; i++) {
 154                if (! (hptep->v & HPTE_V_VALID)) {
 155                        /* retry with lock held */
 156                        native_lock_hpte(hptep);
 157                        if (! (hptep->v & HPTE_V_VALID))
 158                                break;
 159                        native_unlock_hpte(hptep);
 160                }
 161
 162                hptep++;
 163        }
 164
 165        if (i == HPTES_PER_GROUP)
 166                return -1;
 167
 168        hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID;
 169        hpte_r = hpte_encode_r(pa, psize) | rflags;
 170
 171        if (!(vflags & HPTE_V_BOLTED)) {
 172                DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
 173                        i, hpte_v, hpte_r);
 174        }
 175
 176        hptep->r = hpte_r;
 177        /* Guarantee the second dword is visible before the valid bit */
 178        eieio();
 179        /*
 180         * Now set the first dword including the valid bit
 181         * NOTE: this also unlocks the hpte
 182         */
 183        hptep->v = hpte_v;
 184
 185        __asm__ __volatile__ ("ptesync" : : : "memory");
 186
 187        return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
 188}
 189
 190static long native_hpte_remove(unsigned long hpte_group)
 191{
 192        struct hash_pte *hptep;
 193        int i;
 194        int slot_offset;
 195        unsigned long hpte_v;
 196
 197        DBG_LOW("    remove(group=%lx)\n", hpte_group);
 198
 199        /* pick a random entry to start at */
 200        slot_offset = mftb() & 0x7;
 201
 202        for (i = 0; i < HPTES_PER_GROUP; i++) {
 203                hptep = htab_address + hpte_group + slot_offset;
 204                hpte_v = hptep->v;
 205
 206                if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
 207                        /* retry with lock held */
 208                        native_lock_hpte(hptep);
 209                        hpte_v = hptep->v;
 210                        if ((hpte_v & HPTE_V_VALID)
 211                            && !(hpte_v & HPTE_V_BOLTED))
 212                                break;
 213                        native_unlock_hpte(hptep);
 214                }
 215
 216                slot_offset++;
 217                slot_offset &= 0x7;
 218        }
 219
 220        if (i == HPTES_PER_GROUP)
 221                return -1;
 222
 223        /* Invalidate the hpte. NOTE: this also unlocks it */
 224        hptep->v = 0;
 225
 226        return i;
 227}
 228
 229static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 230                                 unsigned long va, int psize, int ssize,
 231                                 int local)
 232{
 233        struct hash_pte *hptep = htab_address + slot;
 234        unsigned long hpte_v, want_v;
 235        int ret = 0;
 236
 237        want_v = hpte_encode_v(va, psize, ssize);
 238
 239        DBG_LOW("    update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)",
 240                va, want_v & HPTE_V_AVPN, slot, newpp);
 241
 242        native_lock_hpte(hptep);
 243
 244        hpte_v = hptep->v;
 245
 246        /* Even if we miss, we need to invalidate the TLB */
 247        if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
 248                DBG_LOW(" -> miss\n");
 249                ret = -1;
 250        } else {
 251                DBG_LOW(" -> hit\n");
 252                /* Update the HPTE */
 253                hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
 254                        (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C));
 255        }
 256        native_unlock_hpte(hptep);
 257
 258        /* Ensure it is out of the tlb too. */
 259        tlbie(va, psize, ssize, local);
 260
 261        return ret;
 262}
 263
 264static long native_hpte_find(unsigned long va, int psize, int ssize)
 265{
 266        struct hash_pte *hptep;
 267        unsigned long hash;
 268        unsigned long i;
 269        long slot;
 270        unsigned long want_v, hpte_v;
 271
 272        hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize);
 273        want_v = hpte_encode_v(va, psize, ssize);
 274
 275        /* Bolted mappings are only ever in the primary group */
 276        slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 277        for (i = 0; i < HPTES_PER_GROUP; i++) {
 278                hptep = htab_address + slot;
 279                hpte_v = hptep->v;
 280
 281                if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
 282                        /* HPTE matches */
 283                        return slot;
 284                ++slot;
 285        }
 286
 287        return -1;
 288}
 289
 290/*
 291 * Update the page protection bits. Intended to be used to create
 292 * guard pages for kernel data structures on pages which are bolted
 293 * in the HPT. Assumes pages being operated on will not be stolen.
 294 *
 295 * No need to lock here because we should be the only user.
 296 */
 297static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
 298                                       int psize, int ssize)
 299{
 300        unsigned long vsid, va;
 301        long slot;
 302        struct hash_pte *hptep;
 303
 304        vsid = get_kernel_vsid(ea, ssize);
 305        va = hpt_va(ea, vsid, ssize);
 306
 307        slot = native_hpte_find(va, psize, ssize);
 308        if (slot == -1)
 309                panic("could not find page to bolt\n");
 310        hptep = htab_address + slot;
 311
 312        /* Update the HPTE */
 313        hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
 314                (newpp & (HPTE_R_PP | HPTE_R_N));
 315
 316        /* Ensure it is out of the tlb too. */
 317        tlbie(va, psize, ssize, 0);
 318}
 319
 320static void native_hpte_invalidate(unsigned long slot, unsigned long va,
 321                                   int psize, int ssize, int local)
 322{
 323        struct hash_pte *hptep = htab_address + slot;
 324        unsigned long hpte_v;
 325        unsigned long want_v;
 326        unsigned long flags;
 327
 328        local_irq_save(flags);
 329
 330        DBG_LOW("    invalidate(va=%016lx, hash: %x)\n", va, slot);
 331
 332        want_v = hpte_encode_v(va, psize, ssize);
 333        native_lock_hpte(hptep);
 334        hpte_v = hptep->v;
 335
 336        /* Even if we miss, we need to invalidate the TLB */
 337        if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
 338                native_unlock_hpte(hptep);
 339        else
 340                /* Invalidate the hpte. NOTE: this also unlocks it */
 341                hptep->v = 0;
 342
 343        /* Invalidate the TLB */
 344        tlbie(va, psize, ssize, local);
 345
 346        local_irq_restore(flags);
 347}
 348
 349#define LP_SHIFT        12
 350#define LP_BITS         8
 351#define LP_MASK(i)      ((0xFF >> (i)) << LP_SHIFT)
 352
 353static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
 354                        int *psize, int *ssize, unsigned long *va)
 355{
 356        unsigned long hpte_r = hpte->r;
 357        unsigned long hpte_v = hpte->v;
 358        unsigned long avpn;
 359        int i, size, shift, penc;
 360
 361        if (!(hpte_v & HPTE_V_LARGE))
 362                size = MMU_PAGE_4K;
 363        else {
 364                for (i = 0; i < LP_BITS; i++) {
 365                        if ((hpte_r & LP_MASK(i+1)) == LP_MASK(i+1))
 366                                break;
 367                }
 368                penc = LP_MASK(i+1) >> LP_SHIFT;
 369                for (size = 0; size < MMU_PAGE_COUNT; size++) {
 370
 371                        /* 4K pages are not represented by LP */
 372                        if (size == MMU_PAGE_4K)
 373                                continue;
 374
 375                        /* valid entries have a shift value */
 376                        if (!mmu_psize_defs[size].shift)
 377                                continue;
 378
 379                        if (penc == mmu_psize_defs[size].penc)
 380                                break;
 381                }
 382        }
 383
 384        /* This works for all page sizes, and for 256M and 1T segments */
 385        shift = mmu_psize_defs[size].shift;
 386        avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm) << 23;
 387
 388        if (shift < 23) {
 389                unsigned long vpi, vsid, pteg;
 390
 391                pteg = slot / HPTES_PER_GROUP;
 392                if (hpte_v & HPTE_V_SECONDARY)
 393                        pteg = ~pteg;
 394                switch (hpte_v >> HPTE_V_SSIZE_SHIFT) {
 395                case MMU_SEGSIZE_256M:
 396                        vpi = ((avpn >> 28) ^ pteg) & htab_hash_mask;
 397                        break;
 398                case MMU_SEGSIZE_1T:
 399                        vsid = avpn >> 40;
 400                        vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
 401                        break;
 402                default:
 403                        avpn = vpi = size = 0;
 404                }
 405                avpn |= (vpi << mmu_psize_defs[size].shift);
 406        }
 407
 408        *va = avpn;
 409        *psize = size;
 410        *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
 411}
 412
 413/*
 414 * clear all mappings on kexec.  All cpus are in real mode (or they will
 415 * be when they isi), and we are the only one left.  We rely on our kernel
 416 * mapping being 0xC0's and the hardware ignoring those two real bits.
 417 *
 418 * TODO: add batching support when enabled.  remember, no dynamic memory here,
 419 * athough there is the control page available...
 420 */
 421static void native_hpte_clear(void)
 422{
 423        unsigned long slot, slots, flags;
 424        struct hash_pte *hptep = htab_address;
 425        unsigned long hpte_v, va;
 426        unsigned long pteg_count;
 427        int psize, ssize;
 428
 429        pteg_count = htab_hash_mask + 1;
 430
 431        local_irq_save(flags);
 432
 433        /* we take the tlbie lock and hold it.  Some hardware will
 434         * deadlock if we try to tlbie from two processors at once.
 435         */
 436        raw_spin_lock(&native_tlbie_lock);
 437
 438        slots = pteg_count * HPTES_PER_GROUP;
 439
 440        for (slot = 0; slot < slots; slot++, hptep++) {
 441                /*
 442                 * we could lock the pte here, but we are the only cpu
 443                 * running,  right?  and for crash dump, we probably
 444                 * don't want to wait for a maybe bad cpu.
 445                 */
 446                hpte_v = hptep->v;
 447
 448                /*
 449                 * Call __tlbie() here rather than tlbie() since we
 450                 * already hold the native_tlbie_lock.
 451                 */
 452                if (hpte_v & HPTE_V_VALID) {
 453                        hpte_decode(hptep, slot, &psize, &ssize, &va);
 454                        hptep->v = 0;
 455                        __tlbie(va, psize, ssize);
 456                }
 457        }
 458
 459        asm volatile("eieio; tlbsync; ptesync":::"memory");
 460        raw_spin_unlock(&native_tlbie_lock);
 461        local_irq_restore(flags);
 462}
 463
 464/*
 465 * Batched hash table flush, we batch the tlbie's to avoid taking/releasing
 466 * the lock all the time
 467 */
 468static void native_flush_hash_range(unsigned long number, int local)
 469{
 470        unsigned long va, hash, index, hidx, shift, slot;
 471        struct hash_pte *hptep;
 472        unsigned long hpte_v;
 473        unsigned long want_v;
 474        unsigned long flags;
 475        real_pte_t pte;
 476        struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
 477        unsigned long psize = batch->psize;
 478        int ssize = batch->ssize;
 479        int i;
 480
 481        local_irq_save(flags);
 482
 483        for (i = 0; i < number; i++) {
 484                va = batch->vaddr[i];
 485                pte = batch->pte[i];
 486
 487                pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
 488                        hash = hpt_hash(va, shift, ssize);
 489                        hidx = __rpte_to_hidx(pte, index);
 490                        if (hidx & _PTEIDX_SECONDARY)
 491                                hash = ~hash;
 492                        slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 493                        slot += hidx & _PTEIDX_GROUP_IX;
 494                        hptep = htab_address + slot;
 495                        want_v = hpte_encode_v(va, psize, ssize);
 496                        native_lock_hpte(hptep);
 497                        hpte_v = hptep->v;
 498                        if (!HPTE_V_COMPARE(hpte_v, want_v) ||
 499                            !(hpte_v & HPTE_V_VALID))
 500                                native_unlock_hpte(hptep);
 501                        else
 502                                hptep->v = 0;
 503                } pte_iterate_hashed_end();
 504        }
 505
 506        if (cpu_has_feature(CPU_FTR_TLBIEL) &&
 507            mmu_psize_defs[psize].tlbiel && local) {
 508                asm volatile("ptesync":::"memory");
 509                for (i = 0; i < number; i++) {
 510                        va = batch->vaddr[i];
 511                        pte = batch->pte[i];
 512
 513                        pte_iterate_hashed_subpages(pte, psize, va, index,
 514                                                    shift) {
 515                                __tlbiel(va, psize, ssize);
 516                        } pte_iterate_hashed_end();
 517                }
 518                asm volatile("ptesync":::"memory");
 519        } else {
 520                int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
 521
 522                if (lock_tlbie)
 523                        raw_spin_lock(&native_tlbie_lock);
 524
 525                asm volatile("ptesync":::"memory");
 526                for (i = 0; i < number; i++) {
 527                        va = batch->vaddr[i];
 528                        pte = batch->pte[i];
 529
 530                        pte_iterate_hashed_subpages(pte, psize, va, index,
 531                                                    shift) {
 532                                __tlbie(va, psize, ssize);
 533                        } pte_iterate_hashed_end();
 534                }
 535                asm volatile("eieio; tlbsync; ptesync":::"memory");
 536
 537                if (lock_tlbie)
 538                        raw_spin_unlock(&native_tlbie_lock);
 539        }
 540
 541        local_irq_restore(flags);
 542}
 543
 544#ifdef CONFIG_PPC_PSERIES
 545/* Disable TLB batching on nighthawk */
 546static inline int tlb_batching_enabled(void)
 547{
 548        struct device_node *root = of_find_node_by_path("/");
 549        int enabled = 1;
 550
 551        if (root) {
 552                const char *model = of_get_property(root, "model", NULL);
 553                if (model && !strcmp(model, "IBM,9076-N81"))
 554                        enabled = 0;
 555                of_node_put(root);
 556        }
 557
 558        return enabled;
 559}
 560#else
 561static inline int tlb_batching_enabled(void)
 562{
 563        return 1;
 564}
 565#endif
 566
 567void __init hpte_init_native(void)
 568{
 569        ppc_md.hpte_invalidate  = native_hpte_invalidate;
 570        ppc_md.hpte_updatepp    = native_hpte_updatepp;
 571        ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
 572        ppc_md.hpte_insert      = native_hpte_insert;
 573        ppc_md.hpte_remove      = native_hpte_remove;
 574        ppc_md.hpte_clear_all   = native_hpte_clear;
 575        if (tlb_batching_enabled())
 576                ppc_md.flush_hash_range = native_flush_hash_range;
 577}
 578