LXR linux/arch/powerpc/platforms/cell/beat

   1/*
   2 * "Cell Reference Set" HTAB support.
   3 *
   4 * (C) Copyright 2006-2007 TOSHIBA CORPORATION
   5 *
   6 * This code is based on arch/powerpc/platforms/pseries/lpar.c:
   7 *  Copyright (C) 2001 Todd Inglett, IBM Corporation
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License as published by
  11 * the Free Software Foundation; either version 2 of the License, or
  12 * (at your option) any later version.
  13 *
  14 * This program is distributed in the hope that it will be useful,
  15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 * GNU General Public License for more details.
  18 *
  19 * You should have received a copy of the GNU General Public License along
  20 * with this program; if not, write to the Free Software Foundation, Inc.,
  21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22 */
  23
  24#undef DEBUG_LOW
  25
  26#include <linux/kernel.h>
  27#include <linux/spinlock.h>
  28
  29#include <asm/mmu.h>
  30#include <asm/page.h>
  31#include <asm/pgtable.h>
  32#include <asm/machdep.h>
  33#include <asm/udbg.h>
  34
  35#include "beat_wrapper.h"
  36
  37#ifdef DEBUG_LOW
  38#define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
  39#else
  40#define DBG_LOW(fmt...) do { } while (0)
  41#endif
  42
  43static DEFINE_RAW_SPINLOCK(beat_htab_lock);
  44
  45static inline unsigned int beat_read_mask(unsigned hpte_group)
  46{
  47        unsigned long rmask = 0;
  48        u64 hpte_v[5];
  49
  50        beat_read_htab_entries(0, hpte_group + 0, hpte_v);
  51        if (!(hpte_v[0] & HPTE_V_BOLTED))
  52                rmask |= 0x8000;
  53        if (!(hpte_v[1] & HPTE_V_BOLTED))
  54                rmask |= 0x4000;
  55        if (!(hpte_v[2] & HPTE_V_BOLTED))
  56                rmask |= 0x2000;
  57        if (!(hpte_v[3] & HPTE_V_BOLTED))
  58                rmask |= 0x1000;
  59        beat_read_htab_entries(0, hpte_group + 4, hpte_v);
  60        if (!(hpte_v[0] & HPTE_V_BOLTED))
  61                rmask |= 0x0800;
  62        if (!(hpte_v[1] & HPTE_V_BOLTED))
  63                rmask |= 0x0400;
  64        if (!(hpte_v[2] & HPTE_V_BOLTED))
  65                rmask |= 0x0200;
  66        if (!(hpte_v[3] & HPTE_V_BOLTED))
  67                rmask |= 0x0100;
  68        hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
  69        beat_read_htab_entries(0, hpte_group + 0, hpte_v);
  70        if (!(hpte_v[0] & HPTE_V_BOLTED))
  71                rmask |= 0x80;
  72        if (!(hpte_v[1] & HPTE_V_BOLTED))
  73                rmask |= 0x40;
  74        if (!(hpte_v[2] & HPTE_V_BOLTED))
  75                rmask |= 0x20;
  76        if (!(hpte_v[3] & HPTE_V_BOLTED))
  77                rmask |= 0x10;
  78        beat_read_htab_entries(0, hpte_group + 4, hpte_v);
  79        if (!(hpte_v[0] & HPTE_V_BOLTED))
  80                rmask |= 0x08;
  81        if (!(hpte_v[1] & HPTE_V_BOLTED))
  82                rmask |= 0x04;
  83        if (!(hpte_v[2] & HPTE_V_BOLTED))
  84                rmask |= 0x02;
  85        if (!(hpte_v[3] & HPTE_V_BOLTED))
  86                rmask |= 0x01;
  87        return rmask;
  88}
  89
  90static long beat_lpar_hpte_insert(unsigned long hpte_group,
  91                                  unsigned long vpn, unsigned long pa,
  92                                  unsigned long rflags, unsigned long vflags,
  93                                  int psize, int apsize, int ssize)
  94{
  95        unsigned long lpar_rc;
  96        u64 hpte_v, hpte_r, slot;
  97
  98        if (vflags & HPTE_V_SECONDARY)
  99                return -1;
 100
 101        if (!(vflags & HPTE_V_BOLTED))
 102                DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
 103                        "rflags=%lx, vflags=%lx, psize=%d)\n",
 104                hpte_group, va, pa, rflags, vflags, psize);
 105
 106        hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
 107                vflags | HPTE_V_VALID;
 108        hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 109
 110        if (!(vflags & HPTE_V_BOLTED))
 111                DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
 112
 113        if (rflags & _PAGE_NO_CACHE)
 114                hpte_r &= ~_PAGE_COHERENT;
 115
 116        raw_spin_lock(&beat_htab_lock);
 117        lpar_rc = beat_read_mask(hpte_group);
 118        if (lpar_rc == 0) {
 119                if (!(vflags & HPTE_V_BOLTED))
 120                        DBG_LOW(" full\n");
 121                raw_spin_unlock(&beat_htab_lock);
 122                return -1;
 123        }
 124
 125        lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
 126                hpte_v, hpte_r, &slot);
 127        raw_spin_unlock(&beat_htab_lock);
 128
 129        /*
 130         * Since we try and ioremap PHBs we don't own, the pte insert
 131         * will fail. However we must catch the failure in hash_page
 132         * or we will loop forever, so return -2 in this case.
 133         */
 134        if (unlikely(lpar_rc != 0)) {
 135                if (!(vflags & HPTE_V_BOLTED))
 136                        DBG_LOW(" lpar err %lx\n", lpar_rc);
 137                return -2;
 138        }
 139        if (!(vflags & HPTE_V_BOLTED))
 140                DBG_LOW(" -> slot: %lx\n", slot);
 141
 142        /* We have to pass down the secondary bucket bit here as well */
 143        return (slot ^ hpte_group) & 15;
 144}
 145
 146static long beat_lpar_hpte_remove(unsigned long hpte_group)
 147{
 148        DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
 149        return -1;
 150}
 151
 152static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
 153{
 154        unsigned long dword0;
 155        unsigned long lpar_rc;
 156        u64 dword[5];
 157
 158        lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);
 159
 160        dword0 = dword[slot&3];
 161
 162        BUG_ON(lpar_rc != 0);
 163
 164        return dword0;
 165}
 166
 167static void beat_lpar_hptab_clear(void)
 168{
 169        unsigned long size_bytes = 1UL << ppc64_pft_size;
 170        unsigned long hpte_count = size_bytes >> 4;
 171        int i;
 172        u64 dummy0, dummy1;
 173
 174        /* TODO: Use bulk call */
 175        for (i = 0; i < hpte_count; i++)
 176                beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
 177}
 178
 179/*
 180 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 181 * the low 3 bits of flags happen to line up.  So no transform is needed.
 182 * We can probably optimize here and assume the high bits of newpp are
 183 * already zero.  For now I am paranoid.
 184 */
 185static long beat_lpar_hpte_updatepp(unsigned long slot,
 186                                    unsigned long newpp,
 187                                    unsigned long vpn,
 188                                    int psize, int ssize, int local)
 189{
 190        unsigned long lpar_rc;
 191        u64 dummy0, dummy1;
 192        unsigned long want_v;
 193
 194        want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 195
 196        DBG_LOW("    update: "
 197                "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
 198                want_v & HPTE_V_AVPN, slot, psize, newpp);
 199
 200        raw_spin_lock(&beat_htab_lock);
 201        dummy0 = beat_lpar_hpte_getword0(slot);
 202        if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
 203                DBG_LOW("not found !\n");
 204                raw_spin_unlock(&beat_htab_lock);
 205                return -1;
 206        }
 207
 208        lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
 209                                        &dummy1);
 210        raw_spin_unlock(&beat_htab_lock);
 211        if (lpar_rc != 0 || dummy0 == 0) {
 212                DBG_LOW("not found !\n");
 213                return -1;
 214        }
 215
 216        DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
 217
 218        BUG_ON(lpar_rc != 0);
 219
 220        return 0;
 221}
 222
 223static long beat_lpar_hpte_find(unsigned long vpn, int psize)
 224{
 225        unsigned long hash;
 226        unsigned long i, j;
 227        long slot;
 228        unsigned long want_v, hpte_v;
 229
 230        hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
 231        want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 232
 233        for (j = 0; j < 2; j++) {
 234                slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 235                for (i = 0; i < HPTES_PER_GROUP; i++) {
 236                        hpte_v = beat_lpar_hpte_getword0(slot);
 237
 238                        if (HPTE_V_COMPARE(hpte_v, want_v)
 239                            && (hpte_v & HPTE_V_VALID)
 240                            && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
 241                                /* HPTE matches */
 242                                if (j)
 243                                        slot = -slot;
 244                                return slot;
 245                        }
 246                        ++slot;
 247                }
 248                hash = ~hash;
 249        }
 250
 251        return -1;
 252}
 253
 254static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
 255                                          unsigned long ea,
 256                                          int psize, int ssize)
 257{
 258        unsigned long vpn;
 259        unsigned long lpar_rc, slot, vsid;
 260        u64 dummy0, dummy1;
 261
 262        vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
 263        vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M);
 264
 265        raw_spin_lock(&beat_htab_lock);
 266        slot = beat_lpar_hpte_find(vpn, psize);
 267        BUG_ON(slot == -1);
 268
 269        lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
 270                &dummy0, &dummy1);
 271        raw_spin_unlock(&beat_htab_lock);
 272
 273        BUG_ON(lpar_rc != 0);
 274}
 275
 276static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
 277                                         int psize, int ssize, int local)
 278{
 279        unsigned long want_v;
 280        unsigned long lpar_rc;
 281        u64 dummy1, dummy2;
 282        unsigned long flags;
 283
 284        DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
 285                slot, va, psize, local);
 286        want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 287
 288        raw_spin_lock_irqsave(&beat_htab_lock, flags);
 289        dummy1 = beat_lpar_hpte_getword0(slot);
 290
 291        if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
 292                DBG_LOW("not found !\n");
 293                raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
 294                return;
 295        }
 296
 297        lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
 298                &dummy1, &dummy2);
 299        raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
 300
 301        BUG_ON(lpar_rc != 0);
 302}
 303
 304void __init hpte_init_beat(void)
 305{
 306        ppc_md.hpte_invalidate  = beat_lpar_hpte_invalidate;
 307        ppc_md.hpte_updatepp    = beat_lpar_hpte_updatepp;
 308        ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
 309        ppc_md.hpte_insert      = beat_lpar_hpte_insert;
 310        ppc_md.hpte_remove      = beat_lpar_hpte_remove;
 311        ppc_md.hpte_clear_all   = beat_lpar_hptab_clear;
 312}
 313
 314static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
 315                                  unsigned long vpn, unsigned long pa,
 316                                  unsigned long rflags, unsigned long vflags,
 317                                  int psize, int apsize, int ssize)
 318{
 319        unsigned long lpar_rc;
 320        u64 hpte_v, hpte_r, slot;
 321
 322        if (vflags & HPTE_V_SECONDARY)
 323                return -1;
 324
 325        if (!(vflags & HPTE_V_BOLTED))
 326                DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
 327                        "rflags=%lx, vflags=%lx, psize=%d)\n",
 328                hpte_group, vpn, pa, rflags, vflags, psize);
 329
 330        hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
 331                vflags | HPTE_V_VALID;
 332        hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 333
 334        if (!(vflags & HPTE_V_BOLTED))
 335                DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
 336
 337        if (rflags & _PAGE_NO_CACHE)
 338                hpte_r &= ~_PAGE_COHERENT;
 339
 340        /* insert into not-volted entry */
 341        lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
 342                HPTE_V_BOLTED, 0, &slot);
 343        /*
 344         * Since we try and ioremap PHBs we don't own, the pte insert
 345         * will fail. However we must catch the failure in hash_page
 346         * or we will loop forever, so return -2 in this case.
 347         */
 348        if (unlikely(lpar_rc != 0)) {
 349                if (!(vflags & HPTE_V_BOLTED))
 350                        DBG_LOW(" lpar err %lx\n", lpar_rc);
 351                return -2;
 352        }
 353        if (!(vflags & HPTE_V_BOLTED))
 354                DBG_LOW(" -> slot: %lx\n", slot);
 355
 356        /* We have to pass down the secondary bucket bit here as well */
 357        return (slot ^ hpte_group) & 15;
 358}
 359
 360/*
 361 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 362 * the low 3 bits of flags happen to line up.  So no transform is needed.
 363 * We can probably optimize here and assume the high bits of newpp are
 364 * already zero.  For now I am paranoid.
 365 */
 366static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
 367                                    unsigned long newpp,
 368                                    unsigned long vpn,
 369                                    int psize, int ssize, int local)
 370{
 371        unsigned long lpar_rc;
 372        unsigned long want_v;
 373        unsigned long pss;
 374
 375        want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 376        pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
 377
 378        DBG_LOW("    update: "
 379                "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
 380                want_v & HPTE_V_AVPN, slot, psize, newpp);
 381
 382        lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
 383
 384        if (lpar_rc == 0xfffffff7) {
 385                DBG_LOW("not found !\n");
 386                return -1;
 387        }
 388
 389        DBG_LOW("ok\n");
 390
 391        BUG_ON(lpar_rc != 0);
 392
 393        return 0;
 394}
 395
 396static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
 397                                         int psize, int ssize, int local)
 398{
 399        unsigned long want_v;
 400        unsigned long lpar_rc;
 401        unsigned long pss;
 402
 403        DBG_LOW("    inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
 404                slot, vpn, psize, local);
 405        want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 406        pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
 407
 408        lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
 409
 410        /* E_busy can be valid output: page may be already replaced */
 411        BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
 412}
 413
 414static int64_t _beat_lpar_hptab_clear_v3(void)
 415{
 416        return beat_clear_htab3(0);
 417}
 418
 419static void beat_lpar_hptab_clear_v3(void)
 420{
 421        _beat_lpar_hptab_clear_v3();
 422}
 423
 424void __init hpte_init_beat_v3(void)
 425{
 426        if (_beat_lpar_hptab_clear_v3() == 0) {
 427                ppc_md.hpte_invalidate  = beat_lpar_hpte_invalidate_v3;
 428                ppc_md.hpte_updatepp    = beat_lpar_hpte_updatepp_v3;
 429                ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
 430                ppc_md.hpte_insert      = beat_lpar_hpte_insert_v3;
 431                ppc_md.hpte_remove      = beat_lpar_hpte_remove;
 432                ppc_md.hpte_clear_all   = beat_lpar_hptab_clear_v3;
 433        } else {
 434                ppc_md.hpte_invalidate  = beat_lpar_hpte_invalidate;
 435                ppc_md.hpte_updatepp    = beat_lpar_hpte_updatepp;
 436                ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
 437                ppc_md.hpte_insert      = beat_lpar_hpte_insert;
 438                ppc_md.hpte_remove      = beat_lpar_hpte_remove;
 439                ppc_md.hpte_clear_all   = beat_lpar_hptab_clear;
 440        }
 441}
 442