linux/arch/mips/kernel/mips-cm.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2013 Imagination Technologies
   3 * Author: Paul Burton <paul.burton@mips.com>
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms of the GNU General Public License as published by the
   7 * Free Software Foundation;  either version 2 of the  License, or (at your
   8 * option) any later version.
   9 */
  10
  11#include <linux/errno.h>
  12#include <linux/percpu.h>
  13#include <linux/spinlock.h>
  14
  15#include <asm/mips-cps.h>
  16#include <asm/mipsregs.h>
  17
  18void __iomem *mips_gcr_base;
  19void __iomem *mips_cm_l2sync_base;
  20int mips_cm_is64;
  21
  22static char *cm2_tr[8] = {
  23        "mem",  "gcr",  "gic",  "mmio",
  24        "0x04", "cpc", "0x06", "0x07"
  25};
  26
  27/* CM3 Tag ECC transaction type */
  28static char *cm3_tr[16] = {
  29        [0x0] = "ReqNoData",
  30        [0x1] = "0x1",
  31        [0x2] = "ReqWData",
  32        [0x3] = "0x3",
  33        [0x4] = "IReqNoResp",
  34        [0x5] = "IReqWResp",
  35        [0x6] = "IReqNoRespDat",
  36        [0x7] = "IReqWRespDat",
  37        [0x8] = "RespNoData",
  38        [0x9] = "RespDataFol",
  39        [0xa] = "RespWData",
  40        [0xb] = "RespDataOnly",
  41        [0xc] = "IRespNoData",
  42        [0xd] = "IRespDataFol",
  43        [0xe] = "IRespWData",
  44        [0xf] = "IRespDataOnly"
  45};
  46
  47static char *cm2_cmd[32] = {
  48        [0x00] = "0x00",
  49        [0x01] = "Legacy Write",
  50        [0x02] = "Legacy Read",
  51        [0x03] = "0x03",
  52        [0x04] = "0x04",
  53        [0x05] = "0x05",
  54        [0x06] = "0x06",
  55        [0x07] = "0x07",
  56        [0x08] = "Coherent Read Own",
  57        [0x09] = "Coherent Read Share",
  58        [0x0a] = "Coherent Read Discard",
  59        [0x0b] = "Coherent Ready Share Always",
  60        [0x0c] = "Coherent Upgrade",
  61        [0x0d] = "Coherent Writeback",
  62        [0x0e] = "0x0e",
  63        [0x0f] = "0x0f",
  64        [0x10] = "Coherent Copyback",
  65        [0x11] = "Coherent Copyback Invalidate",
  66        [0x12] = "Coherent Invalidate",
  67        [0x13] = "Coherent Write Invalidate",
  68        [0x14] = "Coherent Completion Sync",
  69        [0x15] = "0x15",
  70        [0x16] = "0x16",
  71        [0x17] = "0x17",
  72        [0x18] = "0x18",
  73        [0x19] = "0x19",
  74        [0x1a] = "0x1a",
  75        [0x1b] = "0x1b",
  76        [0x1c] = "0x1c",
  77        [0x1d] = "0x1d",
  78        [0x1e] = "0x1e",
  79        [0x1f] = "0x1f"
  80};
  81
  82/* CM3 Tag ECC command type */
  83static char *cm3_cmd[16] = {
  84        [0x0] = "Legacy Read",
  85        [0x1] = "Legacy Write",
  86        [0x2] = "Coherent Read Own",
  87        [0x3] = "Coherent Read Share",
  88        [0x4] = "Coherent Read Discard",
  89        [0x5] = "Coherent Evicted",
  90        [0x6] = "Coherent Upgrade",
  91        [0x7] = "Coherent Upgrade for Store Conditional",
  92        [0x8] = "Coherent Writeback",
  93        [0x9] = "Coherent Write Invalidate",
  94        [0xa] = "0xa",
  95        [0xb] = "0xb",
  96        [0xc] = "0xc",
  97        [0xd] = "0xd",
  98        [0xe] = "0xe",
  99        [0xf] = "0xf"
 100};
 101
 102/* CM3 Tag ECC command group */
 103static char *cm3_cmd_group[8] = {
 104        [0x0] = "Normal",
 105        [0x1] = "Registers",
 106        [0x2] = "TLB",
 107        [0x3] = "0x3",
 108        [0x4] = "L1I",
 109        [0x5] = "L1D",
 110        [0x6] = "L3",
 111        [0x7] = "L2"
 112};
 113
 114static char *cm2_core[8] = {
 115        "Invalid/OK",   "Invalid/Data",
 116        "Shared/OK",    "Shared/Data",
 117        "Modified/OK",  "Modified/Data",
 118        "Exclusive/OK", "Exclusive/Data"
 119};
 120
 121static char *cm2_causes[32] = {
 122        "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR",
 123        "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07",
 124        "0x08", "0x09", "0x0a", "0x0b",
 125        "0x0c", "0x0d", "0x0e", "0x0f",
 126        "0x10", "0x11", "0x12", "0x13",
 127        "0x14", "0x15", "0x16", "INTVN_WR_ERR",
 128        "INTVN_RD_ERR", "0x19", "0x1a", "0x1b",
 129        "0x1c", "0x1d", "0x1e", "0x1f"
 130};
 131
 132static char *cm3_causes[32] = {
 133        "0x0", "MP_CORRECTABLE_ECC_ERR", "MP_REQUEST_DECODE_ERR",
 134        "MP_UNCORRECTABLE_ECC_ERR", "MP_PARITY_ERR", "MP_COHERENCE_ERR",
 135        "CMBIU_REQUEST_DECODE_ERR", "CMBIU_PARITY_ERR", "CMBIU_AXI_RESP_ERR",
 136        "0x9", "RBI_BUS_ERR", "0xb", "0xc", "0xd", "0xe", "0xf", "0x10",
 137        "0x11", "0x12", "0x13", "0x14", "0x15", "0x16", "0x17", "0x18",
 138        "0x19", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f"
 139};
 140
 141static DEFINE_PER_CPU_ALIGNED(spinlock_t, cm_core_lock);
 142static DEFINE_PER_CPU_ALIGNED(unsigned long, cm_core_lock_flags);
 143
 144phys_addr_t __mips_cm_phys_base(void)
 145{
 146        u32 config3 = read_c0_config3();
 147        unsigned long cmgcr;
 148
 149        /* Check the CMGCRBase register is implemented */
 150        if (!(config3 & MIPS_CONF3_CMGCR))
 151                return 0;
 152
 153        /* Read the address from CMGCRBase */
 154        cmgcr = read_c0_cmgcrbase();
 155        return (cmgcr & MIPS_CMGCRF_BASE) << (36 - 32);
 156}
 157
 158phys_addr_t mips_cm_phys_base(void)
 159        __attribute__((weak, alias("__mips_cm_phys_base")));
 160
 161phys_addr_t __mips_cm_l2sync_phys_base(void)
 162{
 163        u32 base_reg;
 164
 165        /*
 166         * If the L2-only sync region is already enabled then leave it at it's
 167         * current location.
 168         */
 169        base_reg = read_gcr_l2_only_sync_base();
 170        if (base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN)
 171                return base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE;
 172
 173        /* Default to following the CM */
 174        return mips_cm_phys_base() + MIPS_CM_GCR_SIZE;
 175}
 176
 177phys_addr_t mips_cm_l2sync_phys_base(void)
 178        __attribute__((weak, alias("__mips_cm_l2sync_phys_base")));
 179
 180static void mips_cm_probe_l2sync(void)
 181{
 182        unsigned major_rev;
 183        phys_addr_t addr;
 184
 185        /* L2-only sync was introduced with CM major revision 6 */
 186        major_rev = (read_gcr_rev() & CM_GCR_REV_MAJOR) >>
 187                __ffs(CM_GCR_REV_MAJOR);
 188        if (major_rev < 6)
 189                return;
 190
 191        /* Find a location for the L2 sync region */
 192        addr = mips_cm_l2sync_phys_base();
 193        BUG_ON((addr & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE) != addr);
 194        if (!addr)
 195                return;
 196
 197        /* Set the region base address & enable it */
 198        write_gcr_l2_only_sync_base(addr | CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN);
 199
 200        /* Map the region */
 201        mips_cm_l2sync_base = ioremap_nocache(addr, MIPS_CM_L2SYNC_SIZE);
 202}
 203
 204int mips_cm_probe(void)
 205{
 206        phys_addr_t addr;
 207        u32 base_reg;
 208        unsigned cpu;
 209
 210        /*
 211         * No need to probe again if we have already been
 212         * here before.
 213         */
 214        if (mips_gcr_base)
 215                return 0;
 216
 217        addr = mips_cm_phys_base();
 218        BUG_ON((addr & CM_GCR_BASE_GCRBASE) != addr);
 219        if (!addr)
 220                return -ENODEV;
 221
 222        mips_gcr_base = ioremap_nocache(addr, MIPS_CM_GCR_SIZE);
 223        if (!mips_gcr_base)
 224                return -ENXIO;
 225
 226        /* sanity check that we're looking at a CM */
 227        base_reg = read_gcr_base();
 228        if ((base_reg & CM_GCR_BASE_GCRBASE) != addr) {
 229                pr_err("GCRs appear to have been moved (expected them at 0x%08lx)!\n",
 230                       (unsigned long)addr);
 231                mips_gcr_base = NULL;
 232                return -ENODEV;
 233        }
 234
 235        /* set default target to memory */
 236        change_gcr_base(CM_GCR_BASE_CMDEFTGT, CM_GCR_BASE_CMDEFTGT_MEM);
 237
 238        /* disable CM regions */
 239        write_gcr_reg0_base(CM_GCR_REGn_BASE_BASEADDR);
 240        write_gcr_reg0_mask(CM_GCR_REGn_MASK_ADDRMASK);
 241        write_gcr_reg1_base(CM_GCR_REGn_BASE_BASEADDR);
 242        write_gcr_reg1_mask(CM_GCR_REGn_MASK_ADDRMASK);
 243        write_gcr_reg2_base(CM_GCR_REGn_BASE_BASEADDR);
 244        write_gcr_reg2_mask(CM_GCR_REGn_MASK_ADDRMASK);
 245        write_gcr_reg3_base(CM_GCR_REGn_BASE_BASEADDR);
 246        write_gcr_reg3_mask(CM_GCR_REGn_MASK_ADDRMASK);
 247
 248        /* probe for an L2-only sync region */
 249        mips_cm_probe_l2sync();
 250
 251        /* determine register width for this CM */
 252        mips_cm_is64 = IS_ENABLED(CONFIG_64BIT) && (mips_cm_revision() >= CM_REV_CM3);
 253
 254        for_each_possible_cpu(cpu)
 255                spin_lock_init(&per_cpu(cm_core_lock, cpu));
 256
 257        return 0;
 258}
 259
 260void mips_cm_lock_other(unsigned int cluster, unsigned int core,
 261                        unsigned int vp, unsigned int block)
 262{
 263        unsigned int curr_core, cm_rev;
 264        u32 val;
 265
 266        cm_rev = mips_cm_revision();
 267        preempt_disable();
 268
 269        if (cm_rev >= CM_REV_CM3) {
 270                val = core << __ffs(CM3_GCR_Cx_OTHER_CORE);
 271                val |= vp << __ffs(CM3_GCR_Cx_OTHER_VP);
 272
 273                if (cm_rev >= CM_REV_CM3_5) {
 274                        val |= CM_GCR_Cx_OTHER_CLUSTER_EN;
 275                        val |= cluster << __ffs(CM_GCR_Cx_OTHER_CLUSTER);
 276                        val |= block << __ffs(CM_GCR_Cx_OTHER_BLOCK);
 277                } else {
 278                        WARN_ON(cluster != 0);
 279                        WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL);
 280                }
 281
 282                /*
 283                 * We need to disable interrupts in SMP systems in order to
 284                 * ensure that we don't interrupt the caller with code which
 285                 * may modify the redirect register. We do so here in a
 286                 * slightly obscure way by using a spin lock, since this has
 287                 * the neat property of also catching any nested uses of
 288                 * mips_cm_lock_other() leading to a deadlock or a nice warning
 289                 * with lockdep enabled.
 290                 */
 291                spin_lock_irqsave(this_cpu_ptr(&cm_core_lock),
 292                                  *this_cpu_ptr(&cm_core_lock_flags));
 293        } else {
 294                WARN_ON(cluster != 0);
 295                WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL);
 296
 297                /*
 298                 * We only have a GCR_CL_OTHER per core in systems with
 299                 * CM 2.5 & older, so have to ensure other VP(E)s don't
 300                 * race with us.
 301                 */
 302                curr_core = cpu_core(&current_cpu_data);
 303                spin_lock_irqsave(&per_cpu(cm_core_lock, curr_core),
 304                                  per_cpu(cm_core_lock_flags, curr_core));
 305
 306                val = core << __ffs(CM_GCR_Cx_OTHER_CORENUM);
 307        }
 308
 309        write_gcr_cl_other(val);
 310
 311        /*
 312         * Ensure the core-other region reflects the appropriate core &
 313         * VP before any accesses to it occur.
 314         */
 315        mb();
 316}
 317
 318void mips_cm_unlock_other(void)
 319{
 320        unsigned int curr_core;
 321
 322        if (mips_cm_revision() < CM_REV_CM3) {
 323                curr_core = cpu_core(&current_cpu_data);
 324                spin_unlock_irqrestore(&per_cpu(cm_core_lock, curr_core),
 325                                       per_cpu(cm_core_lock_flags, curr_core));
 326        } else {
 327                spin_unlock_irqrestore(this_cpu_ptr(&cm_core_lock),
 328                                       *this_cpu_ptr(&cm_core_lock_flags));
 329        }
 330
 331        preempt_enable();
 332}
 333
 334void mips_cm_error_report(void)
 335{
 336        u64 cm_error, cm_addr, cm_other;
 337        unsigned long revision;
 338        int ocause, cause;
 339        char buf[256];
 340
 341        if (!mips_cm_present())
 342                return;
 343
 344        revision = mips_cm_revision();
 345        cm_error = read_gcr_error_cause();
 346        cm_addr = read_gcr_error_addr();
 347        cm_other = read_gcr_error_mult();
 348
 349        if (revision < CM_REV_CM3) { /* CM2 */
 350                cause = cm_error >> __ffs(CM_GCR_ERROR_CAUSE_ERRTYPE);
 351                ocause = cm_other >> __ffs(CM_GCR_ERROR_MULT_ERR2ND);
 352
 353                if (!cause)
 354                        return;
 355
 356                if (cause < 16) {
 357                        unsigned long cca_bits = (cm_error >> 15) & 7;
 358                        unsigned long tr_bits = (cm_error >> 12) & 7;
 359                        unsigned long cmd_bits = (cm_error >> 7) & 0x1f;
 360                        unsigned long stag_bits = (cm_error >> 3) & 15;
 361                        unsigned long sport_bits = (cm_error >> 0) & 7;
 362
 363                        snprintf(buf, sizeof(buf),
 364                                 "CCA=%lu TR=%s MCmd=%s STag=%lu "
 365                                 "SPort=%lu\n", cca_bits, cm2_tr[tr_bits],
 366                                 cm2_cmd[cmd_bits], stag_bits, sport_bits);
 367                } else {
 368                        /* glob state & sresp together */
 369                        unsigned long c3_bits = (cm_error >> 18) & 7;
 370                        unsigned long c2_bits = (cm_error >> 15) & 7;
 371                        unsigned long c1_bits = (cm_error >> 12) & 7;
 372                        unsigned long c0_bits = (cm_error >> 9) & 7;
 373                        unsigned long sc_bit = (cm_error >> 8) & 1;
 374                        unsigned long cmd_bits = (cm_error >> 3) & 0x1f;
 375                        unsigned long sport_bits = (cm_error >> 0) & 7;
 376
 377                        snprintf(buf, sizeof(buf),
 378                                 "C3=%s C2=%s C1=%s C0=%s SC=%s "
 379                                 "MCmd=%s SPort=%lu\n",
 380                                 cm2_core[c3_bits], cm2_core[c2_bits],
 381                                 cm2_core[c1_bits], cm2_core[c0_bits],
 382                                 sc_bit ? "True" : "False",
 383                                 cm2_cmd[cmd_bits], sport_bits);
 384                }
 385                        pr_err("CM_ERROR=%08llx %s <%s>\n", cm_error,
 386                               cm2_causes[cause], buf);
 387                pr_err("CM_ADDR =%08llx\n", cm_addr);
 388                pr_err("CM_OTHER=%08llx %s\n", cm_other, cm2_causes[ocause]);
 389        } else { /* CM3 */
 390                ulong core_id_bits, vp_id_bits, cmd_bits, cmd_group_bits;
 391                ulong cm3_cca_bits, mcp_bits, cm3_tr_bits, sched_bit;
 392
 393                cause = cm_error >> __ffs64(CM3_GCR_ERROR_CAUSE_ERRTYPE);
 394                ocause = cm_other >> __ffs(CM_GCR_ERROR_MULT_ERR2ND);
 395
 396                if (!cause)
 397                        return;
 398
 399                /* Used by cause == {1,2,3} */
 400                core_id_bits = (cm_error >> 22) & 0xf;
 401                vp_id_bits = (cm_error >> 18) & 0xf;
 402                cmd_bits = (cm_error >> 14) & 0xf;
 403                cmd_group_bits = (cm_error >> 11) & 0xf;
 404                cm3_cca_bits = (cm_error >> 8) & 7;
 405                mcp_bits = (cm_error >> 5) & 0xf;
 406                cm3_tr_bits = (cm_error >> 1) & 0xf;
 407                sched_bit = cm_error & 0x1;
 408
 409                if (cause == 1 || cause == 3) { /* Tag ECC */
 410                        unsigned long tag_ecc = (cm_error >> 57) & 0x1;
 411                        unsigned long tag_way_bits = (cm_error >> 29) & 0xffff;
 412                        unsigned long dword_bits = (cm_error >> 49) & 0xff;
 413                        unsigned long data_way_bits = (cm_error >> 45) & 0xf;
 414                        unsigned long data_sets_bits = (cm_error >> 29) & 0xfff;
 415                        unsigned long bank_bit = (cm_error >> 28) & 0x1;
 416                        snprintf(buf, sizeof(buf),
 417                                 "%s ECC Error: Way=%lu (DWORD=%lu, Sets=%lu)"
 418                                 "Bank=%lu CoreID=%lu VPID=%lu Command=%s"
 419                                 "Command Group=%s CCA=%lu MCP=%d"
 420                                 "Transaction type=%s Scheduler=%lu\n",
 421                                 tag_ecc ? "TAG" : "DATA",
 422                                 tag_ecc ? (unsigned long)ffs(tag_way_bits) - 1 :
 423                                 data_way_bits, bank_bit, dword_bits,
 424                                 data_sets_bits,
 425                                 core_id_bits, vp_id_bits,
 426                                 cm3_cmd[cmd_bits],
 427                                 cm3_cmd_group[cmd_group_bits],
 428                                 cm3_cca_bits, 1 << mcp_bits,
 429                                 cm3_tr[cm3_tr_bits], sched_bit);
 430                } else if (cause == 2) {
 431                        unsigned long data_error_type = (cm_error >> 41) & 0xfff;
 432                        unsigned long data_decode_cmd = (cm_error >> 37) & 0xf;
 433                        unsigned long data_decode_group = (cm_error >> 34) & 0x7;
 434                        unsigned long data_decode_destination_id = (cm_error >> 28) & 0x3f;
 435
 436                        snprintf(buf, sizeof(buf),
 437                                 "Decode Request Error: Type=%lu, Command=%lu"
 438                                 "Command Group=%lu Destination ID=%lu"
 439                                 "CoreID=%lu VPID=%lu Command=%s"
 440                                 "Command Group=%s CCA=%lu MCP=%d"
 441                                 "Transaction type=%s Scheduler=%lu\n",
 442                                 data_error_type, data_decode_cmd,
 443                                 data_decode_group, data_decode_destination_id,
 444                                 core_id_bits, vp_id_bits,
 445                                 cm3_cmd[cmd_bits],
 446                                 cm3_cmd_group[cmd_group_bits],
 447                                 cm3_cca_bits, 1 << mcp_bits,
 448                                 cm3_tr[cm3_tr_bits], sched_bit);
 449                } else {
 450                        buf[0] = 0;
 451                }
 452
 453                pr_err("CM_ERROR=%llx %s <%s>\n", cm_error,
 454                       cm3_causes[cause], buf);
 455                pr_err("CM_ADDR =%llx\n", cm_addr);
 456                pr_err("CM_OTHER=%llx %s\n", cm_other, cm3_causes[ocause]);
 457        }
 458
 459        /* reprime cause register */
 460        write_gcr_error_cause(cm_error);
 461}
 462