LXR linux/arch/mips/kernel/perf_event

   1#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \
   2    defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1)
   3
   4#define M_CONFIG1_PC    (1 << 4)
   5
   6#define M_PERFCTL_EXL                   (1UL      <<  0)
   7#define M_PERFCTL_KERNEL                (1UL      <<  1)
   8#define M_PERFCTL_SUPERVISOR            (1UL      <<  2)
   9#define M_PERFCTL_USER                  (1UL      <<  3)
  10#define M_PERFCTL_INTERRUPT_ENABLE      (1UL      <<  4)
  11#define M_PERFCTL_EVENT(event)          (((event) & 0x3ff)  << 5)
  12#define M_PERFCTL_VPEID(vpe)            ((vpe)    << 16)
  13#define M_PERFCTL_MT_EN(filter)         ((filter) << 20)
  14#define    M_TC_EN_ALL                  M_PERFCTL_MT_EN(0)
  15#define    M_TC_EN_VPE                  M_PERFCTL_MT_EN(1)
  16#define    M_TC_EN_TC                   M_PERFCTL_MT_EN(2)
  17#define M_PERFCTL_TCID(tcid)            ((tcid)   << 22)
  18#define M_PERFCTL_WIDE                  (1UL      << 30)
  19#define M_PERFCTL_MORE                  (1UL      << 31)
  20
  21#define M_PERFCTL_COUNT_EVENT_WHENEVER  (M_PERFCTL_EXL |                \
  22                                        M_PERFCTL_KERNEL |              \
  23                                        M_PERFCTL_USER |                \
  24                                        M_PERFCTL_SUPERVISOR |          \
  25                                        M_PERFCTL_INTERRUPT_ENABLE)
  26
  27#ifdef CONFIG_MIPS_MT_SMP
  28#define M_PERFCTL_CONFIG_MASK           0x3fff801f
  29#else
  30#define M_PERFCTL_CONFIG_MASK           0x1f
  31#endif
  32#define M_PERFCTL_EVENT_MASK            0xfe0
  33
  34#define M_COUNTER_OVERFLOW              (1UL      << 31)
  35
  36#ifdef CONFIG_MIPS_MT_SMP
  37static int cpu_has_mipsmt_pertccounters;
  38
  39/*
  40 * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because
  41 * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs.
  42 */
  43#if defined(CONFIG_HW_PERF_EVENTS)
  44#define vpe_id()        (cpu_has_mipsmt_pertccounters ? \
  45                        0 : smp_processor_id())
  46#else
  47#define vpe_id()        (cpu_has_mipsmt_pertccounters ? \
  48                        0 : cpu_data[smp_processor_id()].vpe_id)
  49#endif
  50
  51/* Copied from op_model_mipsxx.c */
  52static inline unsigned int vpe_shift(void)
  53{
  54        if (num_possible_cpus() > 1)
  55                return 1;
  56
  57        return 0;
  58}
  59#else /* !CONFIG_MIPS_MT_SMP */
  60#define vpe_id()        0
  61
  62static inline unsigned int vpe_shift(void)
  63{
  64        return 0;
  65}
  66#endif /* CONFIG_MIPS_MT_SMP */
  67
  68static inline unsigned int
  69counters_total_to_per_cpu(unsigned int counters)
  70{
  71        return counters >> vpe_shift();
  72}
  73
  74static inline unsigned int
  75counters_per_cpu_to_total(unsigned int counters)
  76{
  77        return counters << vpe_shift();
  78}
  79
  80#define __define_perf_accessors(r, n, np)                               \
  81                                                                        \
  82static inline unsigned int r_c0_ ## r ## n(void)                        \
  83{                                                                       \
  84        unsigned int cpu = vpe_id();                                    \
  85                                                                        \
  86        switch (cpu) {                                                  \
  87        case 0:                                                         \
  88                return read_c0_ ## r ## n();                            \
  89        case 1:                                                         \
  90                return read_c0_ ## r ## np();                           \
  91        default:                                                        \
  92                BUG();                                                  \
  93        }                                                               \
  94        return 0;                                                       \
  95}                                                                       \
  96                                                                        \
  97static inline void w_c0_ ## r ## n(unsigned int value)                  \
  98{                                                                       \
  99        unsigned int cpu = vpe_id();                                    \
 100                                                                        \
 101        switch (cpu) {                                                  \
 102        case 0:                                                         \
 103                write_c0_ ## r ## n(value);                             \
 104                return;                                                 \
 105        case 1:                                                         \
 106                write_c0_ ## r ## np(value);                            \
 107                return;                                                 \
 108        default:                                                        \
 109                BUG();                                                  \
 110        }                                                               \
 111        return;                                                         \
 112}                                                                       \
 113
 114__define_perf_accessors(perfcntr, 0, 2)
 115__define_perf_accessors(perfcntr, 1, 3)
 116__define_perf_accessors(perfcntr, 2, 0)
 117__define_perf_accessors(perfcntr, 3, 1)
 118
 119__define_perf_accessors(perfctrl, 0, 2)
 120__define_perf_accessors(perfctrl, 1, 3)
 121__define_perf_accessors(perfctrl, 2, 0)
 122__define_perf_accessors(perfctrl, 3, 1)
 123
 124static inline int __n_counters(void)
 125{
 126        if (!(read_c0_config1() & M_CONFIG1_PC))
 127                return 0;
 128        if (!(read_c0_perfctrl0() & M_PERFCTL_MORE))
 129                return 1;
 130        if (!(read_c0_perfctrl1() & M_PERFCTL_MORE))
 131                return 2;
 132        if (!(read_c0_perfctrl2() & M_PERFCTL_MORE))
 133                return 3;
 134
 135        return 4;
 136}
 137
 138static inline int n_counters(void)
 139{
 140        int counters;
 141
 142        switch (current_cpu_type()) {
 143        case CPU_R10000:
 144                counters = 2;
 145                break;
 146
 147        case CPU_R12000:
 148        case CPU_R14000:
 149                counters = 4;
 150                break;
 151
 152        default:
 153                counters = __n_counters();
 154        }
 155
 156        return counters;
 157}
 158
 159static void reset_counters(void *arg)
 160{
 161        int counters = (int)(long)arg;
 162        switch (counters) {
 163        case 4:
 164                w_c0_perfctrl3(0);
 165                w_c0_perfcntr3(0);
 166        case 3:
 167                w_c0_perfctrl2(0);
 168                w_c0_perfcntr2(0);
 169        case 2:
 170                w_c0_perfctrl1(0);
 171                w_c0_perfcntr1(0);
 172        case 1:
 173                w_c0_perfctrl0(0);
 174                w_c0_perfcntr0(0);
 175        }
 176}
 177
 178static inline u64
 179mipsxx_pmu_read_counter(unsigned int idx)
 180{
 181        switch (idx) {
 182        case 0:
 183                return r_c0_perfcntr0();
 184        case 1:
 185                return r_c0_perfcntr1();
 186        case 2:
 187                return r_c0_perfcntr2();
 188        case 3:
 189                return r_c0_perfcntr3();
 190        default:
 191                WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 192                return 0;
 193        }
 194}
 195
 196static inline void
 197mipsxx_pmu_write_counter(unsigned int idx, u64 val)
 198{
 199        switch (idx) {
 200        case 0:
 201                w_c0_perfcntr0(val);
 202                return;
 203        case 1:
 204                w_c0_perfcntr1(val);
 205                return;
 206        case 2:
 207                w_c0_perfcntr2(val);
 208                return;
 209        case 3:
 210                w_c0_perfcntr3(val);
 211                return;
 212        }
 213}
 214
 215static inline unsigned int
 216mipsxx_pmu_read_control(unsigned int idx)
 217{
 218        switch (idx) {
 219        case 0:
 220                return r_c0_perfctrl0();
 221        case 1:
 222                return r_c0_perfctrl1();
 223        case 2:
 224                return r_c0_perfctrl2();
 225        case 3:
 226                return r_c0_perfctrl3();
 227        default:
 228                WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 229                return 0;
 230        }
 231}
 232
 233static inline void
 234mipsxx_pmu_write_control(unsigned int idx, unsigned int val)
 235{
 236        switch (idx) {
 237        case 0:
 238                w_c0_perfctrl0(val);
 239                return;
 240        case 1:
 241                w_c0_perfctrl1(val);
 242                return;
 243        case 2:
 244                w_c0_perfctrl2(val);
 245                return;
 246        case 3:
 247                w_c0_perfctrl3(val);
 248                return;
 249        }
 250}
 251
 252#ifdef CONFIG_MIPS_MT_SMP
 253static DEFINE_RWLOCK(pmuint_rwlock);
 254#endif
 255
 256/* 24K/34K/1004K cores can share the same event map. */
 257static const struct mips_perf_event mipsxxcore_event_map
 258                                [PERF_COUNT_HW_MAX] = {
 259        [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
 260        [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 261        [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID },
 262        [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID },
 263        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T },
 264        [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T },
 265        [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID },
 266};
 267
 268/* 74K core has different branch event code. */
 269static const struct mips_perf_event mipsxx74Kcore_event_map
 270                                [PERF_COUNT_HW_MAX] = {
 271        [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
 272        [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 273        [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID },
 274        [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID },
 275        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T },
 276        [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T },
 277        [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID },
 278};
 279
 280/* 24K/34K/1004K cores can share the same cache event map. */
 281static const struct mips_perf_event mipsxxcore_cache_map
 282                                [PERF_COUNT_HW_CACHE_MAX]
 283                                [PERF_COUNT_HW_CACHE_OP_MAX]
 284                                [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 285[C(L1D)] = {
 286        /*
 287         * Like some other architectures (e.g. ARM), the performance
 288         * counters don't differentiate between read and write
 289         * accesses/misses, so this isn't strictly correct, but it's the
 290         * best we can do. Writes and reads get combined.
 291         */
 292        [C(OP_READ)] = {
 293                [C(RESULT_ACCESS)]      = { 0x0a, CNTR_EVEN, T },
 294                [C(RESULT_MISS)]        = { 0x0b, CNTR_EVEN | CNTR_ODD, T },
 295        },
 296        [C(OP_WRITE)] = {
 297                [C(RESULT_ACCESS)]      = { 0x0a, CNTR_EVEN, T },
 298                [C(RESULT_MISS)]        = { 0x0b, CNTR_EVEN | CNTR_ODD, T },
 299        },
 300        [C(OP_PREFETCH)] = {
 301                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 302                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 303        },
 304},
 305[C(L1I)] = {
 306        [C(OP_READ)] = {
 307                [C(RESULT_ACCESS)]      = { 0x09, CNTR_EVEN, T },
 308                [C(RESULT_MISS)]        = { 0x09, CNTR_ODD, T },
 309        },
 310        [C(OP_WRITE)] = {
 311                [C(RESULT_ACCESS)]      = { 0x09, CNTR_EVEN, T },
 312                [C(RESULT_MISS)]        = { 0x09, CNTR_ODD, T },
 313        },
 314        [C(OP_PREFETCH)] = {
 315                [C(RESULT_ACCESS)]      = { 0x14, CNTR_EVEN, T },
 316                /*
 317                 * Note that MIPS has only "hit" events countable for
 318                 * the prefetch operation.
 319                 */
 320                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 321        },
 322},
 323[C(LL)] = {
 324        [C(OP_READ)] = {
 325                [C(RESULT_ACCESS)]      = { 0x15, CNTR_ODD, P },
 326                [C(RESULT_MISS)]        = { 0x16, CNTR_EVEN, P },
 327        },
 328        [C(OP_WRITE)] = {
 329                [C(RESULT_ACCESS)]      = { 0x15, CNTR_ODD, P },
 330                [C(RESULT_MISS)]        = { 0x16, CNTR_EVEN, P },
 331        },
 332        [C(OP_PREFETCH)] = {
 333                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 334                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 335        },
 336},
 337[C(DTLB)] = {
 338        [C(OP_READ)] = {
 339                [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 340                [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 341        },
 342        [C(OP_WRITE)] = {
 343                [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 344                [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 345        },
 346        [C(OP_PREFETCH)] = {
 347                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 348                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 349        },
 350},
 351[C(ITLB)] = {
 352        [C(OP_READ)] = {
 353                [C(RESULT_ACCESS)]      = { 0x05, CNTR_EVEN, T },
 354                [C(RESULT_MISS)]        = { 0x05, CNTR_ODD, T },
 355        },
 356        [C(OP_WRITE)] = {
 357                [C(RESULT_ACCESS)]      = { 0x05, CNTR_EVEN, T },
 358                [C(RESULT_MISS)]        = { 0x05, CNTR_ODD, T },
 359        },
 360        [C(OP_PREFETCH)] = {
 361                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 362                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 363        },
 364},
 365[C(BPU)] = {
 366        /* Using the same code for *HW_BRANCH* */
 367        [C(OP_READ)] = {
 368                [C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN, T },
 369                [C(RESULT_MISS)]        = { 0x02, CNTR_ODD, T },
 370        },
 371        [C(OP_WRITE)] = {
 372                [C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN, T },
 373                [C(RESULT_MISS)]        = { 0x02, CNTR_ODD, T },
 374        },
 375        [C(OP_PREFETCH)] = {
 376                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 377                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 378        },
 379},
 380};
 381
 382/* 74K core has completely different cache event map. */
 383static const struct mips_perf_event mipsxx74Kcore_cache_map
 384                                [PERF_COUNT_HW_CACHE_MAX]
 385                                [PERF_COUNT_HW_CACHE_OP_MAX]
 386                                [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 387[C(L1D)] = {
 388        /*
 389         * Like some other architectures (e.g. ARM), the performance
 390         * counters don't differentiate between read and write
 391         * accesses/misses, so this isn't strictly correct, but it's the
 392         * best we can do. Writes and reads get combined.
 393         */
 394        [C(OP_READ)] = {
 395                [C(RESULT_ACCESS)]      = { 0x17, CNTR_ODD, T },
 396                [C(RESULT_MISS)]        = { 0x18, CNTR_ODD, T },
 397        },
 398        [C(OP_WRITE)] = {
 399                [C(RESULT_ACCESS)]      = { 0x17, CNTR_ODD, T },
 400                [C(RESULT_MISS)]        = { 0x18, CNTR_ODD, T },
 401        },
 402        [C(OP_PREFETCH)] = {
 403                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 404                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 405        },
 406},
 407[C(L1I)] = {
 408        [C(OP_READ)] = {
 409                [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 410                [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 411        },
 412        [C(OP_WRITE)] = {
 413                [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 414                [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 415        },
 416        [C(OP_PREFETCH)] = {
 417                [C(RESULT_ACCESS)]      = { 0x34, CNTR_EVEN, T },
 418                /*
 419                 * Note that MIPS has only "hit" events countable for
 420                 * the prefetch operation.
 421                 */
 422                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 423        },
 424},
 425[C(LL)] = {
 426        [C(OP_READ)] = {
 427                [C(RESULT_ACCESS)]      = { 0x1c, CNTR_ODD, P },
 428                [C(RESULT_MISS)]        = { 0x1d, CNTR_EVEN | CNTR_ODD, P },
 429        },
 430        [C(OP_WRITE)] = {
 431                [C(RESULT_ACCESS)]      = { 0x1c, CNTR_ODD, P },
 432                [C(RESULT_MISS)]        = { 0x1d, CNTR_EVEN | CNTR_ODD, P },
 433        },
 434        [C(OP_PREFETCH)] = {
 435                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 436                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 437        },
 438},
 439[C(DTLB)] = {
 440        /* 74K core does not have specific DTLB events. */
 441        [C(OP_READ)] = {
 442                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 443                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 444        },
 445        [C(OP_WRITE)] = {
 446                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 447                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 448        },
 449        [C(OP_PREFETCH)] = {
 450                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 451                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 452        },
 453},
 454[C(ITLB)] = {
 455        [C(OP_READ)] = {
 456                [C(RESULT_ACCESS)]      = { 0x04, CNTR_EVEN, T },
 457                [C(RESULT_MISS)]        = { 0x04, CNTR_ODD, T },
 458        },
 459        [C(OP_WRITE)] = {
 460                [C(RESULT_ACCESS)]      = { 0x04, CNTR_EVEN, T },
 461                [C(RESULT_MISS)]        = { 0x04, CNTR_ODD, T },
 462        },
 463        [C(OP_PREFETCH)] = {
 464                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 465                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 466        },
 467},
 468[C(BPU)] = {
 469        /* Using the same code for *HW_BRANCH* */
 470        [C(OP_READ)] = {
 471                [C(RESULT_ACCESS)]      = { 0x27, CNTR_EVEN, T },
 472                [C(RESULT_MISS)]        = { 0x27, CNTR_ODD, T },
 473        },
 474        [C(OP_WRITE)] = {
 475                [C(RESULT_ACCESS)]      = { 0x27, CNTR_EVEN, T },
 476                [C(RESULT_MISS)]        = { 0x27, CNTR_ODD, T },
 477        },
 478        [C(OP_PREFETCH)] = {
 479                [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 480                [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 481        },
 482},
 483};
 484
 485#ifdef CONFIG_MIPS_MT_SMP
 486static void
 487check_and_calc_range(struct perf_event *event,
 488                        const struct mips_perf_event *pev)
 489{
 490        struct hw_perf_event *hwc = &event->hw;
 491
 492        if (event->cpu >= 0) {
 493                if (pev->range > V) {
 494                        /*
 495                         * The user selected an event that is processor
 496                         * wide, while expecting it to be VPE wide.
 497                         */
 498                        hwc->config_base |= M_TC_EN_ALL;
 499                } else {
 500                        /*
 501                         * FIXME: cpu_data[event->cpu].vpe_id reports 0
 502                         * for both CPUs.
 503                         */
 504                        hwc->config_base |= M_PERFCTL_VPEID(event->cpu);
 505                        hwc->config_base |= M_TC_EN_VPE;
 506                }
 507        } else
 508                hwc->config_base |= M_TC_EN_ALL;
 509}
 510#else
 511static void
 512check_and_calc_range(struct perf_event *event,
 513                        const struct mips_perf_event *pev)
 514{
 515}
 516#endif
 517
 518static int __hw_perf_event_init(struct perf_event *event)
 519{
 520        struct perf_event_attr *attr = &event->attr;
 521        struct hw_perf_event *hwc = &event->hw;
 522        const struct mips_perf_event *pev;
 523        int err;
 524
 525        /* Returning MIPS event descriptor for generic perf event. */
 526        if (PERF_TYPE_HARDWARE == event->attr.type) {
 527                if (event->attr.config >= PERF_COUNT_HW_MAX)
 528                        return -EINVAL;
 529                pev = mipspmu_map_general_event(event->attr.config);
 530        } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
 531                pev = mipspmu_map_cache_event(event->attr.config);
 532        } else if (PERF_TYPE_RAW == event->attr.type) {
 533                /* We are working on the global raw event. */
 534                mutex_lock(&raw_event_mutex);
 535                pev = mipspmu->map_raw_event(event->attr.config);
 536        } else {
 537                /* The event type is not (yet) supported. */
 538                return -EOPNOTSUPP;
 539        }
 540
 541        if (IS_ERR(pev)) {
 542                if (PERF_TYPE_RAW == event->attr.type)
 543                        mutex_unlock(&raw_event_mutex);
 544                return PTR_ERR(pev);
 545        }
 546
 547        /*
 548         * We allow max flexibility on how each individual counter shared
 549         * by the single CPU operates (the mode exclusion and the range).
 550         */
 551        hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE;
 552
 553        /* Calculate range bits and validate it. */
 554        if (num_possible_cpus() > 1)
 555                check_and_calc_range(event, pev);
 556
 557        hwc->event_base = mipspmu_perf_event_encode(pev);
 558        if (PERF_TYPE_RAW == event->attr.type)
 559                mutex_unlock(&raw_event_mutex);
 560
 561        if (!attr->exclude_user)
 562                hwc->config_base |= M_PERFCTL_USER;
 563        if (!attr->exclude_kernel) {
 564                hwc->config_base |= M_PERFCTL_KERNEL;
 565                /* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */
 566                hwc->config_base |= M_PERFCTL_EXL;
 567        }
 568        if (!attr->exclude_hv)
 569                hwc->config_base |= M_PERFCTL_SUPERVISOR;
 570
 571        hwc->config_base &= M_PERFCTL_CONFIG_MASK;
 572        /*
 573         * The event can belong to another cpu. We do not assign a local
 574         * counter for it for now.
 575         */
 576        hwc->idx = -1;
 577        hwc->config = 0;
 578
 579        if (!hwc->sample_period) {
 580                hwc->sample_period  = MAX_PERIOD;
 581                hwc->last_period    = hwc->sample_period;
 582                local64_set(&hwc->period_left, hwc->sample_period);
 583        }
 584
 585        err = 0;
 586        if (event->group_leader != event) {
 587                err = validate_group(event);
 588                if (err)
 589                        return -EINVAL;
 590        }
 591
 592        event->destroy = hw_perf_event_destroy;
 593
 594        return err;
 595}
 596
 597static void pause_local_counters(void)
 598{
 599        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 600        int counters = mipspmu->num_counters;
 601        unsigned long flags;
 602
 603        local_irq_save(flags);
 604        switch (counters) {
 605        case 4:
 606                cpuc->saved_ctrl[3] = r_c0_perfctrl3();
 607                w_c0_perfctrl3(cpuc->saved_ctrl[3] &
 608                        ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 609        case 3:
 610                cpuc->saved_ctrl[2] = r_c0_perfctrl2();
 611                w_c0_perfctrl2(cpuc->saved_ctrl[2] &
 612                        ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 613        case 2:
 614                cpuc->saved_ctrl[1] = r_c0_perfctrl1();
 615                w_c0_perfctrl1(cpuc->saved_ctrl[1] &
 616                        ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 617        case 1:
 618                cpuc->saved_ctrl[0] = r_c0_perfctrl0();
 619                w_c0_perfctrl0(cpuc->saved_ctrl[0] &
 620                        ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 621        }
 622        local_irq_restore(flags);
 623}
 624
 625static void resume_local_counters(void)
 626{
 627        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 628        int counters = mipspmu->num_counters;
 629        unsigned long flags;
 630
 631        local_irq_save(flags);
 632        switch (counters) {
 633        case 4:
 634                w_c0_perfctrl3(cpuc->saved_ctrl[3]);
 635        case 3:
 636                w_c0_perfctrl2(cpuc->saved_ctrl[2]);
 637        case 2:
 638                w_c0_perfctrl1(cpuc->saved_ctrl[1]);
 639        case 1:
 640                w_c0_perfctrl0(cpuc->saved_ctrl[0]);
 641        }
 642        local_irq_restore(flags);
 643}
 644
 645static int mipsxx_pmu_handle_shared_irq(void)
 646{
 647        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 648        struct perf_sample_data data;
 649        unsigned int counters = mipspmu->num_counters;
 650        unsigned int counter;
 651        int handled = IRQ_NONE;
 652        struct pt_regs *regs;
 653
 654        if (cpu_has_mips_r2 && !(read_c0_cause() & (1 << 26)))
 655                return handled;
 656
 657        /*
 658         * First we pause the local counters, so that when we are locked
 659         * here, the counters are all paused. When it gets locked due to
 660         * perf_disable(), the timer interrupt handler will be delayed.
 661         *
 662         * See also mipsxx_pmu_start().
 663         */
 664        pause_local_counters();
 665#ifdef CONFIG_MIPS_MT_SMP
 666        read_lock(&pmuint_rwlock);
 667#endif
 668
 669        regs = get_irq_regs();
 670
 671        perf_sample_data_init(&data, 0);
 672
 673        switch (counters) {
 674#define HANDLE_COUNTER(n)                                               \
 675        case n + 1:                                                     \
 676                if (test_bit(n, cpuc->used_mask)) {                     \
 677                        counter = r_c0_perfcntr ## n();                 \
 678                        if (counter & M_COUNTER_OVERFLOW) {             \
 679                                w_c0_perfcntr ## n(counter &            \
 680                                                VALID_COUNT);           \
 681                                if (test_and_change_bit(n, cpuc->msbs)) \
 682                                        handle_associated_event(cpuc,   \
 683                                                n, &data, regs);        \
 684                                handled = IRQ_HANDLED;                  \
 685                        }                                               \
 686                }
 687        HANDLE_COUNTER(3)
 688        HANDLE_COUNTER(2)
 689        HANDLE_COUNTER(1)
 690        HANDLE_COUNTER(0)
 691        }
 692
 693        /*
 694         * Do all the work for the pending perf events. We can do this
 695         * in here because the performance counter interrupt is a regular
 696         * interrupt, not NMI.
 697         */
 698        if (handled == IRQ_HANDLED)
 699                irq_work_run();
 700
 701#ifdef CONFIG_MIPS_MT_SMP
 702        read_unlock(&pmuint_rwlock);
 703#endif
 704        resume_local_counters();
 705        return handled;
 706}
 707
 708static irqreturn_t
 709mipsxx_pmu_handle_irq(int irq, void *dev)
 710{
 711        return mipsxx_pmu_handle_shared_irq();
 712}
 713
 714static void mipsxx_pmu_start(void)
 715{
 716#ifdef CONFIG_MIPS_MT_SMP
 717        write_unlock(&pmuint_rwlock);
 718#endif
 719        resume_local_counters();
 720}
 721
 722/*
 723 * MIPS performance counters can be per-TC. The control registers can
 724 * not be directly accessed across CPUs. Hence if we want to do global
 725 * control, we need cross CPU calls. on_each_cpu() can help us, but we
 726 * can not make sure this function is called with interrupts enabled. So
 727 * here we pause local counters and then grab a rwlock and leave the
 728 * counters on other CPUs alone. If any counter interrupt raises while
 729 * we own the write lock, simply pause local counters on that CPU and
 730 * spin in the handler. Also we know we won't be switched to another
 731 * CPU after pausing local counters and before grabbing the lock.
 732 */
 733static void mipsxx_pmu_stop(void)
 734{
 735        pause_local_counters();
 736#ifdef CONFIG_MIPS_MT_SMP
 737        write_lock(&pmuint_rwlock);
 738#endif
 739}
 740
 741static int
 742mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc,
 743                        struct hw_perf_event *hwc)
 744{
 745        int i;
 746
 747        /*
 748         * We only need to care the counter mask. The range has been
 749         * checked definitely.
 750         */
 751        unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff;
 752
 753        for (i = mipspmu->num_counters - 1; i >= 0; i--) {
 754                /*
 755                 * Note that some MIPS perf events can be counted by both
 756                 * even and odd counters, wheresas many other are only by
 757                 * even _or_ odd counters. This introduces an issue that
 758                 * when the former kind of event takes the counter the
 759                 * latter kind of event wants to use, then the "counter
 760                 * allocation" for the latter event will fail. In fact if
 761                 * they can be dynamically swapped, they both feel happy.
 762                 * But here we leave this issue alone for now.
 763                 */
 764                if (test_bit(i, &cntr_mask) &&
 765                        !test_and_set_bit(i, cpuc->used_mask))
 766                        return i;
 767        }
 768
 769        return -EAGAIN;
 770}
 771
 772static void
 773mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx)
 774{
 775        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 776        unsigned long flags;
 777
 778        WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
 779
 780        local_irq_save(flags);
 781        cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |
 782                (evt->config_base & M_PERFCTL_CONFIG_MASK) |
 783                /* Make sure interrupt enabled. */
 784                M_PERFCTL_INTERRUPT_ENABLE;
 785        /*
 786         * We do not actually let the counter run. Leave it until start().
 787         */
 788        local_irq_restore(flags);
 789}
 790
 791static void
 792mipsxx_pmu_disable_event(int idx)
 793{
 794        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 795        unsigned long flags;
 796
 797        WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
 798
 799        local_irq_save(flags);
 800        cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) &
 801                ~M_PERFCTL_COUNT_EVENT_WHENEVER;
 802        mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]);
 803        local_irq_restore(flags);
 804}
 805
 806/* 24K */
 807#define IS_UNSUPPORTED_24K_EVENT(r, b)                                  \
 808        ((b) == 12 || (r) == 151 || (r) == 152 || (b) == 26 ||          \
 809         (b) == 27 || (r) == 28 || (r) == 158 || (b) == 31 ||           \
 810         (b) == 32 || (b) == 34 || (b) == 36 || (r) == 168 ||           \
 811         (r) == 172 || (b) == 47 || ((b) >= 56 && (b) <= 63) ||         \
 812         ((b) >= 68 && (b) <= 127))
 813#define IS_BOTH_COUNTERS_24K_EVENT(b)                                   \
 814        ((b) == 0 || (b) == 1 || (b) == 11)
 815
 816/* 34K */
 817#define IS_UNSUPPORTED_34K_EVENT(r, b)                                  \
 818        ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 36 ||           \
 819         (b) == 38 || (r) == 175 || ((b) >= 56 && (b) <= 63) ||         \
 820         ((b) >= 68 && (b) <= 127))
 821#define IS_BOTH_COUNTERS_34K_EVENT(b)                                   \
 822        ((b) == 0 || (b) == 1 || (b) == 11)
 823#ifdef CONFIG_MIPS_MT_SMP
 824#define IS_RANGE_P_34K_EVENT(r, b)                                      \
 825        ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||             \
 826         (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 ||           \
 827         (r) == 176 || ((b) >= 50 && (b) <= 55) ||                      \
 828         ((b) >= 64 && (b) <= 67))
 829#define IS_RANGE_V_34K_EVENT(r) ((r) == 47)
 830#endif
 831
 832/* 74K */
 833#define IS_UNSUPPORTED_74K_EVENT(r, b)                                  \
 834        ((r) == 5 || ((r) >= 135 && (r) <= 137) ||                      \
 835         ((b) >= 10 && (b) <= 12) || (b) == 22 || (b) == 27 ||          \
 836         (b) == 33 || (b) == 34 || ((b) >= 47 && (b) <= 49) ||          \
 837         (r) == 178 || (b) == 55 || (b) == 57 || (b) == 60 ||           \
 838         (b) == 61 || (r) == 62 || (r) == 191 ||                        \
 839         ((b) >= 64 && (b) <= 127))
 840#define IS_BOTH_COUNTERS_74K_EVENT(b)                                   \
 841        ((b) == 0 || (b) == 1)
 842
 843/* 1004K */
 844#define IS_UNSUPPORTED_1004K_EVENT(r, b)                                \
 845        ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 38 ||           \
 846         (r) == 175 || (b) == 63 || ((b) >= 68 && (b) <= 127))
 847#define IS_BOTH_COUNTERS_1004K_EVENT(b)                                 \
 848        ((b) == 0 || (b) == 1 || (b) == 11)
 849#ifdef CONFIG_MIPS_MT_SMP
 850#define IS_RANGE_P_1004K_EVENT(r, b)                                    \
 851        ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||             \
 852         (b) == 25 || (b) == 36 || (b) == 39 || (r) == 44 ||            \
 853         (r) == 174 || (r) == 176 || ((b) >= 50 && (b) <= 59) ||        \
 854         (r) == 188 || (b) == 61 || (b) == 62 ||                        \
 855         ((b) >= 64 && (b) <= 67))
 856#define IS_RANGE_V_1004K_EVENT(r)       ((r) == 47)
 857#endif
 858
 859/*
 860 * User can use 0-255 raw events, where 0-127 for the events of even
 861 * counters, and 128-255 for odd counters. Note that bit 7 is used to
 862 * indicate the parity. So, for example, when user wants to take the
 863 * Event Num of 15 for odd counters (by referring to the user manual),
 864 * then 128 needs to be added to 15 as the input for the event config,
 865 * i.e., 143 (0x8F) to be used.
 866 */
 867static const struct mips_perf_event *
 868mipsxx_pmu_map_raw_event(u64 config)
 869{
 870        unsigned int raw_id = config & 0xff;
 871        unsigned int base_id = raw_id & 0x7f;
 872
 873        switch (current_cpu_type()) {
 874        case CPU_24K:
 875                if (IS_UNSUPPORTED_24K_EVENT(raw_id, base_id))
 876                        return ERR_PTR(-EOPNOTSUPP);
 877                raw_event.event_id = base_id;
 878                if (IS_BOTH_COUNTERS_24K_EVENT(base_id))
 879                        raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 880                else
 881                        raw_event.cntr_mask =
 882                                raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 883#ifdef CONFIG_MIPS_MT_SMP
 884                /*
 885                 * This is actually doing nothing. Non-multithreading
 886                 * CPUs will not check and calculate the range.
 887                 */
 888                raw_event.range = P;
 889#endif
 890                break;
 891        case CPU_34K:
 892                if (IS_UNSUPPORTED_34K_EVENT(raw_id, base_id))
 893                        return ERR_PTR(-EOPNOTSUPP);
 894                raw_event.event_id = base_id;
 895                if (IS_BOTH_COUNTERS_34K_EVENT(base_id))
 896                        raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 897                else
 898                        raw_event.cntr_mask =
 899                                raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 900#ifdef CONFIG_MIPS_MT_SMP
 901                if (IS_RANGE_P_34K_EVENT(raw_id, base_id))
 902                        raw_event.range = P;
 903                else if (unlikely(IS_RANGE_V_34K_EVENT(raw_id)))
 904                        raw_event.range = V;
 905                else
 906                        raw_event.range = T;
 907#endif
 908                break;
 909        case CPU_74K:
 910                if (IS_UNSUPPORTED_74K_EVENT(raw_id, base_id))
 911                        return ERR_PTR(-EOPNOTSUPP);
 912                raw_event.event_id = base_id;
 913                if (IS_BOTH_COUNTERS_74K_EVENT(base_id))
 914                        raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 915                else
 916                        raw_event.cntr_mask =
 917                                raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 918#ifdef CONFIG_MIPS_MT_SMP
 919                raw_event.range = P;
 920#endif
 921                break;
 922        case CPU_1004K:
 923                if (IS_UNSUPPORTED_1004K_EVENT(raw_id, base_id))
 924                        return ERR_PTR(-EOPNOTSUPP);
 925                raw_event.event_id = base_id;
 926                if (IS_BOTH_COUNTERS_1004K_EVENT(base_id))
 927                        raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 928                else
 929                        raw_event.cntr_mask =
 930                                raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 931#ifdef CONFIG_MIPS_MT_SMP
 932                if (IS_RANGE_P_1004K_EVENT(raw_id, base_id))
 933                        raw_event.range = P;
 934                else if (unlikely(IS_RANGE_V_1004K_EVENT(raw_id)))
 935                        raw_event.range = V;
 936                else
 937                        raw_event.range = T;
 938#endif
 939                break;
 940        }
 941
 942        return &raw_event;
 943}
 944
 945static struct mips_pmu mipsxxcore_pmu = {
 946        .handle_irq = mipsxx_pmu_handle_irq,
 947        .handle_shared_irq = mipsxx_pmu_handle_shared_irq,
 948        .start = mipsxx_pmu_start,
 949        .stop = mipsxx_pmu_stop,
 950        .alloc_counter = mipsxx_pmu_alloc_counter,
 951        .read_counter = mipsxx_pmu_read_counter,
 952        .write_counter = mipsxx_pmu_write_counter,
 953        .enable_event = mipsxx_pmu_enable_event,
 954        .disable_event = mipsxx_pmu_disable_event,
 955        .map_raw_event = mipsxx_pmu_map_raw_event,
 956        .general_event_map = &mipsxxcore_event_map,
 957        .cache_event_map = &mipsxxcore_cache_map,
 958};
 959
 960static struct mips_pmu mipsxx74Kcore_pmu = {
 961        .handle_irq = mipsxx_pmu_handle_irq,
 962        .handle_shared_irq = mipsxx_pmu_handle_shared_irq,
 963        .start = mipsxx_pmu_start,
 964        .stop = mipsxx_pmu_stop,
 965        .alloc_counter = mipsxx_pmu_alloc_counter,
 966        .read_counter = mipsxx_pmu_read_counter,
 967        .write_counter = mipsxx_pmu_write_counter,
 968        .enable_event = mipsxx_pmu_enable_event,
 969        .disable_event = mipsxx_pmu_disable_event,
 970        .map_raw_event = mipsxx_pmu_map_raw_event,
 971        .general_event_map = &mipsxx74Kcore_event_map,
 972        .cache_event_map = &mipsxx74Kcore_cache_map,
 973};
 974
 975static int __init
 976init_hw_perf_events(void)
 977{
 978        int counters, irq;
 979
 980        pr_info("Performance counters: ");
 981
 982        counters = n_counters();
 983        if (counters == 0) {
 984                pr_cont("No available PMU.\n");
 985                return -ENODEV;
 986        }
 987
 988#ifdef CONFIG_MIPS_MT_SMP
 989        cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19);
 990        if (!cpu_has_mipsmt_pertccounters)
 991                counters = counters_total_to_per_cpu(counters);
 992#endif
 993
 994#ifdef MSC01E_INT_BASE
 995        if (cpu_has_veic) {
 996                /*
 997                 * Using platform specific interrupt controller defines.
 998                 */
 999                irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR;
1000        } else {

1001#endif
1002                if (cp0_perfcount_irq >= 0)
1003                        irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
1004                else
1005                        irq = -1;
1006#ifdef MSC01E_INT_BASE
1007        }
1008#endif
1009
1010        on_each_cpu(reset_counters, (void *)(long)counters, 1);
1011
1012        switch (current_cpu_type()) {
1013        case CPU_24K:
1014                mipsxxcore_pmu.name = "mips/24K";
1015                mipsxxcore_pmu.num_counters = counters;
1016                mipsxxcore_pmu.irq = irq;
1017                mipspmu = &mipsxxcore_pmu;
1018                break;
1019        case CPU_34K:
1020                mipsxxcore_pmu.name = "mips/34K";
1021                mipsxxcore_pmu.num_counters = counters;
1022                mipsxxcore_pmu.irq = irq;
1023                mipspmu = &mipsxxcore_pmu;
1024                break;
1025        case CPU_74K:
1026                mipsxx74Kcore_pmu.name = "mips/74K";
1027                mipsxx74Kcore_pmu.num_counters = counters;
1028                mipsxx74Kcore_pmu.irq = irq;
1029                mipspmu = &mipsxx74Kcore_pmu;
1030                break;
1031        case CPU_1004K:
1032                mipsxxcore_pmu.name = "mips/1004K";
1033                mipsxxcore_pmu.num_counters = counters;
1034                mipsxxcore_pmu.irq = irq;
1035                mipspmu = &mipsxxcore_pmu;
1036                break;
1037        default:
1038                pr_cont("Either hardware does not support performance "
1039                        "counters, or not yet implemented.\n");
1040                return -ENODEV;
1041        }
1042
1043        if (mipspmu)
1044                pr_cont("%s PMU enabled, %d counters available to each "
1045                        "CPU, irq %d%s\n", mipspmu->name, counters, irq,
1046                        irq < 0 ? " (share with timer interrupt)" : "");
1047
1048        perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1049
1050        return 0;
1051}
1052early_initcall(init_hw_perf_events);
1053
1054#endif /* defined(CONFIG_CPU_MIPS32)... */
1055