linux/virt/kvm/arm/pmu.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2015 Linaro Ltd.
   4 * Author: Shannon Zhao <shannon.zhao@linaro.org>
   5 */
   6
   7#include <linux/cpu.h>
   8#include <linux/kvm.h>
   9#include <linux/kvm_host.h>
  10#include <linux/perf_event.h>
  11#include <linux/perf/arm_pmu.h>
  12#include <linux/uaccess.h>
  13#include <asm/kvm_emulate.h>
  14#include <kvm/arm_pmu.h>
  15#include <kvm/arm_vgic.h>
  16
  17static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx);
  18
  19#define PERF_ATTR_CFG1_KVM_PMU_CHAINED 0x1
  20
  21/**
  22 * kvm_pmu_idx_is_64bit - determine if select_idx is a 64bit counter
  23 * @vcpu: The vcpu pointer
  24 * @select_idx: The counter index
  25 */
  26static bool kvm_pmu_idx_is_64bit(struct kvm_vcpu *vcpu, u64 select_idx)
  27{
  28        return (select_idx == ARMV8_PMU_CYCLE_IDX &&
  29                __vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_LC);
  30}
  31
  32static struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
  33{
  34        struct kvm_pmu *pmu;
  35        struct kvm_vcpu_arch *vcpu_arch;
  36
  37        pmc -= pmc->idx;
  38        pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
  39        vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
  40        return container_of(vcpu_arch, struct kvm_vcpu, arch);
  41}
  42
  43/**
  44 * kvm_pmu_pmc_is_chained - determine if the pmc is chained
  45 * @pmc: The PMU counter pointer
  46 */
  47static bool kvm_pmu_pmc_is_chained(struct kvm_pmc *pmc)
  48{
  49        struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
  50
  51        return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
  52}
  53
  54/**
  55 * kvm_pmu_idx_is_high_counter - determine if select_idx is a high/low counter
  56 * @select_idx: The counter index
  57 */
  58static bool kvm_pmu_idx_is_high_counter(u64 select_idx)
  59{
  60        return select_idx & 0x1;
  61}
  62
  63/**
  64 * kvm_pmu_get_canonical_pmc - obtain the canonical pmc
  65 * @pmc: The PMU counter pointer
  66 *
  67 * When a pair of PMCs are chained together we use the low counter (canonical)
  68 * to hold the underlying perf event.
  69 */
  70static struct kvm_pmc *kvm_pmu_get_canonical_pmc(struct kvm_pmc *pmc)
  71{
  72        if (kvm_pmu_pmc_is_chained(pmc) &&
  73            kvm_pmu_idx_is_high_counter(pmc->idx))
  74                return pmc - 1;
  75
  76        return pmc;
  77}
  78
  79/**
  80 * kvm_pmu_idx_has_chain_evtype - determine if the event type is chain
  81 * @vcpu: The vcpu pointer
  82 * @select_idx: The counter index
  83 */
  84static bool kvm_pmu_idx_has_chain_evtype(struct kvm_vcpu *vcpu, u64 select_idx)
  85{
  86        u64 eventsel, reg;
  87
  88        select_idx |= 0x1;
  89
  90        if (select_idx == ARMV8_PMU_CYCLE_IDX)
  91                return false;
  92
  93        reg = PMEVTYPER0_EL0 + select_idx;
  94        eventsel = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_EVENT;
  95
  96        return eventsel == ARMV8_PMUV3_PERFCTR_CHAIN;
  97}
  98
  99/**
 100 * kvm_pmu_get_pair_counter_value - get PMU counter value
 101 * @vcpu: The vcpu pointer
 102 * @pmc: The PMU counter pointer
 103 */
 104static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu,
 105                                          struct kvm_pmc *pmc)
 106{
 107        u64 counter, counter_high, reg, enabled, running;
 108
 109        if (kvm_pmu_pmc_is_chained(pmc)) {
 110                pmc = kvm_pmu_get_canonical_pmc(pmc);
 111                reg = PMEVCNTR0_EL0 + pmc->idx;
 112
 113                counter = __vcpu_sys_reg(vcpu, reg);
 114                counter_high = __vcpu_sys_reg(vcpu, reg + 1);
 115
 116                counter = lower_32_bits(counter) | (counter_high << 32);
 117        } else {
 118                reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
 119                      ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
 120                counter = __vcpu_sys_reg(vcpu, reg);
 121        }
 122
 123        /*
 124         * The real counter value is equal to the value of counter register plus
 125         * the value perf event counts.
 126         */
 127        if (pmc->perf_event)
 128                counter += perf_event_read_value(pmc->perf_event, &enabled,
 129                                                 &running);
 130
 131        return counter;
 132}
 133
 134/**
 135 * kvm_pmu_get_counter_value - get PMU counter value
 136 * @vcpu: The vcpu pointer
 137 * @select_idx: The counter index
 138 */
 139u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
 140{
 141        u64 counter;
 142        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 143        struct kvm_pmc *pmc = &pmu->pmc[select_idx];
 144
 145        counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
 146
 147        if (kvm_pmu_pmc_is_chained(pmc) &&
 148            kvm_pmu_idx_is_high_counter(select_idx))
 149                counter = upper_32_bits(counter);
 150        else if (select_idx != ARMV8_PMU_CYCLE_IDX)
 151                counter = lower_32_bits(counter);
 152
 153        return counter;
 154}
 155
 156/**
 157 * kvm_pmu_set_counter_value - set PMU counter value
 158 * @vcpu: The vcpu pointer
 159 * @select_idx: The counter index
 160 * @val: The counter value
 161 */
 162void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
 163{
 164        u64 reg;
 165
 166        reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
 167              ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
 168        __vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
 169
 170        /* Recreate the perf event to reflect the updated sample_period */
 171        kvm_pmu_create_perf_event(vcpu, select_idx);
 172}
 173
 174/**
 175 * kvm_pmu_release_perf_event - remove the perf event
 176 * @pmc: The PMU counter pointer
 177 */
 178static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
 179{
 180        pmc = kvm_pmu_get_canonical_pmc(pmc);
 181        if (pmc->perf_event) {
 182                perf_event_disable(pmc->perf_event);
 183                perf_event_release_kernel(pmc->perf_event);
 184                pmc->perf_event = NULL;
 185        }
 186}
 187
 188/**
 189 * kvm_pmu_stop_counter - stop PMU counter
 190 * @pmc: The PMU counter pointer
 191 *
 192 * If this counter has been configured to monitor some event, release it here.
 193 */
 194static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
 195{
 196        u64 counter, reg, val;
 197
 198        pmc = kvm_pmu_get_canonical_pmc(pmc);
 199        if (!pmc->perf_event)
 200                return;
 201
 202        counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
 203
 204        if (pmc->idx == ARMV8_PMU_CYCLE_IDX) {
 205                reg = PMCCNTR_EL0;
 206                val = counter;
 207        } else {
 208                reg = PMEVCNTR0_EL0 + pmc->idx;
 209                val = lower_32_bits(counter);
 210        }
 211
 212        __vcpu_sys_reg(vcpu, reg) = val;
 213
 214        if (kvm_pmu_pmc_is_chained(pmc))
 215                __vcpu_sys_reg(vcpu, reg + 1) = upper_32_bits(counter);
 216
 217        kvm_pmu_release_perf_event(pmc);
 218}
 219
 220/**
 221 * kvm_pmu_vcpu_init - assign pmu counter idx for cpu
 222 * @vcpu: The vcpu pointer
 223 *
 224 */
 225void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu)
 226{
 227        int i;
 228        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 229
 230        for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
 231                pmu->pmc[i].idx = i;
 232}
 233
 234/**
 235 * kvm_pmu_vcpu_reset - reset pmu state for cpu
 236 * @vcpu: The vcpu pointer
 237 *
 238 */
 239void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
 240{
 241        int i;
 242        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 243
 244        for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
 245                kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
 246
 247        bitmap_zero(vcpu->arch.pmu.chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
 248}
 249
 250/**
 251 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
 252 * @vcpu: The vcpu pointer
 253 *
 254 */
 255void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
 256{
 257        int i;
 258        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 259
 260        for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
 261                kvm_pmu_release_perf_event(&pmu->pmc[i]);
 262}
 263
 264u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
 265{
 266        u64 val = __vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
 267
 268        val &= ARMV8_PMU_PMCR_N_MASK;
 269        if (val == 0)
 270                return BIT(ARMV8_PMU_CYCLE_IDX);
 271        else
 272                return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
 273}
 274
 275/**
 276 * kvm_pmu_enable_counter_mask - enable selected PMU counters
 277 * @vcpu: The vcpu pointer
 278 * @val: the value guest writes to PMCNTENSET register
 279 *
 280 * Call perf_event_enable to start counting the perf event
 281 */
 282void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
 283{
 284        int i;
 285        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 286        struct kvm_pmc *pmc;
 287
 288        if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
 289                return;
 290
 291        for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
 292                if (!(val & BIT(i)))
 293                        continue;
 294
 295                pmc = &pmu->pmc[i];
 296
 297                /*
 298                 * For high counters of chained events we must recreate the
 299                 * perf event with the long (64bit) attribute set.
 300                 */
 301                if (kvm_pmu_pmc_is_chained(pmc) &&
 302                    kvm_pmu_idx_is_high_counter(i)) {
 303                        kvm_pmu_create_perf_event(vcpu, i);
 304                        continue;
 305                }
 306
 307                /* At this point, pmc must be the canonical */
 308                if (pmc->perf_event) {
 309                        perf_event_enable(pmc->perf_event);
 310                        if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
 311                                kvm_debug("fail to enable perf event\n");
 312                }
 313        }
 314}
 315
 316/**
 317 * kvm_pmu_disable_counter_mask - disable selected PMU counters
 318 * @vcpu: The vcpu pointer
 319 * @val: the value guest writes to PMCNTENCLR register
 320 *
 321 * Call perf_event_disable to stop counting the perf event
 322 */
 323void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
 324{
 325        int i;
 326        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 327        struct kvm_pmc *pmc;
 328
 329        if (!val)
 330                return;
 331
 332        for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
 333                if (!(val & BIT(i)))
 334                        continue;
 335
 336                pmc = &pmu->pmc[i];
 337
 338                /*
 339                 * For high counters of chained events we must recreate the
 340                 * perf event with the long (64bit) attribute unset.
 341                 */
 342                if (kvm_pmu_pmc_is_chained(pmc) &&
 343                    kvm_pmu_idx_is_high_counter(i)) {
 344                        kvm_pmu_create_perf_event(vcpu, i);
 345                        continue;
 346                }
 347
 348                /* At this point, pmc must be the canonical */
 349                if (pmc->perf_event)
 350                        perf_event_disable(pmc->perf_event);
 351        }
 352}
 353
 354static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
 355{
 356        u64 reg = 0;
 357
 358        if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) {
 359                reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
 360                reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
 361                reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
 362                reg &= kvm_pmu_valid_counter_mask(vcpu);
 363        }
 364
 365        return reg;
 366}
 367
 368static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
 369{
 370        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 371        bool overflow;
 372
 373        if (!kvm_arm_pmu_v3_ready(vcpu))
 374                return;
 375
 376        overflow = !!kvm_pmu_overflow_status(vcpu);
 377        if (pmu->irq_level == overflow)
 378                return;
 379
 380        pmu->irq_level = overflow;
 381
 382        if (likely(irqchip_in_kernel(vcpu->kvm))) {
 383                int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 384                                              pmu->irq_num, overflow, pmu);
 385                WARN_ON(ret);
 386        }
 387}
 388
 389bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
 390{
 391        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 392        struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 393        bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
 394
 395        if (likely(irqchip_in_kernel(vcpu->kvm)))
 396                return false;
 397
 398        return pmu->irq_level != run_level;
 399}
 400
 401/*
 402 * Reflect the PMU overflow interrupt output level into the kvm_run structure
 403 */
 404void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
 405{
 406        struct kvm_sync_regs *regs = &vcpu->run->s.regs;
 407
 408        /* Populate the timer bitmap for user space */
 409        regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
 410        if (vcpu->arch.pmu.irq_level)
 411                regs->device_irq_level |= KVM_ARM_DEV_PMU;
 412}
 413
 414/**
 415 * kvm_pmu_flush_hwstate - flush pmu state to cpu
 416 * @vcpu: The vcpu pointer
 417 *
 418 * Check if the PMU has overflowed while we were running in the host, and inject
 419 * an interrupt if that was the case.
 420 */
 421void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
 422{
 423        kvm_pmu_update_state(vcpu);
 424}
 425
 426/**
 427 * kvm_pmu_sync_hwstate - sync pmu state from cpu
 428 * @vcpu: The vcpu pointer
 429 *
 430 * Check if the PMU has overflowed while we were running in the guest, and
 431 * inject an interrupt if that was the case.
 432 */
 433void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
 434{
 435        kvm_pmu_update_state(vcpu);
 436}
 437
 438/**
 439 * When the perf event overflows, set the overflow status and inform the vcpu.
 440 */
 441static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
 442                                  struct perf_sample_data *data,
 443                                  struct pt_regs *regs)
 444{
 445        struct kvm_pmc *pmc = perf_event->overflow_handler_context;
 446        struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu);
 447        struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
 448        int idx = pmc->idx;
 449        u64 period;
 450
 451        cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE);
 452
 453        /*
 454         * Reset the sample period to the architectural limit,
 455         * i.e. the point where the counter overflows.
 456         */
 457        period = -(local64_read(&perf_event->count));
 458
 459        if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
 460                period &= GENMASK(31, 0);
 461
 462        local64_set(&perf_event->hw.period_left, 0);
 463        perf_event->attr.sample_period = period;
 464        perf_event->hw.sample_period = period;
 465
 466        __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);
 467
 468        if (kvm_pmu_overflow_status(vcpu)) {
 469                kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
 470                kvm_vcpu_kick(vcpu);
 471        }
 472
 473        cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD);
 474}
 475
 476/**
 477 * kvm_pmu_software_increment - do software increment
 478 * @vcpu: The vcpu pointer
 479 * @val: the value guest writes to PMSWINC register
 480 */
 481void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
 482{
 483        int i;
 484        u64 type, enable, reg;
 485
 486        if (val == 0)
 487                return;
 488
 489        enable = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
 490        for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
 491                if (!(val & BIT(i)))
 492                        continue;
 493                type = __vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
 494                       & ARMV8_PMU_EVTYPE_EVENT;
 495                if ((type == ARMV8_PMUV3_PERFCTR_SW_INCR)
 496                    && (enable & BIT(i))) {
 497                        reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
 498                        reg = lower_32_bits(reg);
 499                        __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
 500                        if (!reg)
 501                                __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
 502                }
 503        }
 504}
 505
 506/**
 507 * kvm_pmu_handle_pmcr - handle PMCR register
 508 * @vcpu: The vcpu pointer
 509 * @val: the value guest writes to PMCR register
 510 */
 511void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
 512{
 513        u64 mask;
 514        int i;
 515
 516        mask = kvm_pmu_valid_counter_mask(vcpu);
 517        if (val & ARMV8_PMU_PMCR_E) {
 518                kvm_pmu_enable_counter_mask(vcpu,
 519                       __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
 520        } else {
 521                kvm_pmu_disable_counter_mask(vcpu, mask);
 522        }
 523
 524        if (val & ARMV8_PMU_PMCR_C)
 525                kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
 526
 527        if (val & ARMV8_PMU_PMCR_P) {
 528                for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
 529                        kvm_pmu_set_counter_value(vcpu, i, 0);
 530        }
 531}
 532
 533static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
 534{
 535        return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
 536               (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
 537}
 538
 539/**
 540 * kvm_pmu_create_perf_event - create a perf event for a counter
 541 * @vcpu: The vcpu pointer
 542 * @select_idx: The number of selected counter
 543 */
 544static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx)
 545{
 546        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 547        struct kvm_pmc *pmc;
 548        struct perf_event *event;
 549        struct perf_event_attr attr;
 550        u64 eventsel, counter, reg, data;
 551
 552        /*
 553         * For chained counters the event type and filtering attributes are
 554         * obtained from the low/even counter. We also use this counter to
 555         * determine if the event is enabled/disabled.
 556         */
 557        pmc = kvm_pmu_get_canonical_pmc(&pmu->pmc[select_idx]);
 558
 559        reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
 560              ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + pmc->idx;
 561        data = __vcpu_sys_reg(vcpu, reg);
 562
 563        kvm_pmu_stop_counter(vcpu, pmc);
 564        eventsel = data & ARMV8_PMU_EVTYPE_EVENT;
 565
 566        /* Software increment event does't need to be backed by a perf event */
 567        if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR &&
 568            pmc->idx != ARMV8_PMU_CYCLE_IDX)
 569                return;
 570
 571        memset(&attr, 0, sizeof(struct perf_event_attr));
 572        attr.type = PERF_TYPE_RAW;
 573        attr.size = sizeof(attr);
 574        attr.pinned = 1;
 575        attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);
 576        attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
 577        attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
 578        attr.exclude_hv = 1; /* Don't count EL2 events */
 579        attr.exclude_host = 1; /* Don't count host events */
 580        attr.config = (pmc->idx == ARMV8_PMU_CYCLE_IDX) ?
 581                ARMV8_PMUV3_PERFCTR_CPU_CYCLES : eventsel;
 582
 583        counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
 584
 585        if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {
 586                /**
 587                 * The initial sample period (overflow count) of an event. For
 588                 * chained counters we only support overflow interrupts on the
 589                 * high counter.
 590                 */
 591                attr.sample_period = (-counter) & GENMASK(63, 0);
 592                if (kvm_pmu_counter_is_enabled(vcpu, pmc->idx + 1))
 593                        attr.config1 |= PERF_ATTR_CFG1_KVM_PMU_CHAINED;
 594
 595                event = perf_event_create_kernel_counter(&attr, -1, current,
 596                                                         kvm_pmu_perf_overflow,
 597                                                         pmc + 1);
 598        } else {
 599                /* The initial sample period (overflow count) of an event. */
 600                if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
 601                        attr.sample_period = (-counter) & GENMASK(63, 0);
 602                else
 603                        attr.sample_period = (-counter) & GENMASK(31, 0);
 604
 605                event = perf_event_create_kernel_counter(&attr, -1, current,
 606                                                 kvm_pmu_perf_overflow, pmc);
 607        }
 608
 609        if (IS_ERR(event)) {
 610                pr_err_once("kvm: pmu event creation failed %ld\n",
 611                            PTR_ERR(event));
 612                return;
 613        }
 614
 615        pmc->perf_event = event;
 616}
 617
 618/**
 619 * kvm_pmu_update_pmc_chained - update chained bitmap
 620 * @vcpu: The vcpu pointer
 621 * @select_idx: The number of selected counter
 622 *
 623 * Update the chained bitmap based on the event type written in the
 624 * typer register.
 625 */
 626static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx)
 627{
 628        struct kvm_pmu *pmu = &vcpu->arch.pmu;
 629        struct kvm_pmc *pmc = &pmu->pmc[select_idx];
 630
 631        if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {
 632                /*
 633                 * During promotion from !chained to chained we must ensure
 634                 * the adjacent counter is stopped and its event destroyed
 635                 */
 636                if (!kvm_pmu_pmc_is_chained(pmc))
 637                        kvm_pmu_stop_counter(vcpu, pmc);
 638
 639                set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
 640        } else {
 641                clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
 642        }
 643}
 644
 645/**
 646 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
 647 * @vcpu: The vcpu pointer
 648 * @data: The data guest writes to PMXEVTYPER_EL0
 649 * @select_idx: The number of selected counter
 650 *
 651 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
 652 * event with given hardware event number. Here we call perf_event API to
 653 * emulate this action and create a kernel perf event for it.
 654 */
 655void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
 656                                    u64 select_idx)
 657{
 658        u64 reg, event_type = data & ARMV8_PMU_EVTYPE_MASK;
 659
 660        reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
 661              ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + select_idx;
 662
 663        __vcpu_sys_reg(vcpu, reg) = event_type;
 664
 665        kvm_pmu_update_pmc_chained(vcpu, select_idx);
 666        kvm_pmu_create_perf_event(vcpu, select_idx);
 667}
 668
 669bool kvm_arm_support_pmu_v3(void)
 670{
 671        /*
 672         * Check if HW_PERF_EVENTS are supported by checking the number of
 673         * hardware performance counters. This could ensure the presence of
 674         * a physical PMU and CONFIG_PERF_EVENT is selected.
 675         */
 676        return (perf_num_counters() > 0);
 677}
 678
 679int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)
 680{
 681        if (!vcpu->arch.pmu.created)
 682                return 0;
 683
 684        /*
 685         * A valid interrupt configuration for the PMU is either to have a
 686         * properly configured interrupt number and using an in-kernel
 687         * irqchip, or to not have an in-kernel GIC and not set an IRQ.
 688         */
 689        if (irqchip_in_kernel(vcpu->kvm)) {
 690                int irq = vcpu->arch.pmu.irq_num;
 691                if (!kvm_arm_pmu_irq_initialized(vcpu))
 692                        return -EINVAL;
 693
 694                /*
 695                 * If we are using an in-kernel vgic, at this point we know
 696                 * the vgic will be initialized, so we can check the PMU irq
 697                 * number against the dimensions of the vgic and make sure
 698                 * it's valid.
 699                 */
 700                if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq))
 701                        return -EINVAL;
 702        } else if (kvm_arm_pmu_irq_initialized(vcpu)) {
 703                   return -EINVAL;
 704        }
 705
 706        kvm_pmu_vcpu_reset(vcpu);
 707        vcpu->arch.pmu.ready = true;
 708
 709        return 0;
 710}
 711
 712static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
 713{
 714        if (!kvm_arm_support_pmu_v3())
 715                return -ENODEV;
 716
 717        if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 718                return -ENXIO;
 719
 720        if (vcpu->arch.pmu.created)
 721                return -EBUSY;
 722
 723        if (irqchip_in_kernel(vcpu->kvm)) {
 724                int ret;
 725
 726                /*
 727                 * If using the PMU with an in-kernel virtual GIC
 728                 * implementation, we require the GIC to be already
 729                 * initialized when initializing the PMU.
 730                 */
 731                if (!vgic_initialized(vcpu->kvm))
 732                        return -ENODEV;
 733
 734                if (!kvm_arm_pmu_irq_initialized(vcpu))
 735                        return -ENXIO;
 736
 737                ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num,
 738                                         &vcpu->arch.pmu);
 739                if (ret)
 740                        return ret;
 741        }
 742
 743        vcpu->arch.pmu.created = true;
 744        return 0;
 745}
 746
 747/*
 748 * For one VM the interrupt type must be same for each vcpu.
 749 * As a PPI, the interrupt number is the same for all vcpus,
 750 * while as an SPI it must be a separate number per vcpu.
 751 */
 752static bool pmu_irq_is_valid(struct kvm *kvm, int irq)
 753{
 754        int i;
 755        struct kvm_vcpu *vcpu;
 756
 757        kvm_for_each_vcpu(i, vcpu, kvm) {
 758                if (!kvm_arm_pmu_irq_initialized(vcpu))
 759                        continue;
 760
 761                if (irq_is_ppi(irq)) {
 762                        if (vcpu->arch.pmu.irq_num != irq)
 763                                return false;
 764                } else {
 765                        if (vcpu->arch.pmu.irq_num == irq)
 766                                return false;
 767                }
 768        }
 769
 770        return true;
 771}
 772
 773int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 774{
 775        switch (attr->attr) {
 776        case KVM_ARM_VCPU_PMU_V3_IRQ: {
 777                int __user *uaddr = (int __user *)(long)attr->addr;
 778                int irq;
 779
 780                if (!irqchip_in_kernel(vcpu->kvm))
 781                        return -EINVAL;
 782
 783                if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 784                        return -ENODEV;
 785
 786                if (get_user(irq, uaddr))
 787                        return -EFAULT;
 788
 789                /* The PMU overflow interrupt can be a PPI or a valid SPI. */
 790                if (!(irq_is_ppi(irq) || irq_is_spi(irq)))
 791                        return -EINVAL;
 792
 793                if (!pmu_irq_is_valid(vcpu->kvm, irq))
 794                        return -EINVAL;
 795
 796                if (kvm_arm_pmu_irq_initialized(vcpu))
 797                        return -EBUSY;
 798
 799                kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
 800                vcpu->arch.pmu.irq_num = irq;
 801                return 0;
 802        }
 803        case KVM_ARM_VCPU_PMU_V3_INIT:
 804                return kvm_arm_pmu_v3_init(vcpu);
 805        }
 806
 807        return -ENXIO;
 808}
 809
 810int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 811{
 812        switch (attr->attr) {
 813        case KVM_ARM_VCPU_PMU_V3_IRQ: {
 814                int __user *uaddr = (int __user *)(long)attr->addr;
 815                int irq;
 816
 817                if (!irqchip_in_kernel(vcpu->kvm))
 818                        return -EINVAL;
 819
 820                if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 821                        return -ENODEV;
 822
 823                if (!kvm_arm_pmu_irq_initialized(vcpu))
 824                        return -ENXIO;
 825
 826                irq = vcpu->arch.pmu.irq_num;
 827                return put_user(irq, uaddr);
 828        }
 829        }
 830
 831        return -ENXIO;
 832}
 833
 834int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 835{
 836        switch (attr->attr) {
 837        case KVM_ARM_VCPU_PMU_V3_IRQ:
 838        case KVM_ARM_VCPU_PMU_V3_INIT:
 839                if (kvm_arm_support_pmu_v3() &&
 840                    test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 841                        return 0;
 842        }
 843
 844        return -ENXIO;
 845}
 846