linux/kernel/events/core.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Performance events core code:
   4 *
   5 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   6 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
   7 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
   8 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   9 */
  10
  11#include <linux/fs.h>
  12#include <linux/mm.h>
  13#include <linux/cpu.h>
  14#include <linux/smp.h>
  15#include <linux/idr.h>
  16#include <linux/file.h>
  17#include <linux/poll.h>
  18#include <linux/slab.h>
  19#include <linux/hash.h>
  20#include <linux/tick.h>
  21#include <linux/sysfs.h>
  22#include <linux/dcache.h>
  23#include <linux/percpu.h>
  24#include <linux/ptrace.h>
  25#include <linux/reboot.h>
  26#include <linux/vmstat.h>
  27#include <linux/device.h>
  28#include <linux/export.h>
  29#include <linux/vmalloc.h>
  30#include <linux/hardirq.h>
  31#include <linux/hugetlb.h>
  32#include <linux/rculist.h>
  33#include <linux/uaccess.h>
  34#include <linux/syscalls.h>
  35#include <linux/anon_inodes.h>
  36#include <linux/kernel_stat.h>
  37#include <linux/cgroup.h>
  38#include <linux/perf_event.h>
  39#include <linux/trace_events.h>
  40#include <linux/hw_breakpoint.h>
  41#include <linux/mm_types.h>
  42#include <linux/module.h>
  43#include <linux/mman.h>
  44#include <linux/compat.h>
  45#include <linux/bpf.h>
  46#include <linux/filter.h>
  47#include <linux/namei.h>
  48#include <linux/parser.h>
  49#include <linux/sched/clock.h>
  50#include <linux/sched/mm.h>
  51#include <linux/proc_ns.h>
  52#include <linux/mount.h>
  53#include <linux/min_heap.h>
  54#include <linux/highmem.h>
  55#include <linux/pgtable.h>
  56#include <linux/buildid.h>
  57
  58#include "internal.h"
  59
  60#include <asm/irq_regs.h>
  61
  62typedef int (*remote_function_f)(void *);
  63
  64struct remote_function_call {
  65        struct task_struct      *p;
  66        remote_function_f       func;
  67        void                    *info;
  68        int                     ret;
  69};
  70
  71static void remote_function(void *data)
  72{
  73        struct remote_function_call *tfc = data;
  74        struct task_struct *p = tfc->p;
  75
  76        if (p) {
  77                /* -EAGAIN */
  78                if (task_cpu(p) != smp_processor_id())
  79                        return;
  80
  81                /*
  82                 * Now that we're on right CPU with IRQs disabled, we can test
  83                 * if we hit the right task without races.
  84                 */
  85
  86                tfc->ret = -ESRCH; /* No such (running) process */
  87                if (p != current)
  88                        return;
  89        }
  90
  91        tfc->ret = tfc->func(tfc->info);
  92}
  93
  94/**
  95 * task_function_call - call a function on the cpu on which a task runs
  96 * @p:          the task to evaluate
  97 * @func:       the function to be called
  98 * @info:       the function call argument
  99 *
 100 * Calls the function @func when the task is currently running. This might
 101 * be on the current CPU, which just calls the function directly.  This will
 102 * retry due to any failures in smp_call_function_single(), such as if the
 103 * task_cpu() goes offline concurrently.
 104 *
 105 * returns @func return value or -ESRCH or -ENXIO when the process isn't running
 106 */
 107static int
 108task_function_call(struct task_struct *p, remote_function_f func, void *info)
 109{
 110        struct remote_function_call data = {
 111                .p      = p,
 112                .func   = func,
 113                .info   = info,
 114                .ret    = -EAGAIN,
 115        };
 116        int ret;
 117
 118        for (;;) {
 119                ret = smp_call_function_single(task_cpu(p), remote_function,
 120                                               &data, 1);
 121                if (!ret)
 122                        ret = data.ret;
 123
 124                if (ret != -EAGAIN)
 125                        break;
 126
 127                cond_resched();
 128        }
 129
 130        return ret;
 131}
 132
 133/**
 134 * cpu_function_call - call a function on the cpu
 135 * @cpu:        target cpu to queue this function
 136 * @func:       the function to be called
 137 * @info:       the function call argument
 138 *
 139 * Calls the function @func on the remote cpu.
 140 *
 141 * returns: @func return value or -ENXIO when the cpu is offline
 142 */
 143static int cpu_function_call(int cpu, remote_function_f func, void *info)
 144{
 145        struct remote_function_call data = {
 146                .p      = NULL,
 147                .func   = func,
 148                .info   = info,
 149                .ret    = -ENXIO, /* No such CPU */
 150        };
 151
 152        smp_call_function_single(cpu, remote_function, &data, 1);
 153
 154        return data.ret;
 155}
 156
 157static inline struct perf_cpu_context *
 158__get_cpu_context(struct perf_event_context *ctx)
 159{
 160        return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
 161}
 162
 163static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
 164                          struct perf_event_context *ctx)
 165{
 166        raw_spin_lock(&cpuctx->ctx.lock);
 167        if (ctx)
 168                raw_spin_lock(&ctx->lock);
 169}
 170
 171static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
 172                            struct perf_event_context *ctx)
 173{
 174        if (ctx)
 175                raw_spin_unlock(&ctx->lock);
 176        raw_spin_unlock(&cpuctx->ctx.lock);
 177}
 178
 179#define TASK_TOMBSTONE ((void *)-1L)
 180
 181static bool is_kernel_event(struct perf_event *event)
 182{
 183        return READ_ONCE(event->owner) == TASK_TOMBSTONE;
 184}
 185
 186/*
 187 * On task ctx scheduling...
 188 *
 189 * When !ctx->nr_events a task context will not be scheduled. This means
 190 * we can disable the scheduler hooks (for performance) without leaving
 191 * pending task ctx state.
 192 *
 193 * This however results in two special cases:
 194 *
 195 *  - removing the last event from a task ctx; this is relatively straight
 196 *    forward and is done in __perf_remove_from_context.
 197 *
 198 *  - adding the first event to a task ctx; this is tricky because we cannot
 199 *    rely on ctx->is_active and therefore cannot use event_function_call().
 200 *    See perf_install_in_context().
 201 *
 202 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
 203 */
 204
 205typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
 206                        struct perf_event_context *, void *);
 207
 208struct event_function_struct {
 209        struct perf_event *event;
 210        event_f func;
 211        void *data;
 212};
 213
 214static int event_function(void *info)
 215{
 216        struct event_function_struct *efs = info;
 217        struct perf_event *event = efs->event;
 218        struct perf_event_context *ctx = event->ctx;
 219        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 220        struct perf_event_context *task_ctx = cpuctx->task_ctx;
 221        int ret = 0;
 222
 223        lockdep_assert_irqs_disabled();
 224
 225        perf_ctx_lock(cpuctx, task_ctx);
 226        /*
 227         * Since we do the IPI call without holding ctx->lock things can have
 228         * changed, double check we hit the task we set out to hit.
 229         */
 230        if (ctx->task) {
 231                if (ctx->task != current) {
 232                        ret = -ESRCH;
 233                        goto unlock;
 234                }
 235
 236                /*
 237                 * We only use event_function_call() on established contexts,
 238                 * and event_function() is only ever called when active (or
 239                 * rather, we'll have bailed in task_function_call() or the
 240                 * above ctx->task != current test), therefore we must have
 241                 * ctx->is_active here.
 242                 */
 243                WARN_ON_ONCE(!ctx->is_active);
 244                /*
 245                 * And since we have ctx->is_active, cpuctx->task_ctx must
 246                 * match.
 247                 */
 248                WARN_ON_ONCE(task_ctx != ctx);
 249        } else {
 250                WARN_ON_ONCE(&cpuctx->ctx != ctx);
 251        }
 252
 253        efs->func(event, cpuctx, ctx, efs->data);
 254unlock:
 255        perf_ctx_unlock(cpuctx, task_ctx);
 256
 257        return ret;
 258}
 259
 260static void event_function_call(struct perf_event *event, event_f func, void *data)
 261{
 262        struct perf_event_context *ctx = event->ctx;
 263        struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
 264        struct event_function_struct efs = {
 265                .event = event,
 266                .func = func,
 267                .data = data,
 268        };
 269
 270        if (!event->parent) {
 271                /*
 272                 * If this is a !child event, we must hold ctx::mutex to
 273                 * stabilize the event->ctx relation. See
 274                 * perf_event_ctx_lock().
 275                 */
 276                lockdep_assert_held(&ctx->mutex);
 277        }
 278
 279        if (!task) {
 280                cpu_function_call(event->cpu, event_function, &efs);
 281                return;
 282        }
 283
 284        if (task == TASK_TOMBSTONE)
 285                return;
 286
 287again:
 288        if (!task_function_call(task, event_function, &efs))
 289                return;
 290
 291        raw_spin_lock_irq(&ctx->lock);
 292        /*
 293         * Reload the task pointer, it might have been changed by
 294         * a concurrent perf_event_context_sched_out().
 295         */
 296        task = ctx->task;
 297        if (task == TASK_TOMBSTONE) {
 298                raw_spin_unlock_irq(&ctx->lock);
 299                return;
 300        }
 301        if (ctx->is_active) {
 302                raw_spin_unlock_irq(&ctx->lock);
 303                goto again;
 304        }
 305        func(event, NULL, ctx, data);
 306        raw_spin_unlock_irq(&ctx->lock);
 307}
 308
 309/*
 310 * Similar to event_function_call() + event_function(), but hard assumes IRQs
 311 * are already disabled and we're on the right CPU.
 312 */
 313static void event_function_local(struct perf_event *event, event_f func, void *data)
 314{
 315        struct perf_event_context *ctx = event->ctx;
 316        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 317        struct task_struct *task = READ_ONCE(ctx->task);
 318        struct perf_event_context *task_ctx = NULL;
 319
 320        lockdep_assert_irqs_disabled();
 321
 322        if (task) {
 323                if (task == TASK_TOMBSTONE)
 324                        return;
 325
 326                task_ctx = ctx;
 327        }
 328
 329        perf_ctx_lock(cpuctx, task_ctx);
 330
 331        task = ctx->task;
 332        if (task == TASK_TOMBSTONE)
 333                goto unlock;
 334
 335        if (task) {
 336                /*
 337                 * We must be either inactive or active and the right task,
 338                 * otherwise we're screwed, since we cannot IPI to somewhere
 339                 * else.
 340                 */
 341                if (ctx->is_active) {
 342                        if (WARN_ON_ONCE(task != current))
 343                                goto unlock;
 344
 345                        if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
 346                                goto unlock;
 347                }
 348        } else {
 349                WARN_ON_ONCE(&cpuctx->ctx != ctx);
 350        }
 351
 352        func(event, cpuctx, ctx, data);
 353unlock:
 354        perf_ctx_unlock(cpuctx, task_ctx);
 355}
 356
 357#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
 358                       PERF_FLAG_FD_OUTPUT  |\
 359                       PERF_FLAG_PID_CGROUP |\
 360                       PERF_FLAG_FD_CLOEXEC)
 361
 362/*
 363 * branch priv levels that need permission checks
 364 */
 365#define PERF_SAMPLE_BRANCH_PERM_PLM \
 366        (PERF_SAMPLE_BRANCH_KERNEL |\
 367         PERF_SAMPLE_BRANCH_HV)
 368
 369enum event_type_t {
 370        EVENT_FLEXIBLE = 0x1,
 371        EVENT_PINNED = 0x2,
 372        EVENT_TIME = 0x4,
 373        /* see ctx_resched() for details */
 374        EVENT_CPU = 0x8,
 375        EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
 376};
 377
 378/*
 379 * perf_sched_events : >0 events exist
 380 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
 381 */
 382
 383static void perf_sched_delayed(struct work_struct *work);
 384DEFINE_STATIC_KEY_FALSE(perf_sched_events);
 385static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
 386static DEFINE_MUTEX(perf_sched_mutex);
 387static atomic_t perf_sched_count;
 388
 389static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
 390static DEFINE_PER_CPU(int, perf_sched_cb_usages);
 391static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
 392
 393static atomic_t nr_mmap_events __read_mostly;
 394static atomic_t nr_comm_events __read_mostly;
 395static atomic_t nr_namespaces_events __read_mostly;
 396static atomic_t nr_task_events __read_mostly;
 397static atomic_t nr_freq_events __read_mostly;
 398static atomic_t nr_switch_events __read_mostly;
 399static atomic_t nr_ksymbol_events __read_mostly;
 400static atomic_t nr_bpf_events __read_mostly;
 401static atomic_t nr_cgroup_events __read_mostly;
 402static atomic_t nr_text_poke_events __read_mostly;
 403static atomic_t nr_build_id_events __read_mostly;
 404
 405static LIST_HEAD(pmus);
 406static DEFINE_MUTEX(pmus_lock);
 407static struct srcu_struct pmus_srcu;
 408static cpumask_var_t perf_online_mask;
 409static struct kmem_cache *perf_event_cache;
 410
 411/*
 412 * perf event paranoia level:
 413 *  -1 - not paranoid at all
 414 *   0 - disallow raw tracepoint access for unpriv
 415 *   1 - disallow cpu events for unpriv
 416 *   2 - disallow kernel profiling for unpriv
 417 */
 418int sysctl_perf_event_paranoid __read_mostly = 2;
 419
 420/* Minimum for 512 kiB + 1 user control page */
 421int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
 422
 423/*
 424 * max perf event sample rate
 425 */
 426#define DEFAULT_MAX_SAMPLE_RATE         100000
 427#define DEFAULT_SAMPLE_PERIOD_NS        (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
 428#define DEFAULT_CPU_TIME_MAX_PERCENT    25
 429
 430int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
 431
 432static int max_samples_per_tick __read_mostly   = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
 433static int perf_sample_period_ns __read_mostly  = DEFAULT_SAMPLE_PERIOD_NS;
 434
 435static int perf_sample_allowed_ns __read_mostly =
 436        DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
 437
 438static void update_perf_cpu_limits(void)
 439{
 440        u64 tmp = perf_sample_period_ns;
 441
 442        tmp *= sysctl_perf_cpu_time_max_percent;
 443        tmp = div_u64(tmp, 100);
 444        if (!tmp)
 445                tmp = 1;
 446
 447        WRITE_ONCE(perf_sample_allowed_ns, tmp);
 448}
 449
 450static bool perf_rotate_context(struct perf_cpu_context *cpuctx);
 451
 452int perf_proc_update_handler(struct ctl_table *table, int write,
 453                void *buffer, size_t *lenp, loff_t *ppos)
 454{
 455        int ret;
 456        int perf_cpu = sysctl_perf_cpu_time_max_percent;
 457        /*
 458         * If throttling is disabled don't allow the write:
 459         */
 460        if (write && (perf_cpu == 100 || perf_cpu == 0))
 461                return -EINVAL;
 462
 463        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 464        if (ret || !write)
 465                return ret;
 466
 467        max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
 468        perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
 469        update_perf_cpu_limits();
 470
 471        return 0;
 472}
 473
 474int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
 475
 476int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
 477                void *buffer, size_t *lenp, loff_t *ppos)
 478{
 479        int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 480
 481        if (ret || !write)
 482                return ret;
 483
 484        if (sysctl_perf_cpu_time_max_percent == 100 ||
 485            sysctl_perf_cpu_time_max_percent == 0) {
 486                printk(KERN_WARNING
 487                       "perf: Dynamic interrupt throttling disabled, can hang your system!\n");
 488                WRITE_ONCE(perf_sample_allowed_ns, 0);
 489        } else {
 490                update_perf_cpu_limits();
 491        }
 492
 493        return 0;
 494}
 495
 496/*
 497 * perf samples are done in some very critical code paths (NMIs).
 498 * If they take too much CPU time, the system can lock up and not
 499 * get any real work done.  This will drop the sample rate when
 500 * we detect that events are taking too long.
 501 */
 502#define NR_ACCUMULATED_SAMPLES 128
 503static DEFINE_PER_CPU(u64, running_sample_length);
 504
 505static u64 __report_avg;
 506static u64 __report_allowed;
 507
 508static void perf_duration_warn(struct irq_work *w)
 509{
 510        printk_ratelimited(KERN_INFO
 511                "perf: interrupt took too long (%lld > %lld), lowering "
 512                "kernel.perf_event_max_sample_rate to %d\n",
 513                __report_avg, __report_allowed,
 514                sysctl_perf_event_sample_rate);
 515}
 516
 517static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
 518
 519void perf_sample_event_took(u64 sample_len_ns)
 520{
 521        u64 max_len = READ_ONCE(perf_sample_allowed_ns);
 522        u64 running_len;
 523        u64 avg_len;
 524        u32 max;
 525
 526        if (max_len == 0)
 527                return;
 528
 529        /* Decay the counter by 1 average sample. */
 530        running_len = __this_cpu_read(running_sample_length);
 531        running_len -= running_len/NR_ACCUMULATED_SAMPLES;
 532        running_len += sample_len_ns;
 533        __this_cpu_write(running_sample_length, running_len);
 534
 535        /*
 536         * Note: this will be biased artifically low until we have
 537         * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
 538         * from having to maintain a count.
 539         */
 540        avg_len = running_len/NR_ACCUMULATED_SAMPLES;
 541        if (avg_len <= max_len)
 542                return;
 543
 544        __report_avg = avg_len;
 545        __report_allowed = max_len;
 546
 547        /*
 548         * Compute a throttle threshold 25% below the current duration.
 549         */
 550        avg_len += avg_len / 4;
 551        max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
 552        if (avg_len < max)
 553                max /= (u32)avg_len;
 554        else
 555                max = 1;
 556
 557        WRITE_ONCE(perf_sample_allowed_ns, avg_len);
 558        WRITE_ONCE(max_samples_per_tick, max);
 559
 560        sysctl_perf_event_sample_rate = max * HZ;
 561        perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
 562
 563        if (!irq_work_queue(&perf_duration_work)) {
 564                early_printk("perf: interrupt took too long (%lld > %lld), lowering "
 565                             "kernel.perf_event_max_sample_rate to %d\n",
 566                             __report_avg, __report_allowed,
 567                             sysctl_perf_event_sample_rate);
 568        }
 569}
 570
 571static atomic64_t perf_event_id;
 572
 573static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 574                              enum event_type_t event_type);
 575
 576static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
 577                             enum event_type_t event_type,
 578                             struct task_struct *task);
 579
 580static void update_context_time(struct perf_event_context *ctx);
 581static u64 perf_event_time(struct perf_event *event);
 582
 583void __weak perf_event_print_debug(void)        { }
 584
 585static inline u64 perf_clock(void)
 586{
 587        return local_clock();
 588}
 589
 590static inline u64 perf_event_clock(struct perf_event *event)
 591{
 592        return event->clock();
 593}
 594
 595/*
 596 * State based event timekeeping...
 597 *
 598 * The basic idea is to use event->state to determine which (if any) time
 599 * fields to increment with the current delta. This means we only need to
 600 * update timestamps when we change state or when they are explicitly requested
 601 * (read).
 602 *
 603 * Event groups make things a little more complicated, but not terribly so. The
 604 * rules for a group are that if the group leader is OFF the entire group is
 605 * OFF, irrespecive of what the group member states are. This results in
 606 * __perf_effective_state().
 607 *
 608 * A futher ramification is that when a group leader flips between OFF and
 609 * !OFF, we need to update all group member times.
 610 *
 611 *
 612 * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
 613 * need to make sure the relevant context time is updated before we try and
 614 * update our timestamps.
 615 */
 616
 617static __always_inline enum perf_event_state
 618__perf_effective_state(struct perf_event *event)
 619{
 620        struct perf_event *leader = event->group_leader;
 621
 622        if (leader->state <= PERF_EVENT_STATE_OFF)
 623                return leader->state;
 624
 625        return event->state;
 626}
 627
 628static __always_inline void
 629__perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)
 630{
 631        enum perf_event_state state = __perf_effective_state(event);
 632        u64 delta = now - event->tstamp;
 633
 634        *enabled = event->total_time_enabled;
 635        if (state >= PERF_EVENT_STATE_INACTIVE)
 636                *enabled += delta;
 637
 638        *running = event->total_time_running;
 639        if (state >= PERF_EVENT_STATE_ACTIVE)
 640                *running += delta;
 641}
 642
 643static void perf_event_update_time(struct perf_event *event)
 644{
 645        u64 now = perf_event_time(event);
 646
 647        __perf_update_times(event, now, &event->total_time_enabled,
 648                                        &event->total_time_running);
 649        event->tstamp = now;
 650}
 651
 652static void perf_event_update_sibling_time(struct perf_event *leader)
 653{
 654        struct perf_event *sibling;
 655
 656        for_each_sibling_event(sibling, leader)
 657                perf_event_update_time(sibling);
 658}
 659
 660static void
 661perf_event_set_state(struct perf_event *event, enum perf_event_state state)
 662{
 663        if (event->state == state)
 664                return;
 665
 666        perf_event_update_time(event);
 667        /*
 668         * If a group leader gets enabled/disabled all its siblings
 669         * are affected too.
 670         */
 671        if ((event->state < 0) ^ (state < 0))
 672                perf_event_update_sibling_time(event);
 673
 674        WRITE_ONCE(event->state, state);
 675}
 676
 677#ifdef CONFIG_CGROUP_PERF
 678
 679static inline bool
 680perf_cgroup_match(struct perf_event *event)
 681{
 682        struct perf_event_context *ctx = event->ctx;
 683        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 684
 685        /* @event doesn't care about cgroup */
 686        if (!event->cgrp)
 687                return true;
 688
 689        /* wants specific cgroup scope but @cpuctx isn't associated with any */
 690        if (!cpuctx->cgrp)
 691                return false;
 692
 693        /*
 694         * Cgroup scoping is recursive.  An event enabled for a cgroup is
 695         * also enabled for all its descendant cgroups.  If @cpuctx's
 696         * cgroup is a descendant of @event's (the test covers identity
 697         * case), it's a match.
 698         */
 699        return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
 700                                    event->cgrp->css.cgroup);
 701}
 702
 703static inline void perf_detach_cgroup(struct perf_event *event)
 704{
 705        css_put(&event->cgrp->css);
 706        event->cgrp = NULL;
 707}
 708
 709static inline int is_cgroup_event(struct perf_event *event)
 710{
 711        return event->cgrp != NULL;
 712}
 713
 714static inline u64 perf_cgroup_event_time(struct perf_event *event)
 715{
 716        struct perf_cgroup_info *t;
 717
 718        t = per_cpu_ptr(event->cgrp->info, event->cpu);
 719        return t->time;
 720}
 721
 722static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
 723{
 724        struct perf_cgroup_info *info;
 725        u64 now;
 726
 727        now = perf_clock();
 728
 729        info = this_cpu_ptr(cgrp->info);
 730
 731        info->time += now - info->timestamp;
 732        info->timestamp = now;
 733}
 734
 735static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
 736{
 737        struct perf_cgroup *cgrp = cpuctx->cgrp;
 738        struct cgroup_subsys_state *css;
 739
 740        if (cgrp) {
 741                for (css = &cgrp->css; css; css = css->parent) {
 742                        cgrp = container_of(css, struct perf_cgroup, css);
 743                        __update_cgrp_time(cgrp);
 744                }
 745        }
 746}
 747
 748static inline void update_cgrp_time_from_event(struct perf_event *event)
 749{
 750        struct perf_cgroup *cgrp;
 751
 752        /*
 753         * ensure we access cgroup data only when needed and
 754         * when we know the cgroup is pinned (css_get)
 755         */
 756        if (!is_cgroup_event(event))
 757                return;
 758
 759        cgrp = perf_cgroup_from_task(current, event->ctx);
 760        /*
 761         * Do not update time when cgroup is not active
 762         */
 763        if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
 764                __update_cgrp_time(event->cgrp);
 765}
 766
 767static inline void
 768perf_cgroup_set_timestamp(struct task_struct *task,
 769                          struct perf_event_context *ctx)
 770{
 771        struct perf_cgroup *cgrp;
 772        struct perf_cgroup_info *info;
 773        struct cgroup_subsys_state *css;
 774
 775        /*
 776         * ctx->lock held by caller
 777         * ensure we do not access cgroup data
 778         * unless we have the cgroup pinned (css_get)
 779         */
 780        if (!task || !ctx->nr_cgroups)
 781                return;
 782
 783        cgrp = perf_cgroup_from_task(task, ctx);
 784
 785        for (css = &cgrp->css; css; css = css->parent) {
 786                cgrp = container_of(css, struct perf_cgroup, css);
 787                info = this_cpu_ptr(cgrp->info);
 788                info->timestamp = ctx->timestamp;
 789        }
 790}
 791
 792static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list);
 793
 794#define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
 795#define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
 796
 797/*
 798 * reschedule events based on the cgroup constraint of task.
 799 *
 800 * mode SWOUT : schedule out everything
 801 * mode SWIN : schedule in based on cgroup for next
 802 */
 803static void perf_cgroup_switch(struct task_struct *task, int mode)
 804{
 805        struct perf_cpu_context *cpuctx;
 806        struct list_head *list;
 807        unsigned long flags;
 808
 809        /*
 810         * Disable interrupts and preemption to avoid this CPU's
 811         * cgrp_cpuctx_entry to change under us.
 812         */
 813        local_irq_save(flags);
 814
 815        list = this_cpu_ptr(&cgrp_cpuctx_list);
 816        list_for_each_entry(cpuctx, list, cgrp_cpuctx_entry) {
 817                WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
 818
 819                perf_ctx_lock(cpuctx, cpuctx->task_ctx);
 820                perf_pmu_disable(cpuctx->ctx.pmu);
 821
 822                if (mode & PERF_CGROUP_SWOUT) {
 823                        cpu_ctx_sched_out(cpuctx, EVENT_ALL);
 824                        /*
 825                         * must not be done before ctxswout due
 826                         * to event_filter_match() in event_sched_out()
 827                         */
 828                        cpuctx->cgrp = NULL;
 829                }
 830
 831                if (mode & PERF_CGROUP_SWIN) {
 832                        WARN_ON_ONCE(cpuctx->cgrp);
 833                        /*
 834                         * set cgrp before ctxsw in to allow
 835                         * event_filter_match() to not have to pass
 836                         * task around
 837                         * we pass the cpuctx->ctx to perf_cgroup_from_task()
 838                         * because cgorup events are only per-cpu
 839                         */
 840                        cpuctx->cgrp = perf_cgroup_from_task(task,
 841                                                             &cpuctx->ctx);
 842                        cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
 843                }
 844                perf_pmu_enable(cpuctx->ctx.pmu);
 845                perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
 846        }
 847
 848        local_irq_restore(flags);
 849}
 850
 851static inline void perf_cgroup_sched_out(struct task_struct *task,
 852                                         struct task_struct *next)
 853{
 854        struct perf_cgroup *cgrp1;
 855        struct perf_cgroup *cgrp2 = NULL;
 856
 857        rcu_read_lock();
 858        /*
 859         * we come here when we know perf_cgroup_events > 0
 860         * we do not need to pass the ctx here because we know
 861         * we are holding the rcu lock
 862         */
 863        cgrp1 = perf_cgroup_from_task(task, NULL);
 864        cgrp2 = perf_cgroup_from_task(next, NULL);
 865
 866        /*
 867         * only schedule out current cgroup events if we know
 868         * that we are switching to a different cgroup. Otherwise,
 869         * do no touch the cgroup events.
 870         */
 871        if (cgrp1 != cgrp2)
 872                perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
 873
 874        rcu_read_unlock();
 875}
 876
 877static inline void perf_cgroup_sched_in(struct task_struct *prev,
 878                                        struct task_struct *task)
 879{
 880        struct perf_cgroup *cgrp1;
 881        struct perf_cgroup *cgrp2 = NULL;
 882
 883        rcu_read_lock();
 884        /*
 885         * we come here when we know perf_cgroup_events > 0
 886         * we do not need to pass the ctx here because we know
 887         * we are holding the rcu lock
 888         */
 889        cgrp1 = perf_cgroup_from_task(task, NULL);
 890        cgrp2 = perf_cgroup_from_task(prev, NULL);
 891
 892        /*
 893         * only need to schedule in cgroup events if we are changing
 894         * cgroup during ctxsw. Cgroup events were not scheduled
 895         * out of ctxsw out if that was not the case.
 896         */
 897        if (cgrp1 != cgrp2)
 898                perf_cgroup_switch(task, PERF_CGROUP_SWIN);
 899
 900        rcu_read_unlock();
 901}
 902
 903static int perf_cgroup_ensure_storage(struct perf_event *event,
 904                                struct cgroup_subsys_state *css)
 905{
 906        struct perf_cpu_context *cpuctx;
 907        struct perf_event **storage;
 908        int cpu, heap_size, ret = 0;
 909
 910        /*
 911         * Allow storage to have sufficent space for an iterator for each
 912         * possibly nested cgroup plus an iterator for events with no cgroup.
 913         */
 914        for (heap_size = 1; css; css = css->parent)
 915                heap_size++;
 916
 917        for_each_possible_cpu(cpu) {
 918                cpuctx = per_cpu_ptr(event->pmu->pmu_cpu_context, cpu);
 919                if (heap_size <= cpuctx->heap_size)
 920                        continue;
 921
 922                storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
 923                                       GFP_KERNEL, cpu_to_node(cpu));
 924                if (!storage) {
 925                        ret = -ENOMEM;
 926                        break;
 927                }
 928
 929                raw_spin_lock_irq(&cpuctx->ctx.lock);
 930                if (cpuctx->heap_size < heap_size) {
 931                        swap(cpuctx->heap, storage);
 932                        if (storage == cpuctx->heap_default)
 933                                storage = NULL;
 934                        cpuctx->heap_size = heap_size;
 935                }
 936                raw_spin_unlock_irq(&cpuctx->ctx.lock);
 937
 938                kfree(storage);
 939        }
 940
 941        return ret;
 942}
 943
 944static inline int perf_cgroup_connect(int fd, struct perf_event *event,
 945                                      struct perf_event_attr *attr,
 946                                      struct perf_event *group_leader)
 947{
 948        struct perf_cgroup *cgrp;
 949        struct cgroup_subsys_state *css;
 950        struct fd f = fdget(fd);
 951        int ret = 0;
 952
 953        if (!f.file)
 954                return -EBADF;
 955
 956        css = css_tryget_online_from_dir(f.file->f_path.dentry,
 957                                         &perf_event_cgrp_subsys);
 958        if (IS_ERR(css)) {
 959                ret = PTR_ERR(css);
 960                goto out;
 961        }
 962
 963        ret = perf_cgroup_ensure_storage(event, css);
 964        if (ret)
 965                goto out;
 966
 967        cgrp = container_of(css, struct perf_cgroup, css);
 968        event->cgrp = cgrp;
 969
 970        /*
 971         * all events in a group must monitor
 972         * the same cgroup because a task belongs
 973         * to only one perf cgroup at a time
 974         */
 975        if (group_leader && group_leader->cgrp != cgrp) {
 976                perf_detach_cgroup(event);
 977                ret = -EINVAL;
 978        }
 979out:
 980        fdput(f);
 981        return ret;
 982}
 983
 984static inline void
 985perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
 986{
 987        struct perf_cgroup_info *t;
 988        t = per_cpu_ptr(event->cgrp->info, event->cpu);
 989        event->shadow_ctx_time = now - t->timestamp;
 990}
 991
 992static inline void
 993perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
 994{
 995        struct perf_cpu_context *cpuctx;
 996
 997        if (!is_cgroup_event(event))
 998                return;
 999
1000        /*
1001         * Because cgroup events are always per-cpu events,
1002         * @ctx == &cpuctx->ctx.
1003         */
1004        cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1005
1006        /*
1007         * Since setting cpuctx->cgrp is conditional on the current @cgrp
1008         * matching the event's cgroup, we must do this for every new event,
1009         * because if the first would mismatch, the second would not try again
1010         * and we would leave cpuctx->cgrp unset.
1011         */
1012        if (ctx->is_active && !cpuctx->cgrp) {
1013                struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
1014
1015                if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
1016                        cpuctx->cgrp = cgrp;
1017        }
1018
1019        if (ctx->nr_cgroups++)
1020                return;
1021
1022        list_add(&cpuctx->cgrp_cpuctx_entry,
1023                        per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
1024}
1025
1026static inline void
1027perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1028{
1029        struct perf_cpu_context *cpuctx;
1030
1031        if (!is_cgroup_event(event))
1032                return;
1033
1034        /*
1035         * Because cgroup events are always per-cpu events,
1036         * @ctx == &cpuctx->ctx.
1037         */
1038        cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1039
1040        if (--ctx->nr_cgroups)
1041                return;
1042
1043        if (ctx->is_active && cpuctx->cgrp)
1044                cpuctx->cgrp = NULL;
1045
1046        list_del(&cpuctx->cgrp_cpuctx_entry);
1047}
1048
1049#else /* !CONFIG_CGROUP_PERF */
1050
1051static inline bool
1052perf_cgroup_match(struct perf_event *event)
1053{
1054        return true;
1055}
1056
1057static inline void perf_detach_cgroup(struct perf_event *event)
1058{}
1059
1060static inline int is_cgroup_event(struct perf_event *event)
1061{
1062        return 0;
1063}
1064
1065static inline void update_cgrp_time_from_event(struct perf_event *event)
1066{
1067}
1068
1069static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
1070{
1071}
1072
1073static inline void perf_cgroup_sched_out(struct task_struct *task,
1074                                         struct task_struct *next)
1075{
1076}
1077
1078static inline void perf_cgroup_sched_in(struct task_struct *prev,
1079                                        struct task_struct *task)
1080{
1081}
1082
1083static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
1084                                      struct perf_event_attr *attr,
1085                                      struct perf_event *group_leader)
1086{
1087        return -EINVAL;
1088}
1089
1090static inline void
1091perf_cgroup_set_timestamp(struct task_struct *task,
1092                          struct perf_event_context *ctx)
1093{
1094}
1095
1096static inline void
1097perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
1098{
1099}
1100
1101static inline void
1102perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
1103{
1104}
1105
1106static inline u64 perf_cgroup_event_time(struct perf_event *event)
1107{
1108        return 0;
1109}
1110
1111static inline void
1112perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
1113{
1114}
1115
1116static inline void
1117perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1118{
1119}
1120#endif
1121
1122/*
1123 * set default to be dependent on timer tick just
1124 * like original code
1125 */
1126#define PERF_CPU_HRTIMER (1000 / HZ)
1127/*
1128 * function must be called with interrupts disabled
1129 */
1130static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1131{
1132        struct perf_cpu_context *cpuctx;
1133        bool rotations;
1134
1135        lockdep_assert_irqs_disabled();
1136
1137        cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
1138        rotations = perf_rotate_context(cpuctx);
1139
1140        raw_spin_lock(&cpuctx->hrtimer_lock);
1141        if (rotations)
1142                hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
1143        else
1144                cpuctx->hrtimer_active = 0;
1145        raw_spin_unlock(&cpuctx->hrtimer_lock);
1146
1147        return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1148}
1149
1150static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
1151{
1152        struct hrtimer *timer = &cpuctx->hrtimer;
1153        struct pmu *pmu = cpuctx->ctx.pmu;
1154        u64 interval;
1155
1156        /* no multiplexing needed for SW PMU */
1157        if (pmu->task_ctx_nr == perf_sw_context)
1158                return;
1159
1160        /*
1161         * check default is sane, if not set then force to
1162         * default interval (1/tick)
1163         */
1164        interval = pmu->hrtimer_interval_ms;
1165        if (interval < 1)
1166                interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1167
1168        cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1169
1170        raw_spin_lock_init(&cpuctx->hrtimer_lock);
1171        hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1172        timer->function = perf_mux_hrtimer_handler;
1173}
1174
1175static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
1176{
1177        struct hrtimer *timer = &cpuctx->hrtimer;
1178        struct pmu *pmu = cpuctx->ctx.pmu;
1179        unsigned long flags;
1180
1181        /* not for SW PMU */
1182        if (pmu->task_ctx_nr == perf_sw_context)
1183                return 0;
1184
1185        raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);
1186        if (!cpuctx->hrtimer_active) {
1187                cpuctx->hrtimer_active = 1;
1188                hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
1189                hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
1190        }
1191        raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
1192
1193        return 0;
1194}
1195
1196void perf_pmu_disable(struct pmu *pmu)
1197{
1198        int *count = this_cpu_ptr(pmu->pmu_disable_count);
1199        if (!(*count)++)
1200                pmu->pmu_disable(pmu);
1201}
1202
1203void perf_pmu_enable(struct pmu *pmu)
1204{
1205        int *count = this_cpu_ptr(pmu->pmu_disable_count);
1206        if (!--(*count))
1207                pmu->pmu_enable(pmu);
1208}
1209
1210static DEFINE_PER_CPU(struct list_head, active_ctx_list);
1211
1212/*
1213 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and
1214 * perf_event_task_tick() are fully serialized because they're strictly cpu
1215 * affine and perf_event_ctx{activate,deactivate} are called with IRQs
1216 * disabled, while perf_event_task_tick is called from IRQ context.
1217 */
1218static void perf_event_ctx_activate(struct perf_event_context *ctx)
1219{
1220        struct list_head *head = this_cpu_ptr(&active_ctx_list);
1221
1222        lockdep_assert_irqs_disabled();
1223
1224        WARN_ON(!list_empty(&ctx->active_ctx_list));
1225
1226        list_add(&ctx->active_ctx_list, head);
1227}
1228
1229static void perf_event_ctx_deactivate(struct perf_event_context *ctx)
1230{
1231        lockdep_assert_irqs_disabled();
1232
1233        WARN_ON(list_empty(&ctx->active_ctx_list));
1234
1235        list_del_init(&ctx->active_ctx_list);
1236}
1237
1238static void get_ctx(struct perf_event_context *ctx)
1239{
1240        refcount_inc(&ctx->refcount);
1241}
1242
1243static void *alloc_task_ctx_data(struct pmu *pmu)
1244{
1245        if (pmu->task_ctx_cache)
1246                return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);
1247
1248        return NULL;
1249}
1250
1251static void free_task_ctx_data(struct pmu *pmu, void *task_ctx_data)
1252{
1253        if (pmu->task_ctx_cache && task_ctx_data)
1254                kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);
1255}
1256
1257static void free_ctx(struct rcu_head *head)
1258{
1259        struct perf_event_context *ctx;
1260
1261        ctx = container_of(head, struct perf_event_context, rcu_head);
1262        free_task_ctx_data(ctx->pmu, ctx->task_ctx_data);
1263        kfree(ctx);
1264}
1265
1266static void put_ctx(struct perf_event_context *ctx)
1267{
1268        if (refcount_dec_and_test(&ctx->refcount)) {
1269                if (ctx->parent_ctx)
1270                        put_ctx(ctx->parent_ctx);
1271                if (ctx->task && ctx->task != TASK_TOMBSTONE)
1272                        put_task_struct(ctx->task);
1273                call_rcu(&ctx->rcu_head, free_ctx);
1274        }
1275}
1276
1277/*
1278 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1279 * perf_pmu_migrate_context() we need some magic.
1280 *
1281 * Those places that change perf_event::ctx will hold both
1282 * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1283 *
1284 * Lock ordering is by mutex address. There are two other sites where
1285 * perf_event_context::mutex nests and those are:
1286 *
1287 *  - perf_event_exit_task_context()    [ child , 0 ]
1288 *      perf_event_exit_event()
1289 *        put_event()                   [ parent, 1 ]
1290 *
1291 *  - perf_event_init_context()         [ parent, 0 ]
1292 *      inherit_task_group()
1293 *        inherit_group()
1294 *          inherit_event()
1295 *            perf_event_alloc()
1296 *              perf_init_event()
1297 *                perf_try_init_event() [ child , 1 ]
1298 *
1299 * While it appears there is an obvious deadlock here -- the parent and child
1300 * nesting levels are inverted between the two. This is in fact safe because
1301 * life-time rules separate them. That is an exiting task cannot fork, and a
1302 * spawning task cannot (yet) exit.
1303 *
1304 * But remember that these are parent<->child context relations, and
1305 * migration does not affect children, therefore these two orderings should not
1306 * interact.
1307 *
1308 * The change in perf_event::ctx does not affect children (as claimed above)
1309 * because the sys_perf_event_open() case will install a new event and break
1310 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1311 * concerned with cpuctx and that doesn't have children.
1312 *
1313 * The places that change perf_event::ctx will issue:
1314 *
1315 *   perf_remove_from_context();
1316 *   synchronize_rcu();
1317 *   perf_install_in_context();
1318 *
1319 * to affect the change. The remove_from_context() + synchronize_rcu() should
1320 * quiesce the event, after which we can install it in the new location. This
1321 * means that only external vectors (perf_fops, prctl) can perturb the event
1322 * while in transit. Therefore all such accessors should also acquire
1323 * perf_event_context::mutex to serialize against this.
1324 *
1325 * However; because event->ctx can change while we're waiting to acquire
1326 * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1327 * function.
1328 *
1329 * Lock order:
1330 *    exec_update_lock
1331 *      task_struct::perf_event_mutex
1332 *        perf_event_context::mutex
1333 *          perf_event::child_mutex;
1334 *            perf_event_context::lock
1335 *          perf_event::mmap_mutex
1336 *          mmap_lock
1337 *            perf_addr_filters_head::lock
1338 *
1339 *    cpu_hotplug_lock
1340 *      pmus_lock
1341 *        cpuctx->mutex / perf_event_context::mutex
1342 */
1343static struct perf_event_context *
1344perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1345{
1346        struct perf_event_context *ctx;
1347
1348again:
1349        rcu_read_lock();
1350        ctx = READ_ONCE(event->ctx);
1351        if (!refcount_inc_not_zero(&ctx->refcount)) {
1352                rcu_read_unlock();
1353                goto again;
1354        }
1355        rcu_read_unlock();
1356
1357        mutex_lock_nested(&ctx->mutex, nesting);
1358        if (event->ctx != ctx) {
1359                mutex_unlock(&ctx->mutex);
1360                put_ctx(ctx);
1361                goto again;
1362        }
1363
1364        return ctx;
1365}
1366
1367static inline struct perf_event_context *
1368perf_event_ctx_lock(struct perf_event *event)
1369{
1370        return perf_event_ctx_lock_nested(event, 0);
1371}
1372
1373static void perf_event_ctx_unlock(struct perf_event *event,
1374                                  struct perf_event_context *ctx)
1375{
1376        mutex_unlock(&ctx->mutex);
1377        put_ctx(ctx);
1378}
1379
1380/*
1381 * This must be done under the ctx->lock, such as to serialize against
1382 * context_equiv(), therefore we cannot call put_ctx() since that might end up
1383 * calling scheduler related locks and ctx->lock nests inside those.
1384 */
1385static __must_check struct perf_event_context *
1386unclone_ctx(struct perf_event_context *ctx)
1387{
1388        struct perf_event_context *parent_ctx = ctx->parent_ctx;
1389
1390        lockdep_assert_held(&ctx->lock);
1391
1392        if (parent_ctx)
1393                ctx->parent_ctx = NULL;
1394        ctx->generation++;
1395
1396        return parent_ctx;
1397}
1398
1399static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,
1400                                enum pid_type type)
1401{
1402        u32 nr;
1403        /*
1404         * only top level events have the pid namespace they were created in
1405         */
1406        if (event->parent)
1407                event = event->parent;
1408
1409        nr = __task_pid_nr_ns(p, type, event->ns);
1410        /* avoid -1 if it is idle thread or runs in another ns */
1411        if (!nr && !pid_alive(p))
1412                nr = -1;
1413        return nr;
1414}
1415
1416static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1417{
1418        return perf_event_pid_type(event, p, PIDTYPE_TGID);
1419}
1420
1421static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1422{
1423        return perf_event_pid_type(event, p, PIDTYPE_PID);
1424}
1425
1426/*
1427 * If we inherit events we want to return the parent event id
1428 * to userspace.
1429 */
1430static u64 primary_event_id(struct perf_event *event)
1431{
1432        u64 id = event->id;
1433
1434        if (event->parent)
1435                id = event->parent->id;
1436
1437        return id;
1438}
1439
1440/*
1441 * Get the perf_event_context for a task and lock it.
1442 *
1443 * This has to cope with the fact that until it is locked,
1444 * the context could get moved to another task.
1445 */
1446static struct perf_event_context *
1447perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
1448{
1449        struct perf_event_context *ctx;
1450
1451retry:
1452        /*
1453         * One of the few rules of preemptible RCU is that one cannot do
1454         * rcu_read_unlock() while holding a scheduler (or nested) lock when
1455         * part of the read side critical section was irqs-enabled -- see
1456         * rcu_read_unlock_special().
1457         *
1458         * Since ctx->lock nests under rq->lock we must ensure the entire read
1459         * side critical section has interrupts disabled.
1460         */
1461        local_irq_save(*flags);
1462        rcu_read_lock();
1463        ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
1464        if (ctx) {
1465                /*
1466                 * If this context is a clone of another, it might
1467                 * get swapped for another underneath us by
1468                 * perf_event_task_sched_out, though the
1469                 * rcu_read_lock() protects us from any context
1470                 * getting freed.  Lock the context and check if it
1471                 * got swapped before we could get the lock, and retry
1472                 * if so.  If we locked the right context, then it
1473                 * can't get swapped on us any more.
1474                 */
1475                raw_spin_lock(&ctx->lock);
1476                if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1477                        raw_spin_unlock(&ctx->lock);
1478                        rcu_read_unlock();
1479                        local_irq_restore(*flags);
1480                        goto retry;
1481                }
1482
1483                if (ctx->task == TASK_TOMBSTONE ||
1484                    !refcount_inc_not_zero(&ctx->refcount)) {
1485                        raw_spin_unlock(&ctx->lock);
1486                        ctx = NULL;
1487                } else {
1488                        WARN_ON_ONCE(ctx->task != task);
1489                }
1490        }
1491        rcu_read_unlock();
1492        if (!ctx)
1493                local_irq_restore(*flags);
1494        return ctx;
1495}
1496
1497/*
1498 * Get the context for a task and increment its pin_count so it
1499 * can't get swapped to another task.  This also increments its
1500 * reference count so that the context can't get freed.
1501 */
1502static struct perf_event_context *
1503perf_pin_task_context(struct task_struct *task, int ctxn)
1504{
1505        struct perf_event_context *ctx;
1506        unsigned long flags;
1507
1508        ctx = perf_lock_task_context(task, ctxn, &flags);
1509        if (ctx) {
1510                ++ctx->pin_count;
1511                raw_spin_unlock_irqrestore(&ctx->lock, flags);
1512        }
1513        return ctx;
1514}
1515
1516static void perf_unpin_context(struct perf_event_context *ctx)
1517{
1518        unsigned long flags;
1519
1520        raw_spin_lock_irqsave(&ctx->lock, flags);
1521        --ctx->pin_count;
1522        raw_spin_unlock_irqrestore(&ctx->lock, flags);
1523}
1524
1525/*
1526 * Update the record of the current time in a context.
1527 */
1528static void update_context_time(struct perf_event_context *ctx)
1529{
1530        u64 now = perf_clock();
1531
1532        ctx->time += now - ctx->timestamp;
1533        ctx->timestamp = now;
1534}
1535
1536static u64 perf_event_time(struct perf_event *event)
1537{
1538        struct perf_event_context *ctx = event->ctx;
1539
1540        if (is_cgroup_event(event))
1541                return perf_cgroup_event_time(event);
1542
1543        return ctx ? ctx->time : 0;
1544}
1545
1546static enum event_type_t get_event_type(struct perf_event *event)
1547{
1548        struct perf_event_context *ctx = event->ctx;
1549        enum event_type_t event_type;
1550
1551        lockdep_assert_held(&ctx->lock);
1552
1553        /*
1554         * It's 'group type', really, because if our group leader is
1555         * pinned, so are we.
1556         */
1557        if (event->group_leader != event)
1558                event = event->group_leader;
1559
1560        event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
1561        if (!ctx->task)
1562                event_type |= EVENT_CPU;
1563
1564        return event_type;
1565}
1566
1567/*
1568 * Helper function to initialize event group nodes.
1569 */
1570static void init_event_group(struct perf_event *event)
1571{
1572        RB_CLEAR_NODE(&event->group_node);
1573        event->group_index = 0;
1574}
1575
1576/*
1577 * Extract pinned or flexible groups from the context
1578 * based on event attrs bits.
1579 */
1580static struct perf_event_groups *
1581get_event_groups(struct perf_event *event, struct perf_event_context *ctx)
1582{
1583        if (event->attr.pinned)
1584                return &ctx->pinned_groups;
1585        else
1586                return &ctx->flexible_groups;
1587}
1588
1589/*
1590 * Helper function to initializes perf_event_group trees.
1591 */
1592static void perf_event_groups_init(struct perf_event_groups *groups)
1593{
1594        groups->tree = RB_ROOT;
1595        groups->index = 0;
1596}
1597
1598static inline struct cgroup *event_cgroup(const struct perf_event *event)
1599{
1600        struct cgroup *cgroup = NULL;
1601
1602#ifdef CONFIG_CGROUP_PERF
1603        if (event->cgrp)
1604                cgroup = event->cgrp->css.cgroup;
1605#endif
1606
1607        return cgroup;
1608}
1609
1610/*
1611 * Compare function for event groups;
1612 *
1613 * Implements complex key that first sorts by CPU and then by virtual index
1614 * which provides ordering when rotating groups for the same CPU.
1615 */
1616static __always_inline int
1617perf_event_groups_cmp(const int left_cpu, const struct cgroup *left_cgroup,
1618                      const u64 left_group_index, const struct perf_event *right)
1619{
1620        if (left_cpu < right->cpu)
1621                return -1;
1622        if (left_cpu > right->cpu)
1623                return 1;
1624
1625#ifdef CONFIG_CGROUP_PERF
1626        {
1627                const struct cgroup *right_cgroup = event_cgroup(right);
1628
1629                if (left_cgroup != right_cgroup) {
1630                        if (!left_cgroup) {
1631                                /*
1632                                 * Left has no cgroup but right does, no
1633                                 * cgroups come first.
1634                                 */
1635                                return -1;
1636                        }
1637                        if (!right_cgroup) {
1638                                /*
1639                                 * Right has no cgroup but left does, no
1640                                 * cgroups come first.
1641                                 */
1642                                return 1;
1643                        }
1644                        /* Two dissimilar cgroups, order by id. */
1645                        if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup))
1646                                return -1;
1647
1648                        return 1;
1649                }
1650        }
1651#endif
1652
1653        if (left_group_index < right->group_index)
1654                return -1;
1655        if (left_group_index > right->group_index)
1656                return 1;
1657
1658        return 0;
1659}
1660
1661#define __node_2_pe(node) \
1662        rb_entry((node), struct perf_event, group_node)
1663
1664static inline bool __group_less(struct rb_node *a, const struct rb_node *b)
1665{
1666        struct perf_event *e = __node_2_pe(a);
1667        return perf_event_groups_cmp(e->cpu, event_cgroup(e), e->group_index,
1668                                     __node_2_pe(b)) < 0;
1669}
1670
1671struct __group_key {
1672        int cpu;
1673        struct cgroup *cgroup;
1674};
1675
1676static inline int __group_cmp(const void *key, const struct rb_node *node)
1677{
1678        const struct __group_key *a = key;
1679        const struct perf_event *b = __node_2_pe(node);
1680
1681        /* partial/subtree match: @cpu, @cgroup; ignore: @group_index */
1682        return perf_event_groups_cmp(a->cpu, a->cgroup, b->group_index, b);
1683}
1684
1685/*
1686 * Insert @event into @groups' tree; using {@event->cpu, ++@groups->index} for
1687 * key (see perf_event_groups_less). This places it last inside the CPU
1688 * subtree.
1689 */
1690static void
1691perf_event_groups_insert(struct perf_event_groups *groups,
1692                         struct perf_event *event)
1693{
1694        event->group_index = ++groups->index;
1695
1696        rb_add(&event->group_node, &groups->tree, __group_less);
1697}
1698
1699/*
1700 * Helper function to insert event into the pinned or flexible groups.
1701 */
1702static void
1703add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)
1704{
1705        struct perf_event_groups *groups;
1706
1707        groups = get_event_groups(event, ctx);
1708        perf_event_groups_insert(groups, event);
1709}
1710
1711/*
1712 * Delete a group from a tree.
1713 */
1714static void
1715perf_event_groups_delete(struct perf_event_groups *groups,
1716                         struct perf_event *event)
1717{
1718        WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
1719                     RB_EMPTY_ROOT(&groups->tree));
1720
1721        rb_erase(&event->group_node, &groups->tree);
1722        init_event_group(event);
1723}
1724
1725/*
1726 * Helper function to delete event from its groups.
1727 */
1728static void
1729del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)
1730{
1731        struct perf_event_groups *groups;
1732
1733        groups = get_event_groups(event, ctx);
1734        perf_event_groups_delete(groups, event);
1735}
1736
1737/*
1738 * Get the leftmost event in the cpu/cgroup subtree.
1739 */
1740static struct perf_event *
1741perf_event_groups_first(struct perf_event_groups *groups, int cpu,
1742                        struct cgroup *cgrp)
1743{
1744        struct __group_key key = {
1745                .cpu = cpu,
1746                .cgroup = cgrp,
1747        };
1748        struct rb_node *node;
1749
1750        node = rb_find_first(&key, &groups->tree, __group_cmp);
1751        if (node)
1752                return __node_2_pe(node);
1753
1754        return NULL;
1755}
1756
1757/*
1758 * Like rb_entry_next_safe() for the @cpu subtree.
1759 */
1760static struct perf_event *
1761perf_event_groups_next(struct perf_event *event)
1762{
1763        struct __group_key key = {
1764                .cpu = event->cpu,
1765                .cgroup = event_cgroup(event),
1766        };
1767        struct rb_node *next;
1768
1769        next = rb_next_match(&key, &event->group_node, __group_cmp);
1770        if (next)
1771                return __node_2_pe(next);
1772
1773        return NULL;
1774}
1775
1776/*
1777 * Iterate through the whole groups tree.
1778 */
1779#define perf_event_groups_for_each(event, groups)                       \
1780        for (event = rb_entry_safe(rb_first(&((groups)->tree)),         \
1781                                typeof(*event), group_node); event;     \
1782                event = rb_entry_safe(rb_next(&event->group_node),      \
1783                                typeof(*event), group_node))
1784
1785/*
1786 * Add an event from the lists for its context.
1787 * Must be called with ctx->mutex and ctx->lock held.
1788 */
1789static void
1790list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1791{
1792        lockdep_assert_held(&ctx->lock);
1793
1794        WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1795        event->attach_state |= PERF_ATTACH_CONTEXT;
1796
1797        event->tstamp = perf_event_time(event);
1798
1799        /*
1800         * If we're a stand alone event or group leader, we go to the context
1801         * list, group events are kept attached to the group so that
1802         * perf_group_detach can, at all times, locate all siblings.
1803         */
1804        if (event->group_leader == event) {
1805                event->group_caps = event->event_caps;
1806                add_event_to_groups(event, ctx);
1807        }
1808
1809        list_add_rcu(&event->event_entry, &ctx->event_list);
1810        ctx->nr_events++;
1811        if (event->attr.inherit_stat)
1812                ctx->nr_stat++;
1813
1814        if (event->state > PERF_EVENT_STATE_OFF)
1815                perf_cgroup_event_enable(event, ctx);
1816
1817        ctx->generation++;
1818}
1819
1820/*
1821 * Initialize event state based on the perf_event_attr::disabled.
1822 */
1823static inline void perf_event__state_init(struct perf_event *event)
1824{
1825        event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1826                                              PERF_EVENT_STATE_INACTIVE;
1827}
1828
1829static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
1830{
1831        int entry = sizeof(u64); /* value */
1832        int size = 0;
1833        int nr = 1;
1834
1835        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1836                size += sizeof(u64);
1837
1838        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1839                size += sizeof(u64);
1840
1841        if (event->attr.read_format & PERF_FORMAT_ID)
1842                entry += sizeof(u64);
1843
1844        if (event->attr.read_format & PERF_FORMAT_GROUP) {
1845                nr += nr_siblings;
1846                size += sizeof(u64);
1847        }
1848
1849        size += entry * nr;
1850        event->read_size = size;
1851}
1852
1853static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1854{
1855        struct perf_sample_data *data;
1856        u16 size = 0;
1857
1858        if (sample_type & PERF_SAMPLE_IP)
1859                size += sizeof(data->ip);
1860
1861        if (sample_type & PERF_SAMPLE_ADDR)
1862                size += sizeof(data->addr);
1863
1864        if (sample_type & PERF_SAMPLE_PERIOD)
1865                size += sizeof(data->period);
1866
1867        if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
1868                size += sizeof(data->weight.full);
1869
1870        if (sample_type & PERF_SAMPLE_READ)
1871                size += event->read_size;
1872
1873        if (sample_type & PERF_SAMPLE_DATA_SRC)
1874                size += sizeof(data->data_src.val);
1875
1876        if (sample_type & PERF_SAMPLE_TRANSACTION)
1877                size += sizeof(data->txn);
1878
1879        if (sample_type & PERF_SAMPLE_PHYS_ADDR)
1880                size += sizeof(data->phys_addr);
1881
1882        if (sample_type & PERF_SAMPLE_CGROUP)
1883                size += sizeof(data->cgroup);
1884
1885        if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
1886                size += sizeof(data->data_page_size);
1887
1888        if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
1889                size += sizeof(data->code_page_size);
1890
1891        event->header_size = size;
1892}
1893
1894/*
1895 * Called at perf_event creation and when events are attached/detached from a
1896 * group.
1897 */
1898static void perf_event__header_size(struct perf_event *event)
1899{
1900        __perf_event_read_size(event,
1901                               event->group_leader->nr_siblings);
1902        __perf_event_header_size(event, event->attr.sample_type);
1903}
1904
1905static void perf_event__id_header_size(struct perf_event *event)
1906{
1907        struct perf_sample_data *data;
1908        u64 sample_type = event->attr.sample_type;
1909        u16 size = 0;
1910
1911        if (sample_type & PERF_SAMPLE_TID)
1912                size += sizeof(data->tid_entry);
1913
1914        if (sample_type & PERF_SAMPLE_TIME)
1915                size += sizeof(data->time);
1916
1917        if (sample_type & PERF_SAMPLE_IDENTIFIER)
1918                size += sizeof(data->id);
1919
1920        if (sample_type & PERF_SAMPLE_ID)
1921                size += sizeof(data->id);
1922
1923        if (sample_type & PERF_SAMPLE_STREAM_ID)
1924                size += sizeof(data->stream_id);
1925
1926        if (sample_type & PERF_SAMPLE_CPU)
1927                size += sizeof(data->cpu_entry);
1928
1929        event->id_header_size = size;
1930}
1931
1932static bool perf_event_validate_size(struct perf_event *event)
1933{
1934        /*
1935         * The values computed here will be over-written when we actually
1936         * attach the event.
1937         */
1938        __perf_event_read_size(event, event->group_leader->nr_siblings + 1);
1939        __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
1940        perf_event__id_header_size(event);
1941
1942        /*
1943         * Sum the lot; should not exceed the 64k limit we have on records.
1944         * Conservative limit to allow for callchains and other variable fields.
1945         */
1946        if (event->read_size + event->header_size +
1947            event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
1948                return false;
1949
1950        return true;
1951}
1952
1953static void perf_group_attach(struct perf_event *event)
1954{
1955        struct perf_event *group_leader = event->group_leader, *pos;
1956
1957        lockdep_assert_held(&event->ctx->lock);
1958
1959        /*
1960         * We can have double attach due to group movement in perf_event_open.
1961         */
1962        if (event->attach_state & PERF_ATTACH_GROUP)
1963                return;
1964
1965        event->attach_state |= PERF_ATTACH_GROUP;
1966
1967        if (group_leader == event)
1968                return;
1969
1970        WARN_ON_ONCE(group_leader->ctx != event->ctx);
1971
1972        group_leader->group_caps &= event->event_caps;
1973
1974        list_add_tail(&event->sibling_list, &group_leader->sibling_list);
1975        group_leader->nr_siblings++;
1976
1977        perf_event__header_size(group_leader);
1978
1979        for_each_sibling_event(pos, group_leader)
1980                perf_event__header_size(pos);
1981}
1982
1983/*
1984 * Remove an event from the lists for its context.
1985 * Must be called with ctx->mutex and ctx->lock held.
1986 */
1987static void
1988list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1989{
1990        WARN_ON_ONCE(event->ctx != ctx);
1991        lockdep_assert_held(&ctx->lock);
1992
1993        /*
1994         * We can have double detach due to exit/hot-unplug + close.
1995         */
1996        if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1997                return;
1998
1999        event->attach_state &= ~PERF_ATTACH_CONTEXT;
2000
2001        ctx->nr_events--;
2002        if (event->attr.inherit_stat)
2003                ctx->nr_stat--;
2004
2005        list_del_rcu(&event->event_entry);
2006
2007        if (event->group_leader == event)
2008                del_event_from_groups(event, ctx);
2009
2010        /*
2011         * If event was in error state, then keep it
2012         * that way, otherwise bogus counts will be
2013         * returned on read(). The only way to get out
2014         * of error state is by explicit re-enabling
2015         * of the event
2016         */
2017        if (event->state > PERF_EVENT_STATE_OFF) {
2018                perf_cgroup_event_disable(event, ctx);
2019                perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2020        }
2021
2022        ctx->generation++;
2023}
2024
2025static int
2026perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)
2027{
2028        if (!has_aux(aux_event))
2029                return 0;
2030
2031        if (!event->pmu->aux_output_match)
2032                return 0;
2033
2034        return event->pmu->aux_output_match(aux_event);
2035}
2036
2037static void put_event(struct perf_event *event);
2038static void event_sched_out(struct perf_event *event,
2039                            struct perf_cpu_context *cpuctx,
2040                            struct perf_event_context *ctx);
2041
2042static void perf_put_aux_event(struct perf_event *event)
2043{
2044        struct perf_event_context *ctx = event->ctx;
2045        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2046        struct perf_event *iter;
2047
2048        /*
2049         * If event uses aux_event tear down the link
2050         */
2051        if (event->aux_event) {
2052                iter = event->aux_event;
2053                event->aux_event = NULL;
2054                put_event(iter);
2055                return;
2056        }
2057
2058        /*
2059         * If the event is an aux_event, tear down all links to
2060         * it from other events.
2061         */
2062        for_each_sibling_event(iter, event->group_leader) {
2063                if (iter->aux_event != event)
2064                        continue;
2065
2066                iter->aux_event = NULL;
2067                put_event(event);
2068
2069                /*
2070                 * If it's ACTIVE, schedule it out and put it into ERROR
2071                 * state so that we don't try to schedule it again. Note
2072                 * that perf_event_enable() will clear the ERROR status.
2073                 */
2074                event_sched_out(iter, cpuctx, ctx);
2075                perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
2076        }
2077}
2078
2079static bool perf_need_aux_event(struct perf_event *event)
2080{
2081        return !!event->attr.aux_output || !!event->attr.aux_sample_size;
2082}
2083
2084static int perf_get_aux_event(struct perf_event *event,
2085                              struct perf_event *group_leader)
2086{
2087        /*
2088         * Our group leader must be an aux event if we want to be
2089         * an aux_output. This way, the aux event will precede its
2090         * aux_output events in the group, and therefore will always
2091         * schedule first.
2092         */
2093        if (!group_leader)
2094                return 0;
2095
2096        /*
2097         * aux_output and aux_sample_size are mutually exclusive.
2098         */
2099        if (event->attr.aux_output && event->attr.aux_sample_size)
2100                return 0;
2101
2102        if (event->attr.aux_output &&
2103            !perf_aux_output_match(event, group_leader))
2104                return 0;
2105
2106        if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
2107                return 0;
2108
2109        if (!atomic_long_inc_not_zero(&group_leader->refcount))
2110                return 0;
2111
2112        /*
2113         * Link aux_outputs to their aux event; this is undone in
2114         * perf_group_detach() by perf_put_aux_event(). When the
2115         * group in torn down, the aux_output events loose their
2116         * link to the aux_event and can't schedule any more.
2117         */
2118        event->aux_event = group_leader;
2119
2120        return 1;
2121}
2122
2123static inline struct list_head *get_event_list(struct perf_event *event)
2124{
2125        struct perf_event_context *ctx = event->ctx;
2126        return event->attr.pinned ? &ctx->pinned_active : &ctx->flexible_active;
2127}
2128
2129/*
2130 * Events that have PERF_EV_CAP_SIBLING require being part of a group and
2131 * cannot exist on their own, schedule them out and move them into the ERROR
2132 * state. Also see _perf_event_enable(), it will not be able to recover
2133 * this ERROR state.
2134 */
2135static inline void perf_remove_sibling_event(struct perf_event *event)
2136{
2137        struct perf_event_context *ctx = event->ctx;
2138        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2139
2140        event_sched_out(event, cpuctx, ctx);
2141        perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
2142}
2143
2144static void perf_group_detach(struct perf_event *event)
2145{
2146        struct perf_event *leader = event->group_leader;
2147        struct perf_event *sibling, *tmp;
2148        struct perf_event_context *ctx = event->ctx;
2149
2150        lockdep_assert_held(&ctx->lock);
2151
2152        /*
2153         * We can have double detach due to exit/hot-unplug + close.
2154         */
2155        if (!(event->attach_state & PERF_ATTACH_GROUP))
2156                return;
2157
2158        event->attach_state &= ~PERF_ATTACH_GROUP;
2159
2160        perf_put_aux_event(event);
2161
2162        /*
2163         * If this is a sibling, remove it from its group.
2164         */
2165        if (leader != event) {
2166                list_del_init(&event->sibling_list);
2167                event->group_leader->nr_siblings--;
2168                goto out;
2169        }
2170
2171        /*
2172         * If this was a group event with sibling events then
2173         * upgrade the siblings to singleton events by adding them
2174         * to whatever list we are on.
2175         */
2176        list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
2177
2178                if (sibling->event_caps & PERF_EV_CAP_SIBLING)
2179                        perf_remove_sibling_event(sibling);
2180
2181                sibling->group_leader = sibling;
2182                list_del_init(&sibling->sibling_list);
2183
2184                /* Inherit group flags from the previous leader */
2185                sibling->group_caps = event->group_caps;
2186
2187                if (!RB_EMPTY_NODE(&event->group_node)) {
2188                        add_event_to_groups(sibling, event->ctx);
2189
2190                        if (sibling->state == PERF_EVENT_STATE_ACTIVE)
2191                                list_add_tail(&sibling->active_list, get_event_list(sibling));
2192                }
2193
2194                WARN_ON_ONCE(sibling->ctx != event->ctx);
2195        }
2196
2197out:
2198        for_each_sibling_event(tmp, leader)
2199                perf_event__header_size(tmp);
2200
2201        perf_event__header_size(leader);
2202}
2203
2204static void sync_child_event(struct perf_event *child_event);
2205
2206static void perf_child_detach(struct perf_event *event)
2207{
2208        struct perf_event *parent_event = event->parent;
2209
2210        if (!(event->attach_state & PERF_ATTACH_CHILD))
2211                return;
2212
2213        event->attach_state &= ~PERF_ATTACH_CHILD;
2214
2215        if (WARN_ON_ONCE(!parent_event))
2216                return;
2217
2218        lockdep_assert_held(&parent_event->child_mutex);
2219
2220        sync_child_event(event);
2221        list_del_init(&event->child_list);
2222}
2223
2224static bool is_orphaned_event(struct perf_event *event)
2225{
2226        return event->state == PERF_EVENT_STATE_DEAD;
2227}
2228
2229static inline int __pmu_filter_match(struct perf_event *event)
2230{
2231        struct pmu *pmu = event->pmu;
2232        return pmu->filter_match ? pmu->filter_match(event) : 1;
2233}
2234
2235/*
2236 * Check whether we should attempt to schedule an event group based on
2237 * PMU-specific filtering. An event group can consist of HW and SW events,
2238 * potentially with a SW leader, so we must check all the filters, to
2239 * determine whether a group is schedulable:
2240 */
2241static inline int pmu_filter_match(struct perf_event *event)
2242{
2243        struct perf_event *sibling;
2244
2245        if (!__pmu_filter_match(event))
2246                return 0;
2247
2248        for_each_sibling_event(sibling, event) {
2249                if (!__pmu_filter_match(sibling))
2250                        return 0;
2251        }
2252
2253        return 1;
2254}
2255
2256static inline int
2257event_filter_match(struct perf_event *event)
2258{
2259        return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
2260               perf_cgroup_match(event) && pmu_filter_match(event);
2261}
2262
2263static void
2264event_sched_out(struct perf_event *event,
2265                  struct perf_cpu_context *cpuctx,
2266                  struct perf_event_context *ctx)
2267{
2268        enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
2269
2270        WARN_ON_ONCE(event->ctx != ctx);
2271        lockdep_assert_held(&ctx->lock);
2272
2273        if (event->state != PERF_EVENT_STATE_ACTIVE)
2274                return;
2275
2276        /*
2277         * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
2278         * we can schedule events _OUT_ individually through things like
2279         * __perf_remove_from_context().
2280         */
2281        list_del_init(&event->active_list);
2282
2283        perf_pmu_disable(event->pmu);
2284
2285        event->pmu->del(event, 0);
2286        event->oncpu = -1;
2287
2288        if (READ_ONCE(event->pending_disable) >= 0) {
2289                WRITE_ONCE(event->pending_disable, -1);
2290                perf_cgroup_event_disable(event, ctx);
2291                state = PERF_EVENT_STATE_OFF;
2292        }
2293        perf_event_set_state(event, state);
2294
2295        if (!is_software_event(event))
2296                cpuctx->active_oncpu--;
2297        if (!--ctx->nr_active)
2298                perf_event_ctx_deactivate(ctx);
2299        if (event->attr.freq && event->attr.sample_freq)
2300                ctx->nr_freq--;
2301        if (event->attr.exclusive || !cpuctx->active_oncpu)
2302                cpuctx->exclusive = 0;
2303
2304        perf_pmu_enable(event->pmu);
2305}
2306
2307static void
2308group_sched_out(struct perf_event *group_event,
2309                struct perf_cpu_context *cpuctx,
2310                struct perf_event_context *ctx)
2311{
2312        struct perf_event *event;
2313
2314        if (group_event->state != PERF_EVENT_STATE_ACTIVE)
2315                return;
2316
2317        perf_pmu_disable(ctx->pmu);
2318
2319        event_sched_out(group_event, cpuctx, ctx);
2320
2321        /*
2322         * Schedule out siblings (if any):
2323         */
2324        for_each_sibling_event(event, group_event)
2325                event_sched_out(event, cpuctx, ctx);
2326
2327        perf_pmu_enable(ctx->pmu);
2328}
2329
2330#define DETACH_GROUP    0x01UL
2331#define DETACH_CHILD    0x02UL
2332
2333/*
2334 * Cross CPU call to remove a performance event
2335 *
2336 * We disable the event on the hardware level first. After that we
2337 * remove it from the context list.
2338 */
2339static void
2340__perf_remove_from_context(struct perf_event *event,
2341                           struct perf_cpu_context *cpuctx,
2342                           struct perf_event_context *ctx,
2343                           void *info)
2344{
2345        unsigned long flags = (unsigned long)info;
2346
2347        if (ctx->is_active & EVENT_TIME) {
2348                update_context_time(ctx);
2349                update_cgrp_time_from_cpuctx(cpuctx);
2350        }
2351
2352        event_sched_out(event, cpuctx, ctx);
2353        if (flags & DETACH_GROUP)
2354                perf_group_detach(event);
2355        if (flags & DETACH_CHILD)
2356                perf_child_detach(event);
2357        list_del_event(event, ctx);
2358
2359        if (!ctx->nr_events && ctx->is_active) {
2360                ctx->is_active = 0;
2361                ctx->rotate_necessary = 0;
2362                if (ctx->task) {
2363                        WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2364                        cpuctx->task_ctx = NULL;
2365                }
2366        }
2367}
2368
2369/*
2370 * Remove the event from a task's (or a CPU's) list of events.
2371 *
2372 * If event->ctx is a cloned context, callers must make sure that
2373 * every task struct that event->ctx->task could possibly point to
2374 * remains valid.  This is OK when called from perf_release since
2375 * that only calls us on the top-level context, which can't be a clone.
2376 * When called from perf_event_exit_task, it's OK because the
2377 * context has been detached from its task.
2378 */
2379static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
2380{
2381        struct perf_event_context *ctx = event->ctx;
2382
2383        lockdep_assert_held(&ctx->mutex);
2384
2385        /*
2386         * Because of perf_event_exit_task(), perf_remove_from_context() ought
2387         * to work in the face of TASK_TOMBSTONE, unlike every other
2388         * event_function_call() user.
2389         */
2390        raw_spin_lock_irq(&ctx->lock);
2391        if (!ctx->is_active) {
2392                __perf_remove_from_context(event, __get_cpu_context(ctx),
2393                                           ctx, (void *)flags);
2394                raw_spin_unlock_irq(&ctx->lock);
2395                return;
2396        }
2397        raw_spin_unlock_irq(&ctx->lock);
2398
2399        event_function_call(event, __perf_remove_from_context, (void *)flags);
2400}
2401
2402/*
2403 * Cross CPU call to disable a performance event
2404 */
2405static void __perf_event_disable(struct perf_event *event,
2406                                 struct perf_cpu_context *cpuctx,
2407                                 struct perf_event_context *ctx,
2408                                 void *info)
2409{
2410        if (event->state < PERF_EVENT_STATE_INACTIVE)
2411                return;
2412
2413        if (ctx->is_active & EVENT_TIME) {
2414                update_context_time(ctx);
2415                update_cgrp_time_from_event(event);
2416        }
2417
2418        if (event == event->group_leader)
2419                group_sched_out(event, cpuctx, ctx);
2420        else
2421                event_sched_out(event, cpuctx, ctx);
2422
2423        perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2424        perf_cgroup_event_disable(event, ctx);
2425}
2426
2427/*
2428 * Disable an event.
2429 *
2430 * If event->ctx is a cloned context, callers must make sure that
2431 * every task struct that event->ctx->task could possibly point to
2432 * remains valid.  This condition is satisfied when called through
2433 * perf_event_for_each_child or perf_event_for_each because they
2434 * hold the top-level event's child_mutex, so any descendant that
2435 * goes to exit will block in perf_event_exit_event().
2436 *
2437 * When called from perf_pending_event it's OK because event->ctx
2438 * is the current context on this CPU and preemption is disabled,
2439 * hence we can't get into perf_event_task_sched_out for this context.
2440 */
2441static void _perf_event_disable(struct perf_event *event)
2442{
2443        struct perf_event_context *ctx = event->ctx;
2444
2445        raw_spin_lock_irq(&ctx->lock);
2446        if (event->state <= PERF_EVENT_STATE_OFF) {
2447                raw_spin_unlock_irq(&ctx->lock);
2448                return;
2449        }
2450        raw_spin_unlock_irq(&ctx->lock);
2451
2452        event_function_call(event, __perf_event_disable, NULL);
2453}
2454
2455void perf_event_disable_local(struct perf_event *event)
2456{
2457        event_function_local(event, __perf_event_disable, NULL);
2458}
2459
2460/*
2461 * Strictly speaking kernel users cannot create groups and therefore this
2462 * interface does not need the perf_event_ctx_lock() magic.
2463 */
2464void perf_event_disable(struct perf_event *event)
2465{
2466        struct perf_event_context *ctx;
2467
2468        ctx = perf_event_ctx_lock(event);
2469        _perf_event_disable(event);
2470        perf_event_ctx_unlock(event, ctx);
2471}
2472EXPORT_SYMBOL_GPL(perf_event_disable);
2473
2474void perf_event_disable_inatomic(struct perf_event *event)
2475{
2476        WRITE_ONCE(event->pending_disable, smp_processor_id());
2477        /* can fail, see perf_pending_event_disable() */
2478        irq_work_queue(&event->pending);
2479}
2480
2481static void perf_set_shadow_time(struct perf_event *event,
2482                                 struct perf_event_context *ctx)
2483{
2484        /*
2485         * use the correct time source for the time snapshot
2486         *
2487         * We could get by without this by leveraging the
2488         * fact that to get to this function, the caller
2489         * has most likely already called update_context_time()
2490         * and update_cgrp_time_xx() and thus both timestamp
2491         * are identical (or very close). Given that tstamp is,
2492         * already adjusted for cgroup, we could say that:
2493         *    tstamp - ctx->timestamp
2494         * is equivalent to
2495         *    tstamp - cgrp->timestamp.
2496         *
2497         * Then, in perf_output_read(), the calculation would
2498         * work with no changes because:
2499         * - event is guaranteed scheduled in
2500         * - no scheduled out in between
2501         * - thus the timestamp would be the same
2502         *
2503         * But this is a bit hairy.
2504         *
2505         * So instead, we have an explicit cgroup call to remain
2506         * within the time source all along. We believe it
2507         * is cleaner and simpler to understand.
2508         */
2509        if (is_cgroup_event(event))
2510                perf_cgroup_set_shadow_time(event, event->tstamp);
2511        else
2512                event->shadow_ctx_time = event->tstamp - ctx->timestamp;
2513}
2514
2515#define MAX_INTERRUPTS (~0ULL)
2516
2517static void perf_log_throttle(struct perf_event *event, int enable);
2518static void perf_log_itrace_start(struct perf_event *event);
2519
2520static int
2521event_sched_in(struct perf_event *event,
2522                 struct perf_cpu_context *cpuctx,
2523                 struct perf_event_context *ctx)
2524{
2525        int ret = 0;
2526
2527        WARN_ON_ONCE(event->ctx != ctx);
2528
2529        lockdep_assert_held(&ctx->lock);
2530
2531        if (event->state <= PERF_EVENT_STATE_OFF)
2532                return 0;
2533
2534        WRITE_ONCE(event->oncpu, smp_processor_id());
2535        /*
2536         * Order event::oncpu write to happen before the ACTIVE state is
2537         * visible. This allows perf_event_{stop,read}() to observe the correct
2538         * ->oncpu if it sees ACTIVE.
2539         */
2540        smp_wmb();
2541        perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
2542
2543        /*
2544         * Unthrottle events, since we scheduled we might have missed several
2545         * ticks already, also for a heavily scheduling task there is little
2546         * guarantee it'll get a tick in a timely manner.
2547         */
2548        if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
2549                perf_log_throttle(event, 1);
2550                event->hw.interrupts = 0;
2551        }
2552
2553        perf_pmu_disable(event->pmu);
2554
2555        perf_set_shadow_time(event, ctx);
2556
2557        perf_log_itrace_start(event);
2558
2559        if (event->pmu->add(event, PERF_EF_START)) {
2560                perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2561                event->oncpu = -1;
2562                ret = -EAGAIN;
2563                goto out;
2564        }
2565
2566        if (!is_software_event(event))
2567                cpuctx->active_oncpu++;
2568        if (!ctx->nr_active++)
2569                perf_event_ctx_activate(ctx);
2570        if (event->attr.freq && event->attr.sample_freq)
2571                ctx->nr_freq++;
2572
2573        if (event->attr.exclusive)
2574                cpuctx->exclusive = 1;
2575
2576out:
2577        perf_pmu_enable(event->pmu);
2578
2579        return ret;
2580}
2581
2582static int
2583group_sched_in(struct perf_event *group_event,
2584               struct perf_cpu_context *cpuctx,
2585               struct perf_event_context *ctx)
2586{
2587        struct perf_event *event, *partial_group = NULL;
2588        struct pmu *pmu = ctx->pmu;
2589
2590        if (group_event->state == PERF_EVENT_STATE_OFF)
2591                return 0;
2592
2593        pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2594
2595        if (event_sched_in(group_event, cpuctx, ctx))
2596                goto error;
2597
2598        /*
2599         * Schedule in siblings as one group (if any):
2600         */
2601        for_each_sibling_event(event, group_event) {
2602                if (event_sched_in(event, cpuctx, ctx)) {
2603                        partial_group = event;
2604                        goto group_error;
2605                }
2606        }
2607
2608        if (!pmu->commit_txn(pmu))
2609                return 0;
2610
2611group_error:
2612        /*
2613         * Groups can be scheduled in as one unit only, so undo any
2614         * partial group before returning:
2615         * The events up to the failed event are scheduled out normally.
2616         */
2617        for_each_sibling_event(event, group_event) {
2618                if (event == partial_group)
2619                        break;
2620
2621                event_sched_out(event, cpuctx, ctx);
2622        }
2623        event_sched_out(group_event, cpuctx, ctx);
2624
2625error:
2626        pmu->cancel_txn(pmu);
2627        return -EAGAIN;
2628}
2629
2630/*
2631 * Work out whether we can put this event group on the CPU now.
2632 */
2633static int group_can_go_on(struct perf_event *event,
2634                           struct perf_cpu_context *cpuctx,
2635                           int can_add_hw)
2636{
2637        /*
2638         * Groups consisting entirely of software events can always go on.
2639         */
2640        if (event->group_caps & PERF_EV_CAP_SOFTWARE)
2641                return 1;
2642        /*
2643         * If an exclusive group is already on, no other hardware
2644         * events can go on.
2645         */
2646        if (cpuctx->exclusive)
2647                return 0;
2648        /*
2649         * If this group is exclusive and there are already
2650         * events on the CPU, it can't go on.
2651         */
2652        if (event->attr.exclusive && !list_empty(get_event_list(event)))
2653                return 0;
2654        /*
2655         * Otherwise, try to add it if all previous groups were able
2656         * to go on.
2657         */
2658        return can_add_hw;
2659}
2660
2661static void add_event_to_ctx(struct perf_event *event,
2662                               struct perf_event_context *ctx)
2663{
2664        list_add_event(event, ctx);
2665        perf_group_attach(event);
2666}
2667
2668static void ctx_sched_out(struct perf_event_context *ctx,
2669                          struct perf_cpu_context *cpuctx,
2670                          enum event_type_t event_type);
2671static void
2672ctx_sched_in(struct perf_event_context *ctx,
2673             struct perf_cpu_context *cpuctx,
2674             enum event_type_t event_type,
2675             struct task_struct *task);
2676
2677static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
2678                               struct perf_event_context *ctx,
2679                               enum event_type_t event_type)
2680{
2681        if (!cpuctx->task_ctx)
2682                return;
2683
2684        if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2685                return;
2686
2687        ctx_sched_out(ctx, cpuctx, event_type);
2688}
2689
2690static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2691                                struct perf_event_context *ctx,
2692                                struct task_struct *task)
2693{
2694        cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
2695        if (ctx)
2696                ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
2697        cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
2698        if (ctx)
2699                ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
2700}
2701
2702/*
2703 * We want to maintain the following priority of scheduling:
2704 *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
2705 *  - task pinned (EVENT_PINNED)
2706 *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2707 *  - task flexible (EVENT_FLEXIBLE).
2708 *
2709 * In order to avoid unscheduling and scheduling back in everything every
2710 * time an event is added, only do it for the groups of equal priority and
2711 * below.
2712 *
2713 * This can be called after a batch operation on task events, in which case
2714 * event_type is a bit mask of the types of events involved. For CPU events,
2715 * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
2716 */
2717static void ctx_resched(struct perf_cpu_context *cpuctx,
2718                        struct perf_event_context *task_ctx,
2719                        enum event_type_t event_type)
2720{
2721        enum event_type_t ctx_event_type;
2722        bool cpu_event = !!(event_type & EVENT_CPU);
2723
2724        /*
2725         * If pinned groups are involved, flexible groups also need to be
2726         * scheduled out.
2727         */
2728        if (event_type & EVENT_PINNED)
2729                event_type |= EVENT_FLEXIBLE;
2730
2731        ctx_event_type = event_type & EVENT_ALL;
2732
2733        perf_pmu_disable(cpuctx->ctx.pmu);
2734        if (task_ctx)
2735                task_ctx_sched_out(cpuctx, task_ctx, event_type);
2736
2737        /*
2738         * Decide which cpu ctx groups to schedule out based on the types
2739         * of events that caused rescheduling:
2740         *  - EVENT_CPU: schedule out corresponding groups;
2741         *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
2742         *  - otherwise, do nothing more.
2743         */
2744        if (cpu_event)
2745                cpu_ctx_sched_out(cpuctx, ctx_event_type);
2746        else if (ctx_event_type & EVENT_PINNED)
2747                cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2748
2749        perf_event_sched_in(cpuctx, task_ctx, current);
2750        perf_pmu_enable(cpuctx->ctx.pmu);
2751}
2752
2753void perf_pmu_resched(struct pmu *pmu)
2754{
2755        struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2756        struct perf_event_context *task_ctx = cpuctx->task_ctx;
2757
2758        perf_ctx_lock(cpuctx, task_ctx);
2759        ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU);
2760        perf_ctx_unlock(cpuctx, task_ctx);
2761}
2762
2763/*
2764 * Cross CPU call to install and enable a performance event
2765 *
2766 * Very similar to remote_function() + event_function() but cannot assume that
2767 * things like ctx->is_active and cpuctx->task_ctx are set.
2768 */
2769static int  __perf_install_in_context(void *info)
2770{
2771        struct perf_event *event = info;
2772        struct perf_event_context *ctx = event->ctx;
2773        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2774        struct perf_event_context *task_ctx = cpuctx->task_ctx;
2775        bool reprogram = true;
2776        int ret = 0;
2777
2778        raw_spin_lock(&cpuctx->ctx.lock);
2779        if (ctx->task) {
2780                raw_spin_lock(&ctx->lock);
2781                task_ctx = ctx;
2782
2783                reprogram = (ctx->task == current);
2784
2785                /*
2786                 * If the task is running, it must be running on this CPU,
2787                 * otherwise we cannot reprogram things.
2788                 *
2789                 * If its not running, we don't care, ctx->lock will
2790                 * serialize against it becoming runnable.
2791                 */
2792                if (task_curr(ctx->task) && !reprogram) {
2793                        ret = -ESRCH;
2794                        goto unlock;
2795                }
2796
2797                WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
2798        } else if (task_ctx) {
2799                raw_spin_lock(&task_ctx->lock);
2800        }
2801
2802#ifdef CONFIG_CGROUP_PERF
2803        if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
2804                /*
2805                 * If the current cgroup doesn't match the event's
2806                 * cgroup, we should not try to schedule it.
2807                 */
2808                struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
2809                reprogram = cgroup_is_descendant(cgrp->css.cgroup,
2810                                        event->cgrp->css.cgroup);
2811        }
2812#endif
2813
2814        if (reprogram) {
2815                ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2816                add_event_to_ctx(event, ctx);
2817                ctx_resched(cpuctx, task_ctx, get_event_type(event));
2818        } else {
2819                add_event_to_ctx(event, ctx);
2820        }
2821
2822unlock:
2823        perf_ctx_unlock(cpuctx, task_ctx);
2824
2825        return ret;
2826}
2827
2828static bool exclusive_event_installable(struct perf_event *event,
2829                                        struct perf_event_context *ctx);
2830
2831/*
2832 * Attach a performance event to a context.
2833 *
2834 * Very similar to event_function_call, see comment there.
2835 */
2836static void
2837perf_install_in_context(struct perf_event_context *ctx,
2838                        struct perf_event *event,
2839                        int cpu)
2840{
2841        struct task_struct *task = READ_ONCE(ctx->task);
2842
2843        lockdep_assert_held(&ctx->mutex);
2844
2845        WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
2846
2847        if (event->cpu != -1)
2848                event->cpu = cpu;
2849
2850        /*
2851         * Ensures that if we can observe event->ctx, both the event and ctx
2852         * will be 'complete'. See perf_iterate_sb_cpu().
2853         */
2854        smp_store_release(&event->ctx, ctx);
2855
2856        /*
2857         * perf_event_attr::disabled events will not run and can be initialized
2858         * without IPI. Except when this is the first event for the context, in
2859         * that case we need the magic of the IPI to set ctx->is_active.
2860         *
2861         * The IOC_ENABLE that is sure to follow the creation of a disabled
2862         * event will issue the IPI and reprogram the hardware.
2863         */
2864        if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && ctx->nr_events) {
2865                raw_spin_lock_irq(&ctx->lock);
2866                if (ctx->task == TASK_TOMBSTONE) {
2867                        raw_spin_unlock_irq(&ctx->lock);
2868                        return;
2869                }
2870                add_event_to_ctx(event, ctx);
2871                raw_spin_unlock_irq(&ctx->lock);
2872                return;
2873        }
2874
2875        if (!task) {
2876                cpu_function_call(cpu, __perf_install_in_context, event);
2877                return;
2878        }
2879
2880        /*
2881         * Should not happen, we validate the ctx is still alive before calling.
2882         */
2883        if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
2884                return;
2885
2886        /*
2887         * Installing events is tricky because we cannot rely on ctx->is_active
2888         * to be set in case this is the nr_events 0 -> 1 transition.
2889         *
2890         * Instead we use task_curr(), which tells us if the task is running.
2891         * However, since we use task_curr() outside of rq::lock, we can race
2892         * against the actual state. This means the result can be wrong.
2893         *
2894         * If we get a false positive, we retry, this is harmless.
2895         *
2896         * If we get a false negative, things are complicated. If we are after
2897         * perf_event_context_sched_in() ctx::lock will serialize us, and the
2898         * value must be correct. If we're before, it doesn't matter since
2899         * perf_event_context_sched_in() will program the counter.
2900         *
2901         * However, this hinges on the remote context switch having observed
2902         * our task->perf_event_ctxp[] store, such that it will in fact take
2903         * ctx::lock in perf_event_context_sched_in().
2904         *
2905         * We do this by task_function_call(), if the IPI fails to hit the task
2906         * we know any future context switch of task must see the
2907         * perf_event_ctpx[] store.
2908         */
2909
2910        /*
2911         * This smp_mb() orders the task->perf_event_ctxp[] store with the
2912         * task_cpu() load, such that if the IPI then does not find the task
2913         * running, a future context switch of that task must observe the
2914         * store.
2915         */
2916        smp_mb();
2917again:
2918        if (!task_function_call(task, __perf_install_in_context, event))
2919                return;
2920
2921        raw_spin_lock_irq(&ctx->lock);
2922        task = ctx->task;
2923        if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
2924                /*
2925                 * Cannot happen because we already checked above (which also
2926                 * cannot happen), and we hold ctx->mutex, which serializes us
2927                 * against perf_event_exit_task_context().
2928                 */
2929                raw_spin_unlock_irq(&ctx->lock);
2930                return;
2931        }
2932        /*
2933         * If the task is not running, ctx->lock will avoid it becoming so,
2934         * thus we can safely install the event.
2935         */
2936        if (task_curr(task)) {
2937                raw_spin_unlock_irq(&ctx->lock);
2938                goto again;
2939        }
2940        add_event_to_ctx(event, ctx);
2941        raw_spin_unlock_irq(&ctx->lock);
2942}
2943
2944/*
2945 * Cross CPU call to enable a performance event
2946 */
2947static void __perf_event_enable(struct perf_event *event,
2948                                struct perf_cpu_context *cpuctx,
2949                                struct perf_event_context *ctx,
2950                                void *info)
2951{
2952        struct perf_event *leader = event->group_leader;
2953        struct perf_event_context *task_ctx;
2954
2955        if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2956            event->state <= PERF_EVENT_STATE_ERROR)
2957                return;
2958
2959        if (ctx->is_active)
2960                ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2961
2962        perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2963        perf_cgroup_event_enable(event, ctx);
2964
2965        if (!ctx->is_active)
2966                return;
2967
2968        if (!event_filter_match(event)) {
2969                ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
2970                return;
2971        }
2972
2973        /*
2974         * If the event is in a group and isn't the group leader,
2975         * then don't put it on unless the group is on.
2976         */
2977        if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
2978                ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
2979                return;
2980        }
2981
2982        task_ctx = cpuctx->task_ctx;
2983        if (ctx->task)
2984                WARN_ON_ONCE(task_ctx != ctx);
2985
2986        ctx_resched(cpuctx, task_ctx, get_event_type(event));
2987}
2988
2989/*
2990 * Enable an event.
2991 *
2992 * If event->ctx is a cloned context, callers must make sure that
2993 * every task struct that event->ctx->task could possibly point to
2994 * remains valid.  This condition is satisfied when called through
2995 * perf_event_for_each_child or perf_event_for_each as described
2996 * for perf_event_disable.
2997 */
2998static void _perf_event_enable(struct perf_event *event)
2999{
3000        struct perf_event_context *ctx = event->ctx;
3001
3002        raw_spin_lock_irq(&ctx->lock);
3003        if (event->state >= PERF_EVENT_STATE_INACTIVE ||
3004            event->state <  PERF_EVENT_STATE_ERROR) {
3005out:
3006                raw_spin_unlock_irq(&ctx->lock);
3007                return;
3008        }
3009
3010        /*
3011         * If the event is in error state, clear that first.
3012         *
3013         * That way, if we see the event in error state below, we know that it
3014         * has gone back into error state, as distinct from the task having
3015         * been scheduled away before the cross-call arrived.
3016         */
3017        if (event->state == PERF_EVENT_STATE_ERROR) {
3018                /*
3019                 * Detached SIBLING events cannot leave ERROR state.
3020                 */
3021                if (event->event_caps & PERF_EV_CAP_SIBLING &&
3022                    event->group_leader == event)
3023                        goto out;
3024
3025                event->state = PERF_EVENT_STATE_OFF;
3026        }
3027        raw_spin_unlock_irq(&ctx->lock);
3028
3029        event_function_call(event, __perf_event_enable, NULL);
3030}
3031
3032/*
3033 * See perf_event_disable();
3034 */
3035void perf_event_enable(struct perf_event *event)
3036{
3037        struct perf_event_context *ctx;
3038
3039        ctx = perf_event_ctx_lock(event);
3040        _perf_event_enable(event);
3041        perf_event_ctx_unlock(event, ctx);
3042}
3043EXPORT_SYMBOL_GPL(perf_event_enable);
3044
3045struct stop_event_data {
3046        struct perf_event       *event;
3047        unsigned int            restart;
3048};
3049
3050static int __perf_event_stop(void *info)
3051{
3052        struct stop_event_data *sd = info;
3053        struct perf_event *event = sd->event;
3054
3055        /* if it's already INACTIVE, do nothing */
3056        if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3057                return 0;
3058
3059        /* matches smp_wmb() in event_sched_in() */
3060        smp_rmb();
3061
3062        /*
3063         * There is a window with interrupts enabled before we get here,
3064         * so we need to check again lest we try to stop another CPU's event.
3065         */
3066        if (READ_ONCE(event->oncpu) != smp_processor_id())
3067                return -EAGAIN;
3068
3069        event->pmu->stop(event, PERF_EF_UPDATE);
3070
3071        /*
3072         * May race with the actual stop (through perf_pmu_output_stop()),
3073         * but it is only used for events with AUX ring buffer, and such
3074         * events will refuse to restart because of rb::aux_mmap_count==0,
3075         * see comments in perf_aux_output_begin().
3076         *
3077         * Since this is happening on an event-local CPU, no trace is lost
3078         * while restarting.
3079         */
3080        if (sd->restart)
3081                event->pmu->start(event, 0);
3082
3083        return 0;
3084}
3085
3086static int perf_event_stop(struct perf_event *event, int restart)
3087{
3088        struct stop_event_data sd = {
3089                .event          = event,
3090                .restart        = restart,
3091        };
3092        int ret = 0;
3093
3094        do {
3095                if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3096                        return 0;
3097
3098                /* matches smp_wmb() in event_sched_in() */
3099                smp_rmb();
3100
3101                /*
3102                 * We only want to restart ACTIVE events, so if the event goes
3103                 * inactive here (event->oncpu==-1), there's nothing more to do;
3104                 * fall through with ret==-ENXIO.
3105                 */
3106                ret = cpu_function_call(READ_ONCE(event->oncpu),
3107                                        __perf_event_stop, &sd);
3108        } while (ret == -EAGAIN);
3109
3110        return ret;
3111}
3112
3113/*
3114 * In order to contain the amount of racy and tricky in the address filter
3115 * configuration management, it is a two part process:
3116 *
3117 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
3118 *      we update the addresses of corresponding vmas in
3119 *      event::addr_filter_ranges array and bump the event::addr_filters_gen;
3120 * (p2) when an event is scheduled in (pmu::add), it calls
3121 *      perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
3122 *      if the generation has changed since the previous call.
3123 *
3124 * If (p1) happens while the event is active, we restart it to force (p2).
3125 *
3126 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on
3127 *     pre-existing mappings, called once when new filters arrive via SET_FILTER
3128 *     ioctl;
3129 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
3130 *     registered mapping, called for every new mmap(), with mm::mmap_lock down
3131 *     for reading;
3132 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
3133 *     of exec.
3134 */
3135void perf_event_addr_filters_sync(struct perf_event *event)
3136{
3137        struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
3138
3139        if (!has_addr_filter(event))
3140                return;
3141
3142        raw_spin_lock(&ifh->lock);
3143        if (event->addr_filters_gen != event->hw.addr_filters_gen) {
3144                event->pmu->addr_filters_sync(event);
3145                event->hw.addr_filters_gen = event->addr_filters_gen;
3146        }
3147        raw_spin_unlock(&ifh->lock);
3148}
3149EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
3150
3151static int _perf_event_refresh(struct perf_event *event, int refresh)
3152{
3153        /*
3154         * not supported on inherited events
3155         */
3156        if (event->attr.inherit || !is_sampling_event(event))
3157                return -EINVAL;
3158
3159        atomic_add(refresh, &event->event_limit);
3160        _perf_event_enable(event);
3161
3162        return 0;
3163}
3164
3165/*
3166 * See perf_event_disable()
3167 */
3168int perf_event_refresh(struct perf_event *event, int refresh)
3169{
3170        struct perf_event_context *ctx;
3171        int ret;
3172
3173        ctx = perf_event_ctx_lock(event);
3174        ret = _perf_event_refresh(event, refresh);
3175        perf_event_ctx_unlock(event, ctx);
3176
3177        return ret;
3178}
3179EXPORT_SYMBOL_GPL(perf_event_refresh);
3180
3181static int perf_event_modify_breakpoint(struct perf_event *bp,
3182                                         struct perf_event_attr *attr)
3183{
3184        int err;
3185
3186        _perf_event_disable(bp);
3187
3188        err = modify_user_hw_breakpoint_check(bp, attr, true);
3189
3190        if (!bp->attr.disabled)
3191                _perf_event_enable(bp);
3192
3193        return err;
3194}
3195
3196static int perf_event_modify_attr(struct perf_event *event,
3197                                  struct perf_event_attr *attr)
3198{
3199        int (*func)(struct perf_event *, struct perf_event_attr *);
3200        struct perf_event *child;
3201        int err;
3202
3203        if (event->attr.type != attr->type)
3204                return -EINVAL;
3205
3206        switch (event->attr.type) {
3207        case PERF_TYPE_BREAKPOINT:
3208                func = perf_event_modify_breakpoint;
3209                break;
3210        default:
3211                /* Place holder for future additions. */
3212                return -EOPNOTSUPP;
3213        }
3214
3215        WARN_ON_ONCE(event->ctx->parent_ctx);
3216
3217        mutex_lock(&event->child_mutex);
3218        err = func(event, attr);
3219        if (err)
3220                goto out;
3221        list_for_each_entry(child, &event->child_list, child_list) {
3222                err = func(child, attr);
3223                if (err)
3224                        goto out;
3225        }
3226out:
3227        mutex_unlock(&event->child_mutex);
3228        return err;
3229}
3230
3231static void ctx_sched_out(struct perf_event_context *ctx,
3232                          struct perf_cpu_context *cpuctx,
3233                          enum event_type_t event_type)
3234{
3235        struct perf_event *event, *tmp;
3236        int is_active = ctx->is_active;
3237
3238        lockdep_assert_held(&ctx->lock);
3239
3240        if (likely(!ctx->nr_events)) {
3241                /*
3242                 * See __perf_remove_from_context().
3243                 */
3244                WARN_ON_ONCE(ctx->is_active);
3245                if (ctx->task)
3246                        WARN_ON_ONCE(cpuctx->task_ctx);
3247                return;
3248        }
3249
3250        ctx->is_active &= ~event_type;
3251        if (!(ctx->is_active & EVENT_ALL))
3252                ctx->is_active = 0;
3253
3254        if (ctx->task) {
3255                WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3256                if (!ctx->is_active)
3257                        cpuctx->task_ctx = NULL;
3258        }
3259
3260        /*
3261         * Always update time if it was set; not only when it changes.
3262         * Otherwise we can 'forget' to update time for any but the last
3263         * context we sched out. For example:
3264         *
3265         *   ctx_sched_out(.event_type = EVENT_FLEXIBLE)
3266         *   ctx_sched_out(.event_type = EVENT_PINNED)
3267         *
3268         * would only update time for the pinned events.
3269         */
3270        if (is_active & EVENT_TIME) {
3271                /* update (and stop) ctx time */
3272                update_context_time(ctx);
3273                update_cgrp_time_from_cpuctx(cpuctx);
3274        }
3275
3276        is_active ^= ctx->is_active; /* changed bits */
3277
3278        if (!ctx->nr_active || !(is_active & EVENT_ALL))
3279                return;
3280
3281        perf_pmu_disable(ctx->pmu);
3282        if (is_active & EVENT_PINNED) {
3283                list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list)
3284                        group_sched_out(event, cpuctx, ctx);
3285        }
3286
3287        if (is_active & EVENT_FLEXIBLE) {
3288                list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list)
3289                        group_sched_out(event, cpuctx, ctx);
3290
3291                /*
3292                 * Since we cleared EVENT_FLEXIBLE, also clear
3293                 * rotate_necessary, is will be reset by
3294                 * ctx_flexible_sched_in() when needed.
3295                 */
3296                ctx->rotate_necessary = 0;
3297        }
3298        perf_pmu_enable(ctx->pmu);
3299}
3300
3301/*
3302 * Test whether two contexts are equivalent, i.e. whether they have both been
3303 * cloned from the same version of the same context.
3304 *
3305 * Equivalence is measured using a generation number in the context that is
3306 * incremented on each modification to it; see unclone_ctx(), list_add_event()
3307 * and list_del_event().
3308 */
3309static int context_equiv(struct perf_event_context *ctx1,
3310                         struct perf_event_context *ctx2)
3311{
3312        lockdep_assert_held(&ctx1->lock);
3313        lockdep_assert_held(&ctx2->lock);
3314
3315        /* Pinning disables the swap optimization */
3316        if (ctx1->pin_count || ctx2->pin_count)
3317                return 0;
3318
3319        /* If ctx1 is the parent of ctx2 */
3320        if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
3321                return 1;
3322
3323        /* If ctx2 is the parent of ctx1 */
3324        if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
3325                return 1;
3326
3327        /*
3328         * If ctx1 and ctx2 have the same parent; we flatten the parent
3329         * hierarchy, see perf_event_init_context().
3330         */
3331        if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
3332                        ctx1->parent_gen == ctx2->parent_gen)
3333                return 1;
3334
3335        /* Unmatched */
3336        return 0;
3337}
3338
3339static void __perf_event_sync_stat(struct perf_event *event,
3340                                     struct perf_event *next_event)
3341{
3342        u64 value;
3343
3344        if (!event->attr.inherit_stat)
3345                return;
3346
3347        /*
3348         * Update the event value, we cannot use perf_event_read()
3349         * because we're in the middle of a context switch and have IRQs
3350         * disabled, which upsets smp_call_function_single(), however
3351         * we know the event must be on the current CPU, therefore we
3352         * don't need to use it.
3353         */
3354        if (event->state == PERF_EVENT_STATE_ACTIVE)
3355                event->pmu->read(event);
3356
3357        perf_event_update_time(event);
3358
3359        /*
3360         * In order to keep per-task stats reliable we need to flip the event
3361         * values when we flip the contexts.
3362         */
3363        value = local64_read(&next_event->count);
3364        value = local64_xchg(&event->count, value);
3365        local64_set(&next_event->count, value);
3366
3367        swap(event->total_time_enabled, next_event->total_time_enabled);
3368        swap(event->total_time_running, next_event->total_time_running);
3369
3370        /*
3371         * Since we swizzled the values, update the user visible data too.
3372         */
3373        perf_event_update_userpage(event);
3374        perf_event_update_userpage(next_event);
3375}
3376
3377static void perf_event_sync_stat(struct perf_event_context *ctx,
3378                                   struct perf_event_context *next_ctx)
3379{
3380        struct perf_event *event, *next_event;
3381
3382        if (!ctx->nr_stat)
3383                return;
3384
3385        update_context_time(ctx);
3386
3387        event = list_first_entry(&ctx->event_list,
3388                                   struct perf_event, event_entry);
3389
3390        next_event = list_first_entry(&next_ctx->event_list,
3391                                        struct perf_event, event_entry);
3392
3393        while (&event->event_entry != &ctx->event_list &&
3394               &next_event->event_entry != &next_ctx->event_list) {
3395
3396                __perf_event_sync_stat(event, next_event);
3397
3398                event = list_next_entry(event, event_entry);
3399                next_event = list_next_entry(next_event, event_entry);
3400        }
3401}
3402
3403static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
3404                                         struct task_struct *next)
3405{
3406        struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
3407        struct perf_event_context *next_ctx;
3408        struct perf_event_context *parent, *next_parent;
3409        struct perf_cpu_context *cpuctx;
3410        int do_switch = 1;
3411        struct pmu *pmu;
3412
3413        if (likely(!ctx))
3414                return;
3415
3416        pmu = ctx->pmu;
3417        cpuctx = __get_cpu_context(ctx);
3418        if (!cpuctx->task_ctx)
3419                return;
3420
3421        rcu_read_lock();
3422        next_ctx = next->perf_event_ctxp[ctxn];
3423        if (!next_ctx)
3424                goto unlock;
3425
3426        parent = rcu_dereference(ctx->parent_ctx);
3427        next_parent = rcu_dereference(next_ctx->parent_ctx);
3428
3429        /* If neither context have a parent context; they cannot be clones. */
3430        if (!parent && !next_parent)
3431                goto unlock;
3432
3433        if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
3434                /*
3435                 * Looks like the two contexts are clones, so we might be
3436                 * able to optimize the context switch.  We lock both
3437                 * contexts and check that they are clones under the
3438                 * lock (including re-checking that neither has been
3439                 * uncloned in the meantime).  It doesn't matter which
3440                 * order we take the locks because no other cpu could
3441                 * be trying to lock both of these tasks.
3442                 */
3443                raw_spin_lock(&ctx->lock);
3444                raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
3445                if (context_equiv(ctx, next_ctx)) {
3446
3447                        WRITE_ONCE(ctx->task, next);
3448                        WRITE_ONCE(next_ctx->task, task);
3449
3450                        perf_pmu_disable(pmu);
3451
3452                        if (cpuctx->sched_cb_usage && pmu->sched_task)
3453                                pmu->sched_task(ctx, false);
3454
3455                        /*
3456                         * PMU specific parts of task perf context can require
3457                         * additional synchronization. As an example of such
3458                         * synchronization see implementation details of Intel
3459                         * LBR call stack data profiling;
3460                         */
3461                        if (pmu->swap_task_ctx)
3462                                pmu->swap_task_ctx(ctx, next_ctx);
3463                        else
3464                                swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
3465
3466                        perf_pmu_enable(pmu);
3467
3468                        /*
3469                         * RCU_INIT_POINTER here is safe because we've not
3470                         * modified the ctx and the above modification of
3471                         * ctx->task and ctx->task_ctx_data are immaterial
3472                         * since those values are always verified under
3473                         * ctx->lock which we're now holding.
3474                         */
3475                        RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx);
3476                        RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx);
3477
3478                        do_switch = 0;
3479
3480                        perf_event_sync_stat(ctx, next_ctx);
3481                }
3482                raw_spin_unlock(&next_ctx->lock);
3483                raw_spin_unlock(&ctx->lock);
3484        }
3485unlock:
3486        rcu_read_unlock();
3487
3488        if (do_switch) {
3489                raw_spin_lock(&ctx->lock);
3490                perf_pmu_disable(pmu);
3491
3492                if (cpuctx->sched_cb_usage && pmu->sched_task)
3493                        pmu->sched_task(ctx, false);
3494                task_ctx_sched_out(cpuctx, ctx, EVENT_ALL);
3495
3496                perf_pmu_enable(pmu);
3497                raw_spin_unlock(&ctx->lock);
3498        }
3499}
3500
3501static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3502
3503void perf_sched_cb_dec(struct pmu *pmu)
3504{
3505        struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
3506
3507        this_cpu_dec(perf_sched_cb_usages);
3508
3509        if (!--cpuctx->sched_cb_usage)
3510                list_del(&cpuctx->sched_cb_entry);
3511}
3512
3513
3514void perf_sched_cb_inc(struct pmu *pmu)
3515{
3516        struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
3517
3518        if (!cpuctx->sched_cb_usage++)
3519                list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3520
3521        this_cpu_inc(perf_sched_cb_usages);
3522}
3523
3524/*
3525 * This function provides the context switch callback to the lower code
3526 * layer. It is invoked ONLY when the context switch callback is enabled.
3527 *
3528 * This callback is relevant even to per-cpu events; for example multi event
3529 * PEBS requires this to provide PID/TID information. This requires we flush
3530 * all queued PEBS records before we context switch to a new task.
3531 */
3532static void __perf_pmu_sched_task(struct perf_cpu_context *cpuctx, bool sched_in)
3533{
3534        struct pmu *pmu;
3535
3536        pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */
3537
3538        if (WARN_ON_ONCE(!pmu->sched_task))
3539                return;
3540
3541        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3542        perf_pmu_disable(pmu);
3543
3544        pmu->sched_task(cpuctx->task_ctx, sched_in);
3545
3546        perf_pmu_enable(pmu);
3547        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3548}
3549
3550static void perf_pmu_sched_task(struct task_struct *prev,
3551                                struct task_struct *next,
3552                                bool sched_in)
3553{
3554        struct perf_cpu_context *cpuctx;
3555
3556        if (prev == next)
3557                return;
3558
3559        list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
3560                /* will be handled in perf_event_context_sched_in/out */
3561                if (cpuctx->task_ctx)
3562                        continue;
3563
3564                __perf_pmu_sched_task(cpuctx, sched_in);
3565        }
3566}
3567
3568static void perf_event_switch(struct task_struct *task,
3569                              struct task_struct *next_prev, bool sched_in);
3570
3571#define for_each_task_context_nr(ctxn)                                  \
3572        for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
3573
3574/*
3575 * Called from scheduler to remove the events of the current task,
3576 * with interrupts disabled.
3577 *
3578 * We stop each event and update the event value in event->count.
3579 *
3580 * This does not protect us against NMI, but disable()
3581 * sets the disabled bit in the control field of event _before_
3582 * accessing the event control register. If a NMI hits, then it will
3583 * not restart the event.
3584 */
3585void __perf_event_task_sched_out(struct task_struct *task,
3586                                 struct task_struct *next)
3587{
3588        int ctxn;
3589
3590        if (__this_cpu_read(perf_sched_cb_usages))
3591                perf_pmu_sched_task(task, next, false);
3592
3593        if (atomic_read(&nr_switch_events))
3594                perf_event_switch(task, next, false);
3595
3596        for_each_task_context_nr(ctxn)
3597                perf_event_context_sched_out(task, ctxn, next);
3598
3599        /*
3600         * if cgroup events exist on this CPU, then we need
3601         * to check if we have to switch out PMU state.
3602         * cgroup event are system-wide mode only
3603         */
3604        if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
3605                perf_cgroup_sched_out(task, next);
3606}
3607
3608/*
3609 * Called with IRQs disabled
3610 */
3611static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
3612                              enum event_type_t event_type)
3613{
3614        ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
3615}
3616
3617static bool perf_less_group_idx(const void *l, const void *r)
3618{
3619        const struct perf_event *le = *(const struct perf_event **)l;
3620        const struct perf_event *re = *(const struct perf_event **)r;
3621
3622        return le->group_index < re->group_index;
3623}
3624
3625static void swap_ptr(void *l, void *r)
3626{
3627        void **lp = l, **rp = r;
3628
3629        swap(*lp, *rp);
3630}
3631
3632static const struct min_heap_callbacks perf_min_heap = {
3633        .elem_size = sizeof(struct perf_event *),
3634        .less = perf_less_group_idx,
3635        .swp = swap_ptr,
3636};
3637
3638static void __heap_add(struct min_heap *heap, struct perf_event *event)
3639{
3640        struct perf_event **itrs = heap->data;
3641
3642        if (event) {
3643                itrs[heap->nr] = event;
3644                heap->nr++;
3645        }
3646}
3647
3648static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx,
3649                                struct perf_event_groups *groups, int cpu,
3650                                int (*func)(struct perf_event *, void *),
3651                                void *data)
3652{
3653#ifdef CONFIG_CGROUP_PERF
3654        struct cgroup_subsys_state *css = NULL;
3655#endif
3656        /* Space for per CPU and/or any CPU event iterators. */
3657        struct perf_event *itrs[2];
3658        struct min_heap event_heap;
3659        struct perf_event **evt;
3660        int ret;
3661
3662        if (cpuctx) {
3663                event_heap = (struct min_heap){
3664                        .data = cpuctx->heap,
3665                        .nr = 0,
3666                        .size = cpuctx->heap_size,
3667                };
3668
3669                lockdep_assert_held(&cpuctx->ctx.lock);
3670
3671#ifdef CONFIG_CGROUP_PERF
3672                if (cpuctx->cgrp)
3673                        css = &cpuctx->cgrp->css;
3674#endif
3675        } else {
3676                event_heap = (struct min_heap){
3677                        .data = itrs,
3678                        .nr = 0,
3679                        .size = ARRAY_SIZE(itrs),
3680                };
3681                /* Events not within a CPU context may be on any CPU. */
3682                __heap_add(&event_heap, perf_event_groups_first(groups, -1, NULL));
3683        }
3684        evt = event_heap.data;
3685
3686        __heap_add(&event_heap, perf_event_groups_first(groups, cpu, NULL));
3687
3688#ifdef CONFIG_CGROUP_PERF
3689        for (; css; css = css->parent)
3690                __heap_add(&event_heap, perf_event_groups_first(groups, cpu, css->cgroup));
3691#endif
3692
3693        min_heapify_all(&event_heap, &perf_min_heap);
3694
3695        while (event_heap.nr) {
3696                ret = func(*evt, data);
3697                if (ret)
3698                        return ret;
3699
3700                *evt = perf_event_groups_next(*evt);
3701                if (*evt)
3702                        min_heapify(&event_heap, 0, &perf_min_heap);
3703                else
3704                        min_heap_pop(&event_heap, &perf_min_heap);
3705        }
3706
3707        return 0;
3708}
3709
3710static inline bool event_update_userpage(struct perf_event *event)
3711{
3712        if (likely(!atomic_read(&event->mmap_count)))
3713                return false;
3714
3715        perf_event_update_time(event);
3716        perf_set_shadow_time(event, event->ctx);
3717        perf_event_update_userpage(event);
3718
3719        return true;
3720}
3721
3722static inline void group_update_userpage(struct perf_event *group_event)
3723{
3724        struct perf_event *event;
3725
3726        if (!event_update_userpage(group_event))
3727                return;
3728
3729        for_each_sibling_event(event, group_event)
3730                event_update_userpage(event);
3731}
3732
3733static int merge_sched_in(struct perf_event *event, void *data)
3734{
3735        struct perf_event_context *ctx = event->ctx;
3736        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3737        int *can_add_hw = data;
3738
3739        if (event->state <= PERF_EVENT_STATE_OFF)
3740                return 0;
3741
3742        if (!event_filter_match(event))
3743                return 0;
3744
3745        if (group_can_go_on(event, cpuctx, *can_add_hw)) {
3746                if (!group_sched_in(event, cpuctx, ctx))
3747                        list_add_tail(&event->active_list, get_event_list(event));
3748        }
3749
3750        if (event->state == PERF_EVENT_STATE_INACTIVE) {
3751                *can_add_hw = 0;
3752                if (event->attr.pinned) {
3753                        perf_cgroup_event_disable(event, ctx);
3754                        perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
3755                } else {
3756                        ctx->rotate_necessary = 1;
3757                        perf_mux_hrtimer_restart(cpuctx);
3758                        group_update_userpage(event);
3759                }
3760        }
3761
3762        return 0;
3763}
3764
3765static void
3766ctx_pinned_sched_in(struct perf_event_context *ctx,
3767                    struct perf_cpu_context *cpuctx)
3768{
3769        int can_add_hw = 1;
3770
3771        if (ctx != &cpuctx->ctx)
3772                cpuctx = NULL;
3773
3774        visit_groups_merge(cpuctx, &ctx->pinned_groups,
3775                           smp_processor_id(),
3776                           merge_sched_in, &can_add_hw);
3777}
3778
3779static void
3780ctx_flexible_sched_in(struct perf_event_context *ctx,
3781                      struct perf_cpu_context *cpuctx)
3782{
3783        int can_add_hw = 1;
3784
3785        if (ctx != &cpuctx->ctx)
3786                cpuctx = NULL;
3787
3788        visit_groups_merge(cpuctx, &ctx->flexible_groups,
3789                           smp_processor_id(),
3790                           merge_sched_in, &can_add_hw);
3791}
3792
3793static void
3794ctx_sched_in(struct perf_event_context *ctx,
3795             struct perf_cpu_context *cpuctx,
3796             enum event_type_t event_type,
3797             struct task_struct *task)
3798{
3799        int is_active = ctx->is_active;
3800        u64 now;
3801
3802        lockdep_assert_held(&ctx->lock);
3803
3804        if (likely(!ctx->nr_events))
3805                return;
3806
3807        ctx->is_active |= (event_type | EVENT_TIME);
3808        if (ctx->task) {
3809                if (!is_active)
3810                        cpuctx->task_ctx = ctx;
3811                else
3812                        WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3813        }
3814
3815        is_active ^= ctx->is_active; /* changed bits */
3816
3817        if (is_active & EVENT_TIME) {
3818                /* start ctx time */
3819                now = perf_clock();
3820                ctx->timestamp = now;
3821                perf_cgroup_set_timestamp(task, ctx);
3822        }
3823
3824        /*
3825         * First go through the list and put on any pinned groups
3826         * in order to give them the best chance of going on.
3827         */
3828        if (is_active & EVENT_PINNED)
3829                ctx_pinned_sched_in(ctx, cpuctx);
3830
3831        /* Then walk through the lower prio flexible groups */
3832        if (is_active & EVENT_FLEXIBLE)
3833                ctx_flexible_sched_in(ctx, cpuctx);
3834}
3835
3836static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
3837                             enum event_type_t event_type,
3838                             struct task_struct *task)
3839{
3840        struct perf_event_context *ctx = &cpuctx->ctx;
3841
3842        ctx_sched_in(ctx, cpuctx, event_type, task);
3843}
3844
3845static void perf_event_context_sched_in(struct perf_event_context *ctx,
3846                                        struct task_struct *task)
3847{
3848        struct perf_cpu_context *cpuctx;
3849        struct pmu *pmu;
3850
3851        cpuctx = __get_cpu_context(ctx);
3852
3853        /*
3854         * HACK: for HETEROGENEOUS the task context might have switched to a
3855         * different PMU, force (re)set the context,
3856         */
3857        pmu = ctx->pmu = cpuctx->ctx.pmu;
3858
3859        if (cpuctx->task_ctx == ctx) {
3860                if (cpuctx->sched_cb_usage)
3861                        __perf_pmu_sched_task(cpuctx, true);
3862                return;
3863        }
3864
3865        perf_ctx_lock(cpuctx, ctx);
3866        /*
3867         * We must check ctx->nr_events while holding ctx->lock, such
3868         * that we serialize against perf_install_in_context().
3869         */
3870        if (!ctx->nr_events)
3871                goto unlock;
3872
3873        perf_pmu_disable(pmu);
3874        /*
3875         * We want to keep the following priority order:
3876         * cpu pinned (that don't need to move), task pinned,
3877         * cpu flexible, task flexible.
3878         *
3879         * However, if task's ctx is not carrying any pinned
3880         * events, no need to flip the cpuctx's events around.
3881         */
3882        if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
3883                cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
3884        perf_event_sched_in(cpuctx, ctx, task);
3885
3886        if (cpuctx->sched_cb_usage && pmu->sched_task)
3887                pmu->sched_task(cpuctx->task_ctx, true);
3888
3889        perf_pmu_enable(pmu);
3890
3891unlock:
3892        perf_ctx_unlock(cpuctx, ctx);
3893}
3894
3895/*
3896 * Called from scheduler to add the events of the current task
3897 * with interrupts disabled.
3898 *
3899 * We restore the event value and then enable it.
3900 *
3901 * This does not protect us against NMI, but enable()
3902 * sets the enabled bit in the control field of event _before_
3903 * accessing the event control register. If a NMI hits, then it will
3904 * keep the event running.
3905 */
3906void __perf_event_task_sched_in(struct task_struct *prev,
3907                                struct task_struct *task)
3908{
3909        struct perf_event_context *ctx;
3910        int ctxn;
3911
3912        /*
3913         * If cgroup events exist on this CPU, then we need to check if we have
3914         * to switch in PMU state; cgroup event are system-wide mode only.
3915         *
3916         * Since cgroup events are CPU events, we must schedule these in before
3917         * we schedule in the task events.
3918         */
3919        if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
3920                perf_cgroup_sched_in(prev, task);
3921
3922        for_each_task_context_nr(ctxn) {
3923                ctx = task->perf_event_ctxp[ctxn];
3924                if (likely(!ctx))
3925                        continue;
3926
3927                perf_event_context_sched_in(ctx, task);
3928        }
3929
3930        if (atomic_read(&nr_switch_events))
3931                perf_event_switch(task, prev, true);
3932
3933        if (__this_cpu_read(perf_sched_cb_usages))
3934                perf_pmu_sched_task(prev, task, true);
3935}
3936
3937static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
3938{
3939        u64 frequency = event->attr.sample_freq;
3940        u64 sec = NSEC_PER_SEC;
3941        u64 divisor, dividend;
3942
3943        int count_fls, nsec_fls, frequency_fls, sec_fls;
3944
3945        count_fls = fls64(count);
3946        nsec_fls = fls64(nsec);
3947        frequency_fls = fls64(frequency);
3948        sec_fls = 30;
3949
3950        /*
3951         * We got @count in @nsec, with a target of sample_freq HZ
3952         * the target period becomes:
3953         *
3954         *             @count * 10^9
3955         * period = -------------------
3956         *          @nsec * sample_freq
3957         *
3958         */
3959
3960        /*
3961         * Reduce accuracy by one bit such that @a and @b converge
3962         * to a similar magnitude.
3963         */
3964#define REDUCE_FLS(a, b)                \
3965do {                                    \
3966        if (a##_fls > b##_fls) {        \
3967                a >>= 1;                \
3968                a##_fls--;              \
3969        } else {                        \
3970                b >>= 1;                \
3971                b##_fls--;              \
3972        }                               \
3973} while (0)
3974
3975        /*
3976         * Reduce accuracy until either term fits in a u64, then proceed with
3977         * the other, so that finally we can do a u64/u64 division.
3978         */
3979        while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
3980                REDUCE_FLS(nsec, frequency);
3981                REDUCE_FLS(sec, count);
3982        }
3983
3984        if (count_fls + sec_fls > 64) {
3985                divisor = nsec * frequency;
3986
3987                while (count_fls + sec_fls > 64) {
3988                        REDUCE_FLS(count, sec);
3989                        divisor >>= 1;
3990                }
3991
3992                dividend = count * sec;
3993        } else {
3994                dividend = count * sec;
3995
3996                while (nsec_fls + frequency_fls > 64) {
3997                        REDUCE_FLS(nsec, frequency);
3998                        dividend >>= 1;
3999                }
4000
4001                divisor = nsec * frequency;
4002        }
4003
4004        if (!divisor)
4005                return dividend;
4006
4007        return div64_u64(dividend, divisor);
4008}
4009
4010static DEFINE_PER_CPU(int, perf_throttled_count);
4011static DEFINE_PER_CPU(u64, perf_throttled_seq);
4012
4013static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
4014{
4015        struct hw_perf_event *hwc = &event->hw;
4016        s64 period, sample_period;
4017        s64 delta;
4018
4019        period = perf_calculate_period(event, nsec, count);
4020
4021        delta = (s64)(period - hwc->sample_period);
4022        delta = (delta + 7) / 8; /* low pass filter */
4023
4024        sample_period = hwc->sample_period + delta;
4025
4026        if (!sample_period)
4027                sample_period = 1;
4028
4029        hwc->sample_period = sample_period;
4030
4031        if (local64_read(&hwc->period_left) > 8*sample_period) {
4032                if (disable)
4033                        event->pmu->stop(event, PERF_EF_UPDATE);
4034
4035                local64_set(&hwc->period_left, 0);
4036
4037                if (disable)
4038                        event->pmu->start(event, PERF_EF_RELOAD);
4039        }
4040}
4041
4042/*
4043 * combine freq adjustment with unthrottling to avoid two passes over the
4044 * events. At the same time, make sure, having freq events does not change
4045 * the rate of unthrottling as that would introduce bias.
4046 */
4047static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
4048                                           int needs_unthr)
4049{
4050        struct perf_event *event;
4051        struct hw_perf_event *hwc;
4052        u64 now, period = TICK_NSEC;
4053        s64 delta;
4054
4055        /*
4056         * only need to iterate over all events iff:
4057         * - context have events in frequency mode (needs freq adjust)
4058         * - there are events to unthrottle on this cpu
4059         */
4060        if (!(ctx->nr_freq || needs_unthr))
4061                return;
4062
4063        raw_spin_lock(&ctx->lock);
4064        perf_pmu_disable(ctx->pmu);
4065
4066        list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4067                if (event->state != PERF_EVENT_STATE_ACTIVE)
4068                        continue;
4069
4070                if (!event_filter_match(event))
4071                        continue;
4072
4073                perf_pmu_disable(event->pmu);
4074
4075                hwc = &event->hw;
4076
4077                if (hwc->interrupts == MAX_INTERRUPTS) {
4078                        hwc->interrupts = 0;
4079                        perf_log_throttle(event, 1);
4080                        event->pmu->start(event, 0);
4081                }
4082
4083                if (!event->attr.freq || !event->attr.sample_freq)
4084                        goto next;
4085
4086                /*
4087                 * stop the event and update event->count
4088                 */
4089                event->pmu->stop(event, PERF_EF_UPDATE);
4090
4091                now = local64_read(&event->count);
4092                delta = now - hwc->freq_count_stamp;
4093                hwc->freq_count_stamp = now;
4094
4095                /*
4096                 * restart the event
4097                 * reload only if value has changed
4098                 * we have stopped the event so tell that
4099                 * to perf_adjust_period() to avoid stopping it
4100                 * twice.
4101                 */
4102                if (delta > 0)
4103                        perf_adjust_period(event, period, delta, false);
4104
4105                event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
4106        next:
4107                perf_pmu_enable(event->pmu);
4108        }
4109
4110        perf_pmu_enable(ctx->pmu);
4111        raw_spin_unlock(&ctx->lock);
4112}
4113
4114/*
4115 * Move @event to the tail of the @ctx's elegible events.
4116 */
4117static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)
4118{
4119        /*
4120         * Rotate the first entry last of non-pinned groups. Rotation might be
4121         * disabled by the inheritance code.
4122         */
4123        if (ctx->rotate_disable)
4124                return;
4125
4126        perf_event_groups_delete(&ctx->flexible_groups, event);
4127        perf_event_groups_insert(&ctx->flexible_groups, event);
4128}
4129
4130/* pick an event from the flexible_groups to rotate */
4131static inline struct perf_event *
4132ctx_event_to_rotate(struct perf_event_context *ctx)
4133{
4134        struct perf_event *event;
4135
4136        /* pick the first active flexible event */
4137        event = list_first_entry_or_null(&ctx->flexible_active,
4138                                         struct perf_event, active_list);
4139
4140        /* if no active flexible event, pick the first event */
4141        if (!event) {
4142                event = rb_entry_safe(rb_first(&ctx->flexible_groups.tree),
4143                                      typeof(*event), group_node);
4144        }
4145
4146        /*
4147         * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
4148         * finds there are unschedulable events, it will set it again.
4149         */
4150        ctx->rotate_necessary = 0;
4151
4152        return event;
4153}
4154
4155static bool perf_rotate_context(struct perf_cpu_context *cpuctx)
4156{
4157        struct perf_event *cpu_event = NULL, *task_event = NULL;
4158        struct perf_event_context *task_ctx = NULL;
4159        int cpu_rotate, task_rotate;
4160
4161        /*
4162         * Since we run this from IRQ context, nobody can install new
4163         * events, thus the event count values are stable.
4164         */
4165
4166        cpu_rotate = cpuctx->ctx.rotate_necessary;
4167        task_ctx = cpuctx->task_ctx;
4168        task_rotate = task_ctx ? task_ctx->rotate_necessary : 0;
4169
4170        if (!(cpu_rotate || task_rotate))
4171                return false;
4172
4173        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
4174        perf_pmu_disable(cpuctx->ctx.pmu);
4175
4176        if (task_rotate)
4177                task_event = ctx_event_to_rotate(task_ctx);
4178        if (cpu_rotate)
4179                cpu_event = ctx_event_to_rotate(&cpuctx->ctx);
4180
4181        /*
4182         * As per the order given at ctx_resched() first 'pop' task flexible
4183         * and then, if needed CPU flexible.
4184         */
4185        if (task_event || (task_ctx && cpu_event))
4186                ctx_sched_out(task_ctx, cpuctx, EVENT_FLEXIBLE);
4187        if (cpu_event)
4188                cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
4189
4190        if (task_event)
4191                rotate_ctx(task_ctx, task_event);
4192        if (cpu_event)
4193                rotate_ctx(&cpuctx->ctx, cpu_event);
4194
4195        perf_event_sched_in(cpuctx, task_ctx, current);
4196
4197        perf_pmu_enable(cpuctx->ctx.pmu);
4198        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
4199
4200        return true;
4201}
4202
4203void perf_event_task_tick(void)
4204{
4205        struct list_head *head = this_cpu_ptr(&active_ctx_list);
4206        struct perf_event_context *ctx, *tmp;
4207        int throttled;
4208
4209        lockdep_assert_irqs_disabled();
4210
4211        __this_cpu_inc(perf_throttled_seq);
4212        throttled = __this_cpu_xchg(perf_throttled_count, 0);
4213        tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
4214
4215        list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
4216                perf_adjust_freq_unthr_context(ctx, throttled);
4217}
4218
4219static int event_enable_on_exec(struct perf_event *event,
4220                                struct perf_event_context *ctx)
4221{
4222        if (!event->attr.enable_on_exec)
4223                return 0;
4224
4225        event->attr.enable_on_exec = 0;
4226        if (event->state >= PERF_EVENT_STATE_INACTIVE)
4227                return 0;
4228
4229        perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
4230
4231        return 1;
4232}
4233
4234/*
4235 * Enable all of a task's events that have been marked enable-on-exec.
4236 * This expects task == current.
4237 */
4238static void perf_event_enable_on_exec(int ctxn)
4239{
4240        struct perf_event_context *ctx, *clone_ctx = NULL;
4241        enum event_type_t event_type = 0;
4242        struct perf_cpu_context *cpuctx;
4243        struct perf_event *event;
4244        unsigned long flags;
4245        int enabled = 0;
4246
4247        local_irq_save(flags);
4248        ctx = current->perf_event_ctxp[ctxn];
4249        if (!ctx || !ctx->nr_events)
4250                goto out;
4251
4252        cpuctx = __get_cpu_context(ctx);
4253        perf_ctx_lock(cpuctx, ctx);
4254        ctx_sched_out(ctx, cpuctx, EVENT_TIME);
4255        list_for_each_entry(event, &ctx->event_list, event_entry) {
4256                enabled |= event_enable_on_exec(event, ctx);
4257                event_type |= get_event_type(event);
4258        }
4259
4260        /*
4261         * Unclone and reschedule this context if we enabled any event.
4262         */
4263        if (enabled) {
4264                clone_ctx = unclone_ctx(ctx);
4265                ctx_resched(cpuctx, ctx, event_type);
4266        } else {
4267                ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
4268        }
4269        perf_ctx_unlock(cpuctx, ctx);
4270
4271out:
4272        local_irq_restore(flags);
4273
4274        if (clone_ctx)
4275                put_ctx(clone_ctx);
4276}
4277
4278static void perf_remove_from_owner(struct perf_event *event);
4279static void perf_event_exit_event(struct perf_event *event,
4280                                  struct perf_event_context *ctx);
4281
4282/*
4283 * Removes all events from the current task that have been marked
4284 * remove-on-exec, and feeds their values back to parent events.
4285 */
4286static void perf_event_remove_on_exec(int ctxn)
4287{
4288        struct perf_event_context *ctx, *clone_ctx = NULL;
4289        struct perf_event *event, *next;
4290        LIST_HEAD(free_list);
4291        unsigned long flags;
4292        bool modified = false;
4293
4294        ctx = perf_pin_task_context(current, ctxn);
4295        if (!ctx)
4296                return;
4297
4298        mutex_lock(&ctx->mutex);
4299
4300        if (WARN_ON_ONCE(ctx->task != current))
4301                goto unlock;
4302
4303        list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
4304                if (!event->attr.remove_on_exec)
4305                        continue;
4306
4307                if (!is_kernel_event(event))
4308                        perf_remove_from_owner(event);
4309
4310                modified = true;
4311
4312                perf_event_exit_event(event, ctx);
4313        }
4314
4315        raw_spin_lock_irqsave(&ctx->lock, flags);
4316        if (modified)
4317                clone_ctx = unclone_ctx(ctx);
4318        --ctx->pin_count;
4319        raw_spin_unlock_irqrestore(&ctx->lock, flags);
4320
4321unlock:
4322        mutex_unlock(&ctx->mutex);
4323
4324        put_ctx(ctx);
4325        if (clone_ctx)
4326                put_ctx(clone_ctx);
4327}
4328
4329struct perf_read_data {
4330        struct perf_event *event;
4331        bool group;
4332        int ret;
4333};
4334
4335static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
4336{
4337        u16 local_pkg, event_pkg;
4338
4339        if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
4340                int local_cpu = smp_processor_id();
4341
4342                event_pkg = topology_physical_package_id(event_cpu);
4343                local_pkg = topology_physical_package_id(local_cpu);
4344
4345                if (event_pkg == local_pkg)
4346                        return local_cpu;
4347        }
4348
4349        return event_cpu;
4350}
4351
4352/*
4353 * Cross CPU call to read the hardware event
4354 */
4355static void __perf_event_read(void *info)
4356{
4357        struct perf_read_data *data = info;
4358        struct perf_event *sub, *event = data->event;
4359        struct perf_event_context *ctx = event->ctx;
4360        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
4361        struct pmu *pmu = event->pmu;
4362
4363        /*
4364         * If this is a task context, we need to check whether it is
4365         * the current task context of this cpu.  If not it has been
4366         * scheduled out before the smp call arrived.  In that case
4367         * event->count would have been updated to a recent sample
4368         * when the event was scheduled out.
4369         */
4370        if (ctx->task && cpuctx->task_ctx != ctx)
4371                return;
4372
4373        raw_spin_lock(&ctx->lock);
4374        if (ctx->is_active & EVENT_TIME) {
4375                update_context_time(ctx);
4376                update_cgrp_time_from_event(event);
4377        }
4378
4379        perf_event_update_time(event);
4380        if (data->group)
4381                perf_event_update_sibling_time(event);
4382
4383        if (event->state != PERF_EVENT_STATE_ACTIVE)
4384                goto unlock;
4385
4386        if (!data->group) {
4387                pmu->read(event);
4388                data->ret = 0;
4389                goto unlock;
4390        }
4391
4392        pmu->start_txn(pmu, PERF_PMU_TXN_READ);
4393
4394        pmu->read(event);
4395
4396        for_each_sibling_event(sub, event) {
4397                if (sub->state == PERF_EVENT_STATE_ACTIVE) {
4398                        /*
4399                         * Use sibling's PMU rather than @event's since
4400                         * sibling could be on different (eg: software) PMU.
4401                         */
4402                        sub->pmu->read(sub);
4403                }
4404        }
4405
4406        data->ret = pmu->commit_txn(pmu);
4407
4408unlock:
4409        raw_spin_unlock(&ctx->lock);
4410}
4411
4412static inline u64 perf_event_count(struct perf_event *event)
4413{
4414        return local64_read(&event->count) + atomic64_read(&event->child_count);
4415}
4416
4417/*
4418 * NMI-safe method to read a local event, that is an event that
4419 * is:
4420 *   - either for the current task, or for this CPU
4421 *   - does not have inherit set, for inherited task events
4422 *     will not be local and we cannot read them atomically
4423 *   - must not have a pmu::count method
4424 */
4425int perf_event_read_local(struct perf_event *event, u64 *value,
4426                          u64 *enabled, u64 *running)
4427{
4428        unsigned long flags;
4429        int ret = 0;
4430
4431        /*
4432         * Disabling interrupts avoids all counter scheduling (context
4433         * switches, timer based rotation and IPIs).
4434         */
4435        local_irq_save(flags);
4436
4437        /*
4438         * It must not be an event with inherit set, we cannot read
4439         * all child counters from atomic context.
4440         */
4441        if (event->attr.inherit) {
4442                ret = -EOPNOTSUPP;
4443                goto out;
4444        }
4445
4446        /* If this is a per-task event, it must be for current */
4447        if ((event->attach_state & PERF_ATTACH_TASK) &&
4448            event->hw.target != current) {
4449                ret = -EINVAL;
4450                goto out;
4451        }
4452
4453        /* If this is a per-CPU event, it must be for this CPU */
4454        if (!(event->attach_state & PERF_ATTACH_TASK) &&
4455            event->cpu != smp_processor_id()) {
4456                ret = -EINVAL;
4457                goto out;
4458        }
4459
4460        /* If this is a pinned event it must be running on this CPU */
4461        if (event->attr.pinned && event->oncpu != smp_processor_id()) {
4462                ret = -EBUSY;
4463                goto out;
4464        }
4465
4466        /*
4467         * If the event is currently on this CPU, its either a per-task event,
4468         * or local to this CPU. Furthermore it means its ACTIVE (otherwise
4469         * oncpu == -1).
4470         */
4471        if (event->oncpu == smp_processor_id())
4472                event->pmu->read(event);
4473
4474        *value = local64_read(&event->count);
4475        if (enabled || running) {
4476                u64 now = event->shadow_ctx_time + perf_clock();
4477                u64 __enabled, __running;
4478
4479                __perf_update_times(event, now, &__enabled, &__running);
4480                if (enabled)
4481                        *enabled = __enabled;
4482                if (running)
4483                        *running = __running;
4484        }
4485out:
4486        local_irq_restore(flags);
4487
4488        return ret;
4489}
4490
4491static int perf_event_read(struct perf_event *event, bool group)
4492{
4493        enum perf_event_state state = READ_ONCE(event->state);
4494        int event_cpu, ret = 0;
4495
4496        /*
4497         * If event is enabled and currently active on a CPU, update the
4498         * value in the event structure:
4499         */
4500again:
4501        if (state == PERF_EVENT_STATE_ACTIVE) {
4502                struct perf_read_data data;
4503
4504                /*
4505                 * Orders the ->state and ->oncpu loads such that if we see
4506                 * ACTIVE we must also see the right ->oncpu.
4507                 *
4508                 * Matches the smp_wmb() from event_sched_in().
4509                 */
4510                smp_rmb();
4511
4512                event_cpu = READ_ONCE(event->oncpu);
4513                if ((unsigned)event_cpu >= nr_cpu_ids)
4514                        return 0;
4515
4516                data = (struct perf_read_data){
4517                        .event = event,
4518                        .group = group,
4519                        .ret = 0,
4520                };
4521
4522                preempt_disable();
4523                event_cpu = __perf_event_read_cpu(event, event_cpu);
4524
4525                /*
4526                 * Purposely ignore the smp_call_function_single() return
4527                 * value.
4528                 *
4529                 * If event_cpu isn't a valid CPU it means the event got
4530                 * scheduled out and that will have updated the event count.
4531                 *
4532                 * Therefore, either way, we'll have an up-to-date event count
4533                 * after this.
4534                 */
4535                (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
4536                preempt_enable();
4537                ret = data.ret;
4538
4539        } else if (state == PERF_EVENT_STATE_INACTIVE) {
4540                struct perf_event_context *ctx = event->ctx;
4541                unsigned long flags;
4542
4543                raw_spin_lock_irqsave(&ctx->lock, flags);
4544                state = event->state;
4545                if (state != PERF_EVENT_STATE_INACTIVE) {
4546                        raw_spin_unlock_irqrestore(&ctx->lock, flags);
4547                        goto again;
4548                }
4549
4550                /*
4551                 * May read while context is not active (e.g., thread is
4552                 * blocked), in that case we cannot update context time
4553                 */
4554                if (ctx->is_active & EVENT_TIME) {
4555                        update_context_time(ctx);
4556                        update_cgrp_time_from_event(event);
4557                }
4558
4559                perf_event_update_time(event);
4560                if (group)
4561                        perf_event_update_sibling_time(event);
4562                raw_spin_unlock_irqrestore(&ctx->lock, flags);
4563        }
4564
4565        return ret;
4566}
4567
4568/*
4569 * Initialize the perf_event context in a task_struct:
4570 */
4571static void __perf_event_init_context(struct perf_event_context *ctx)
4572{
4573        raw_spin_lock_init(&ctx->lock);
4574        mutex_init(&ctx->mutex);
4575        INIT_LIST_HEAD(&ctx->active_ctx_list);
4576        perf_event_groups_init(&ctx->pinned_groups);
4577        perf_event_groups_init(&ctx->flexible_groups);
4578        INIT_LIST_HEAD(&ctx->event_list);
4579        INIT_LIST_HEAD(&ctx->pinned_active);
4580        INIT_LIST_HEAD(&ctx->flexible_active);
4581        refcount_set(&ctx->refcount, 1);
4582}
4583
4584static struct perf_event_context *
4585alloc_perf_context(struct pmu *pmu, struct task_struct *task)
4586{
4587        struct perf_event_context *ctx;
4588
4589        ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
4590        if (!ctx)
4591                return NULL;
4592
4593        __perf_event_init_context(ctx);
4594        if (task)
4595                ctx->task = get_task_struct(task);
4596        ctx->pmu = pmu;
4597
4598        return ctx;
4599}
4600
4601static struct task_struct *
4602find_lively_task_by_vpid(pid_t vpid)
4603{
4604        struct task_struct *task;
4605
4606        rcu_read_lock();
4607        if (!vpid)
4608                task = current;
4609        else
4610                task = find_task_by_vpid(vpid);
4611        if (task)
4612                get_task_struct(task);
4613        rcu_read_unlock();
4614
4615        if (!task)
4616                return ERR_PTR(-ESRCH);
4617
4618        return task;
4619}
4620
4621/*
4622 * Returns a matching context with refcount and pincount.
4623 */
4624static struct perf_event_context *
4625find_get_context(struct pmu *pmu, struct task_struct *task,
4626                struct perf_event *event)
4627{
4628        struct perf_event_context *ctx, *clone_ctx = NULL;
4629        struct perf_cpu_context *cpuctx;
4630        void *task_ctx_data = NULL;
4631        unsigned long flags;
4632        int ctxn, err;
4633        int cpu = event->cpu;
4634
4635        if (!task) {
4636                /* Must be root to operate on a CPU event: */
4637                err = perf_allow_cpu(&event->attr);
4638                if (err)
4639                        return ERR_PTR(err);
4640
4641                cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
4642                ctx = &cpuctx->ctx;
4643                get_ctx(ctx);
4644                raw_spin_lock_irqsave(&ctx->lock, flags);
4645                ++ctx->pin_count;
4646                raw_spin_unlock_irqrestore(&ctx->lock, flags);
4647
4648                return ctx;
4649        }
4650
4651        err = -EINVAL;
4652        ctxn = pmu->task_ctx_nr;
4653        if (ctxn < 0)
4654                goto errout;
4655
4656        if (event->attach_state & PERF_ATTACH_TASK_DATA) {
4657                task_ctx_data = alloc_task_ctx_data(pmu);
4658                if (!task_ctx_data) {
4659                        err = -ENOMEM;
4660                        goto errout;
4661                }
4662        }
4663
4664retry:
4665        ctx = perf_lock_task_context(task, ctxn, &flags);
4666        if (ctx) {
4667                clone_ctx = unclone_ctx(ctx);
4668                ++ctx->pin_count;
4669
4670                if (task_ctx_data && !ctx->task_ctx_data) {
4671                        ctx->task_ctx_data = task_ctx_data;
4672                        task_ctx_data = NULL;
4673                }
4674                raw_spin_unlock_irqrestore(&ctx->lock, flags);
4675
4676                if (clone_ctx)
4677                        put_ctx(clone_ctx);
4678        } else {
4679                ctx = alloc_perf_context(pmu, task);
4680                err = -ENOMEM;
4681                if (!ctx)
4682                        goto errout;
4683
4684                if (task_ctx_data) {
4685                        ctx->task_ctx_data = task_ctx_data;
4686                        task_ctx_data = NULL;
4687                }
4688
4689                err = 0;
4690                mutex_lock(&task->perf_event_mutex);
4691                /*
4692                 * If it has already passed perf_event_exit_task().
4693                 * we must see PF_EXITING, it takes this mutex too.
4694                 */
4695                if (task->flags & PF_EXITING)
4696                        err = -ESRCH;
4697                else if (task->perf_event_ctxp[ctxn])
4698                        err = -EAGAIN;
4699                else {
4700                        get_ctx(ctx);
4701                        ++ctx->pin_count;
4702                        rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
4703                }
4704                mutex_unlock(&task->perf_event_mutex);
4705
4706                if (unlikely(err)) {
4707                        put_ctx(ctx);
4708
4709                        if (err == -EAGAIN)
4710                                goto retry;
4711                        goto errout;
4712                }
4713        }
4714
4715        free_task_ctx_data(pmu, task_ctx_data);
4716        return ctx;
4717
4718errout:
4719        free_task_ctx_data(pmu, task_ctx_data);
4720        return ERR_PTR(err);
4721}
4722
4723static void perf_event_free_filter(struct perf_event *event);
4724
4725static void free_event_rcu(struct rcu_head *head)
4726{
4727        struct perf_event *event;
4728
4729        event = container_of(head, struct perf_event, rcu_head);
4730        if (event->ns)
4731                put_pid_ns(event->ns);
4732        perf_event_free_filter(event);
4733        kmem_cache_free(perf_event_cache, event);
4734}
4735
4736static void ring_buffer_attach(struct perf_event *event,
4737                               struct perf_buffer *rb);
4738
4739static void detach_sb_event(struct perf_event *event)
4740{
4741        struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
4742
4743        raw_spin_lock(&pel->lock);
4744        list_del_rcu(&event->sb_list);
4745        raw_spin_unlock(&pel->lock);
4746}
4747
4748static bool is_sb_event(struct perf_event *event)
4749{
4750        struct perf_event_attr *attr = &event->attr;
4751
4752        if (event->parent)
4753                return false;
4754
4755        if (event->attach_state & PERF_ATTACH_TASK)
4756                return false;
4757
4758        if (attr->mmap || attr->mmap_data || attr->mmap2 ||
4759            attr->comm || attr->comm_exec ||
4760            attr->task || attr->ksymbol ||
4761            attr->context_switch || attr->text_poke ||
4762            attr->bpf_event)
4763                return true;
4764        return false;
4765}
4766
4767static void unaccount_pmu_sb_event(struct perf_event *event)
4768{
4769        if (is_sb_event(event))
4770                detach_sb_event(event);
4771}
4772
4773static void unaccount_event_cpu(struct perf_event *event, int cpu)
4774{
4775        if (event->parent)
4776                return;
4777
4778        if (is_cgroup_event(event))
4779                atomic_dec(&per_cpu(perf_cgroup_events, cpu));
4780}
4781
4782#ifdef CONFIG_NO_HZ_FULL
4783static DEFINE_SPINLOCK(nr_freq_lock);
4784#endif
4785
4786static void unaccount_freq_event_nohz(void)
4787{
4788#ifdef CONFIG_NO_HZ_FULL
4789        spin_lock(&nr_freq_lock);
4790        if (atomic_dec_and_test(&nr_freq_events))
4791                tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
4792        spin_unlock(&nr_freq_lock);
4793#endif
4794}
4795
4796static void unaccount_freq_event(void)
4797{
4798        if (tick_nohz_full_enabled())
4799                unaccount_freq_event_nohz();
4800        else
4801                atomic_dec(&nr_freq_events);
4802}
4803
4804static void unaccount_event(struct perf_event *event)
4805{
4806        bool dec = false;
4807
4808        if (event->parent)
4809                return;
4810
4811        if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
4812                dec = true;
4813        if (event->attr.mmap || event->attr.mmap_data)
4814                atomic_dec(&nr_mmap_events);
4815        if (event->attr.build_id)
4816                atomic_dec(&nr_build_id_events);
4817        if (event->attr.comm)
4818                atomic_dec(&nr_comm_events);
4819        if (event->attr.namespaces)
4820                atomic_dec(&nr_namespaces_events);
4821        if (event->attr.cgroup)
4822                atomic_dec(&nr_cgroup_events);
4823        if (event->attr.task)
4824                atomic_dec(&nr_task_events);
4825        if (event->attr.freq)
4826                unaccount_freq_event();
4827        if (event->attr.context_switch) {
4828                dec = true;
4829                atomic_dec(&nr_switch_events);
4830        }
4831        if (is_cgroup_event(event))
4832                dec = true;
4833        if (has_branch_stack(event))
4834                dec = true;
4835        if (event->attr.ksymbol)
4836                atomic_dec(&nr_ksymbol_events);
4837        if (event->attr.bpf_event)
4838                atomic_dec(&nr_bpf_events);
4839        if (event->attr.text_poke)
4840                atomic_dec(&nr_text_poke_events);
4841
4842        if (dec) {
4843                if (!atomic_add_unless(&perf_sched_count, -1, 1))
4844                        schedule_delayed_work(&perf_sched_work, HZ);
4845        }
4846
4847        unaccount_event_cpu(event, event->cpu);
4848
4849        unaccount_pmu_sb_event(event);
4850}
4851
4852static void perf_sched_delayed(struct work_struct *work)
4853{
4854        mutex_lock(&perf_sched_mutex);
4855        if (atomic_dec_and_test(&perf_sched_count))
4856                static_branch_disable(&perf_sched_events);
4857        mutex_unlock(&perf_sched_mutex);
4858}
4859
4860/*
4861 * The following implement mutual exclusion of events on "exclusive" pmus
4862 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
4863 * at a time, so we disallow creating events that might conflict, namely:
4864 *
4865 *  1) cpu-wide events in the presence of per-task events,
4866 *  2) per-task events in the presence of cpu-wide events,
4867 *  3) two matching events on the same context.
4868 *
4869 * The former two cases are handled in the allocation path (perf_event_alloc(),
4870 * _free_event()), the latter -- before the first perf_install_in_context().
4871 */
4872static int exclusive_event_init(struct perf_event *event)
4873{
4874        struct pmu *pmu = event->pmu;
4875
4876        if (!is_exclusive_pmu(pmu))
4877                return 0;
4878
4879        /*
4880         * Prevent co-existence of per-task and cpu-wide events on the
4881         * same exclusive pmu.
4882         *
4883         * Negative pmu::exclusive_cnt means there are cpu-wide
4884         * events on this "exclusive" pmu, positive means there are
4885         * per-task events.
4886         *
4887         * Since this is called in perf_event_alloc() path, event::ctx
4888         * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
4889         * to mean "per-task event", because unlike other attach states it
4890         * never gets cleared.
4891         */
4892        if (event->attach_state & PERF_ATTACH_TASK) {
4893                if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
4894                        return -EBUSY;
4895        } else {
4896                if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
4897                        return -EBUSY;
4898        }
4899
4900        return 0;
4901}
4902
4903static void exclusive_event_destroy(struct perf_event *event)
4904{
4905        struct pmu *pmu = event->pmu;
4906
4907        if (!is_exclusive_pmu(pmu))
4908                return;
4909
4910        /* see comment in exclusive_event_init() */
4911        if (event->attach_state & PERF_ATTACH_TASK)
4912                atomic_dec(&pmu->exclusive_cnt);
4913        else
4914                atomic_inc(&pmu->exclusive_cnt);
4915}
4916
4917static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
4918{
4919        if ((e1->pmu == e2->pmu) &&
4920            (e1->cpu == e2->cpu ||
4921             e1->cpu == -1 ||
4922             e2->cpu == -1))
4923                return true;
4924        return false;
4925}
4926
4927static bool exclusive_event_installable(struct perf_event *event,
4928                                        struct perf_event_context *ctx)
4929{
4930        struct perf_event *iter_event;
4931        struct pmu *pmu = event->pmu;
4932
4933        lockdep_assert_held(&ctx->mutex);
4934
4935        if (!is_exclusive_pmu(pmu))
4936                return true;
4937
4938        list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
4939                if (exclusive_event_match(iter_event, event))
4940                        return false;
4941        }
4942
4943        return true;
4944}
4945
4946static void perf_addr_filters_splice(struct perf_event *event,
4947                                       struct list_head *head);
4948
4949static void _free_event(struct perf_event *event)
4950{
4951        irq_work_sync(&event->pending);
4952
4953        unaccount_event(event);
4954
4955        security_perf_event_free(event);
4956
4957        if (event->rb) {
4958                /*
4959                 * Can happen when we close an event with re-directed output.
4960                 *
4961                 * Since we have a 0 refcount, perf_mmap_close() will skip
4962                 * over us; possibly making our ring_buffer_put() the last.
4963                 */
4964                mutex_lock(&event->mmap_mutex);
4965                ring_buffer_attach(event, NULL);
4966                mutex_unlock(&event->mmap_mutex);
4967        }
4968
4969        if (is_cgroup_event(event))
4970                perf_detach_cgroup(event);
4971
4972        if (!event->parent) {
4973                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
4974                        put_callchain_buffers();
4975        }
4976
4977        perf_event_free_bpf_prog(event);
4978        perf_addr_filters_splice(event, NULL);
4979        kfree(event->addr_filter_ranges);
4980
4981        if (event->destroy)
4982                event->destroy(event);
4983
4984        /*
4985         * Must be after ->destroy(), due to uprobe_perf_close() using
4986         * hw.target.
4987         */
4988        if (event->hw.target)
4989                put_task_struct(event->hw.target);
4990
4991        /*
4992         * perf_event_free_task() relies on put_ctx() being 'last', in particular
4993         * all task references must be cleaned up.
4994         */
4995        if (event->ctx)
4996                put_ctx(event->ctx);
4997
4998        exclusive_event_destroy(event);
4999        module_put(event->pmu->module);
5000
5001        call_rcu(&event->rcu_head, free_event_rcu);
5002}
5003
5004/*
5005 * Used to free events which have a known refcount of 1, such as in error paths
5006 * where the event isn't exposed yet and inherited events.
5007 */
5008static void free_event(struct perf_event *event)
5009{
5010        if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5011                                "unexpected event refcount: %ld; ptr=%p\n",
5012                                atomic_long_read(&event->refcount), event)) {
5013                /* leak to avoid use-after-free */
5014                return;
5015        }
5016
5017        _free_event(event);
5018}
5019
5020/*
5021 * Remove user event from the owner task.
5022 */
5023static void perf_remove_from_owner(struct perf_event *event)
5024{
5025        struct task_struct *owner;
5026
5027        rcu_read_lock();
5028        /*
5029         * Matches the smp_store_release() in perf_event_exit_task(). If we
5030         * observe !owner it means the list deletion is complete and we can
5031         * indeed free this event, otherwise we need to serialize on
5032         * owner->perf_event_mutex.
5033         */
5034        owner = READ_ONCE(event->owner);
5035        if (owner) {
5036                /*
5037                 * Since delayed_put_task_struct() also drops the last
5038                 * task reference we can safely take a new reference
5039                 * while holding the rcu_read_lock().
5040                 */
5041                get_task_struct(owner);
5042        }
5043        rcu_read_unlock();
5044
5045        if (owner) {
5046                /*
5047                 * If we're here through perf_event_exit_task() we're already
5048                 * holding ctx->mutex which would be an inversion wrt. the
5049                 * normal lock order.
5050                 *
5051                 * However we can safely take this lock because its the child
5052                 * ctx->mutex.
5053                 */
5054                mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
5055
5056                /*
5057                 * We have to re-check the event->owner field, if it is cleared
5058                 * we raced with perf_event_exit_task(), acquiring the mutex
5059                 * ensured they're done, and we can proceed with freeing the
5060                 * event.
5061                 */
5062                if (event->owner) {
5063                        list_del_init(&event->owner_entry);
5064                        smp_store_release(&event->owner, NULL);
5065                }
5066                mutex_unlock(&owner->perf_event_mutex);
5067                put_task_struct(owner);
5068        }
5069}
5070
5071static void put_event(struct perf_event *event)
5072{
5073        if (!atomic_long_dec_and_test(&event->refcount))
5074                return;
5075
5076        _free_event(event);
5077}
5078
5079/*
5080 * Kill an event dead; while event:refcount will preserve the event
5081 * object, it will not preserve its functionality. Once the last 'user'
5082 * gives up the object, we'll destroy the thing.
5083 */
5084int perf_event_release_kernel(struct perf_event *event)
5085{
5086        struct perf_event_context *ctx = event->ctx;
5087        struct perf_event *child, *tmp;
5088        LIST_HEAD(free_list);
5089
5090        /*
5091         * If we got here through err_file: fput(event_file); we will not have
5092         * attached to a context yet.
5093         */
5094        if (!ctx) {
5095                WARN_ON_ONCE(event->attach_state &
5096                                (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
5097                goto no_ctx;
5098        }
5099
5100        if (!is_kernel_event(event))
5101                perf_remove_from_owner(event);
5102
5103        ctx = perf_event_ctx_lock(event);
5104        WARN_ON_ONCE(ctx->parent_ctx);
5105        perf_remove_from_context(event, DETACH_GROUP);
5106
5107        raw_spin_lock_irq(&ctx->lock);
5108        /*
5109         * Mark this event as STATE_DEAD, there is no external reference to it
5110         * anymore.
5111         *
5112         * Anybody acquiring event->child_mutex after the below loop _must_
5113         * also see this, most importantly inherit_event() which will avoid
5114         * placing more children on the list.
5115         *
5116         * Thus this guarantees that we will in fact observe and kill _ALL_
5117         * child events.
5118         */
5119        event->state = PERF_EVENT_STATE_DEAD;
5120        raw_spin_unlock_irq(&ctx->lock);
5121
5122        perf_event_ctx_unlock(event, ctx);
5123
5124again:
5125        mutex_lock(&event->child_mutex);
5126        list_for_each_entry(child, &event->child_list, child_list) {
5127
5128                /*
5129                 * Cannot change, child events are not migrated, see the
5130                 * comment with perf_event_ctx_lock_nested().
5131                 */
5132                ctx = READ_ONCE(child->ctx);
5133                /*
5134                 * Since child_mutex nests inside ctx::mutex, we must jump
5135                 * through hoops. We start by grabbing a reference on the ctx.
5136                 *
5137                 * Since the event cannot get freed while we hold the
5138                 * child_mutex, the context must also exist and have a !0
5139                 * reference count.
5140                 */
5141                get_ctx(ctx);
5142
5143                /*
5144                 * Now that we have a ctx ref, we can drop child_mutex, and
5145                 * acquire ctx::mutex without fear of it going away. Then we
5146                 * can re-acquire child_mutex.
5147                 */
5148                mutex_unlock(&event->child_mutex);
5149                mutex_lock(&ctx->mutex);
5150                mutex_lock(&event->child_mutex);
5151
5152                /*
5153                 * Now that we hold ctx::mutex and child_mutex, revalidate our
5154                 * state, if child is still the first entry, it didn't get freed
5155                 * and we can continue doing so.
5156                 */
5157                tmp = list_first_entry_or_null(&event->child_list,
5158                                               struct perf_event, child_list);
5159                if (tmp == child) {
5160                        perf_remove_from_context(child, DETACH_GROUP);
5161                        list_move(&child->child_list, &free_list);
5162                        /*
5163                         * This matches the refcount bump in inherit_event();
5164                         * this can't be the last reference.
5165                         */
5166                        put_event(event);
5167                }
5168
5169                mutex_unlock(&event->child_mutex);
5170                mutex_unlock(&ctx->mutex);
5171                put_ctx(ctx);
5172                goto again;
5173        }
5174        mutex_unlock(&event->child_mutex);
5175
5176        list_for_each_entry_safe(child, tmp, &free_list, child_list) {
5177                void *var = &child->ctx->refcount;
5178
5179                list_del(&child->child_list);
5180                free_event(child);
5181
5182                /*
5183                 * Wake any perf_event_free_task() waiting for this event to be
5184                 * freed.
5185                 */
5186                smp_mb(); /* pairs with wait_var_event() */
5187                wake_up_var(var);
5188        }
5189
5190no_ctx:
5191        put_event(event); /* Must be the 'last' reference */
5192        return 0;
5193}
5194EXPORT_SYMBOL_GPL(perf_event_release_kernel);
5195
5196/*
5197 * Called when the last reference to the file is gone.
5198 */
5199static int perf_release(struct inode *inode, struct file *file)
5200{
5201        perf_event_release_kernel(file->private_data);
5202        return 0;
5203}
5204
5205static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5206{
5207        struct perf_event *child;
5208        u64 total = 0;
5209
5210        *enabled = 0;
5211        *running = 0;
5212
5213        mutex_lock(&event->child_mutex);
5214
5215        (void)perf_event_read(event, false);
5216        total += perf_event_count(event);
5217
5218        *enabled += event->total_time_enabled +
5219                        atomic64_read(&event->child_total_time_enabled);
5220        *running += event->total_time_running +
5221                        atomic64_read(&event->child_total_time_running);
5222
5223        list_for_each_entry(child, &event->child_list, child_list) {
5224                (void)perf_event_read(child, false);
5225                total += perf_event_count(child);
5226                *enabled += child->total_time_enabled;
5227                *running += child->total_time_running;
5228        }
5229        mutex_unlock(&event->child_mutex);
5230
5231        return total;
5232}
5233
5234u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5235{
5236        struct perf_event_context *ctx;
5237        u64 count;
5238
5239        ctx = perf_event_ctx_lock(event);
5240        count = __perf_event_read_value(event, enabled, running);
5241        perf_event_ctx_unlock(event, ctx);
5242
5243        return count;
5244}
5245EXPORT_SYMBOL_GPL(perf_event_read_value);
5246
5247static int __perf_read_group_add(struct perf_event *leader,
5248                                        u64 read_format, u64 *values)
5249{
5250        struct perf_event_context *ctx = leader->ctx;
5251        struct perf_event *sub;
5252        unsigned long flags;
5253        int n = 1; /* skip @nr */
5254        int ret;
5255
5256        ret = perf_event_read(leader, true);
5257        if (ret)
5258                return ret;
5259
5260        raw_spin_lock_irqsave(&ctx->lock, flags);
5261
5262        /*
5263         * Since we co-schedule groups, {enabled,running} times of siblings
5264         * will be identical to those of the leader, so we only publish one
5265         * set.
5266         */
5267        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5268                values[n++] += leader->total_time_enabled +
5269                        atomic64_read(&leader->child_total_time_enabled);
5270        }
5271
5272        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5273                values[n++] += leader->total_time_running +
5274                        atomic64_read(&leader->child_total_time_running);
5275        }
5276
5277        /*
5278         * Write {count,id} tuples for every sibling.
5279         */
5280        values[n++] += perf_event_count(leader);
5281        if (read_format & PERF_FORMAT_ID)
5282                values[n++] = primary_event_id(leader);
5283
5284        for_each_sibling_event(sub, leader) {
5285                values[n++] += perf_event_count(sub);
5286                if (read_format & PERF_FORMAT_ID)
5287                        values[n++] = primary_event_id(sub);
5288        }
5289
5290        raw_spin_unlock_irqrestore(&ctx->lock, flags);
5291        return 0;
5292}
5293
5294static int perf_read_group(struct perf_event *event,
5295                                   u64 read_format, char __user *buf)
5296{
5297        struct perf_event *leader = event->group_leader, *child;
5298        struct perf_event_context *ctx = leader->ctx;
5299        int ret;
5300        u64 *values;
5301
5302        lockdep_assert_held(&ctx->mutex);
5303
5304        values = kzalloc(event->read_size, GFP_KERNEL);
5305        if (!values)
5306                return -ENOMEM;
5307
5308        values[0] = 1 + leader->nr_siblings;
5309
5310        /*
5311         * By locking the child_mutex of the leader we effectively
5312         * lock the child list of all siblings.. XXX explain how.
5313         */
5314        mutex_lock(&leader->child_mutex);
5315
5316        ret = __perf_read_group_add(leader, read_format, values);
5317        if (ret)
5318                goto unlock;
5319
5320        list_for_each_entry(child, &leader->child_list, child_list) {
5321                ret = __perf_read_group_add(child, read_format, values);
5322                if (ret)
5323                        goto unlock;
5324        }
5325
5326        mutex_unlock(&leader->child_mutex);
5327
5328        ret = event->read_size;
5329        if (copy_to_user(buf, values, event->read_size))
5330                ret = -EFAULT;
5331        goto out;
5332
5333unlock:
5334        mutex_unlock(&leader->child_mutex);
5335out:
5336        kfree(values);
5337        return ret;
5338}
5339
5340static int perf_read_one(struct perf_event *event,
5341                                 u64 read_format, char __user *buf)
5342{
5343        u64 enabled, running;
5344        u64 values[4];
5345        int n = 0;
5346
5347        values[n++] = __perf_event_read_value(event, &enabled, &running);
5348        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5349                values[n++] = enabled;
5350        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5351                values[n++] = running;
5352        if (read_format & PERF_FORMAT_ID)
5353                values[n++] = primary_event_id(event);
5354
5355        if (copy_to_user(buf, values, n * sizeof(u64)))
5356                return -EFAULT;
5357
5358        return n * sizeof(u64);
5359}
5360
5361static bool is_event_hup(struct perf_event *event)
5362{
5363        bool no_children;
5364
5365        if (event->state > PERF_EVENT_STATE_EXIT)
5366                return false;
5367
5368        mutex_lock(&event->child_mutex);
5369        no_children = list_empty(&event->child_list);
5370        mutex_unlock(&event->child_mutex);
5371        return no_children;
5372}
5373
5374/*
5375 * Read the performance event - simple non blocking version for now
5376 */
5377static ssize_t
5378__perf_read(struct perf_event *event, char __user *buf, size_t count)
5379{
5380        u64 read_format = event->attr.read_format;
5381        int ret;
5382
5383        /*
5384         * Return end-of-file for a read on an event that is in
5385         * error state (i.e. because it was pinned but it couldn't be
5386         * scheduled on to the CPU at some point).
5387         */
5388        if (event->state == PERF_EVENT_STATE_ERROR)
5389                return 0;
5390
5391        if (count < event->read_size)
5392                return -ENOSPC;
5393
5394        WARN_ON_ONCE(event->ctx->parent_ctx);
5395        if (read_format & PERF_FORMAT_GROUP)
5396                ret = perf_read_group(event, read_format, buf);
5397        else
5398                ret = perf_read_one(event, read_format, buf);
5399
5400        return ret;
5401}
5402
5403static ssize_t
5404perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
5405{
5406        struct perf_event *event = file->private_data;
5407        struct perf_event_context *ctx;
5408        int ret;
5409
5410        ret = security_perf_event_read(event);
5411        if (ret)
5412                return ret;
5413
5414        ctx = perf_event_ctx_lock(event);
5415        ret = __perf_read(event, buf, count);
5416        perf_event_ctx_unlock(event, ctx);
5417
5418        return ret;
5419}
5420
5421static __poll_t perf_poll(struct file *file, poll_table *wait)
5422{
5423        struct perf_event *event = file->private_data;
5424        struct perf_buffer *rb;
5425        __poll_t events = EPOLLHUP;
5426
5427        poll_wait(file, &event->waitq, wait);
5428
5429        if (is_event_hup(event))
5430                return events;
5431
5432        /*
5433         * Pin the event->rb by taking event->mmap_mutex; otherwise
5434         * perf_event_set_output() can swizzle our rb and make us miss wakeups.
5435         */
5436        mutex_lock(&event->mmap_mutex);
5437        rb = event->rb;
5438        if (rb)
5439                events = atomic_xchg(&rb->poll, 0);
5440        mutex_unlock(&event->mmap_mutex);
5441        return events;
5442}
5443
5444static void _perf_event_reset(struct perf_event *event)
5445{
5446        (void)perf_event_read(event, false);
5447        local64_set(&event->count, 0);
5448        perf_event_update_userpage(event);
5449}
5450
5451/* Assume it's not an event with inherit set. */
5452u64 perf_event_pause(struct perf_event *event, bool reset)
5453{
5454        struct perf_event_context *ctx;
5455        u64 count;
5456
5457        ctx = perf_event_ctx_lock(event);
5458        WARN_ON_ONCE(event->attr.inherit);
5459        _perf_event_disable(event);
5460        count = local64_read(&event->count);
5461        if (reset)
5462                local64_set(&event->count, 0);
5463        perf_event_ctx_unlock(event, ctx);
5464
5465        return count;
5466}
5467EXPORT_SYMBOL_GPL(perf_event_pause);
5468
5469/*
5470 * Holding the top-level event's child_mutex means that any
5471 * descendant process that has inherited this event will block
5472 * in perf_event_exit_event() if it goes to exit, thus satisfying the
5473 * task existence requirements of perf_event_enable/disable.
5474 */
5475static void perf_event_for_each_child(struct perf_event *event,
5476                                        void (*func)(struct perf_event *))
5477{
5478        struct perf_event *child;
5479
5480        WARN_ON_ONCE(event->ctx->parent_ctx);
5481
5482        mutex_lock(&event->child_mutex);
5483        func(event);
5484        list_for_each_entry(child, &event->child_list, child_list)
5485                func(child);
5486        mutex_unlock(&event->child_mutex);
5487}
5488
5489static void perf_event_for_each(struct perf_event *event,
5490                                  void (*func)(struct perf_event *))
5491{
5492        struct perf_event_context *ctx = event->ctx;
5493        struct perf_event *sibling;
5494
5495        lockdep_assert_held(&ctx->mutex);
5496
5497        event = event->group_leader;
5498
5499        perf_event_for_each_child(event, func);
5500        for_each_sibling_event(sibling, event)
5501                perf_event_for_each_child(sibling, func);
5502}
5503
5504static void __perf_event_period(struct perf_event *event,
5505                                struct perf_cpu_context *cpuctx,
5506                                struct perf_event_context *ctx,
5507                                void *info)
5508{
5509        u64 value = *((u64 *)info);
5510        bool active;
5511
5512        if (event->attr.freq) {
5513                event->attr.sample_freq = value;
5514        } else {
5515                event->attr.sample_period = value;
5516                event->hw.sample_period = value;
5517        }
5518
5519        active = (event->state == PERF_EVENT_STATE_ACTIVE);
5520        if (active) {
5521                perf_pmu_disable(ctx->pmu);
5522                /*
5523                 * We could be throttled; unthrottle now to avoid the tick
5524                 * trying to unthrottle while we already re-started the event.
5525                 */
5526                if (event->hw.interrupts == MAX_INTERRUPTS) {
5527                        event->hw.interrupts = 0;
5528                        perf_log_throttle(event, 1);
5529                }
5530                event->pmu->stop(event, PERF_EF_UPDATE);
5531        }
5532
5533        local64_set(&event->hw.period_left, 0);
5534
5535        if (active) {
5536                event->pmu->start(event, PERF_EF_RELOAD);
5537                perf_pmu_enable(ctx->pmu);
5538        }
5539}
5540
5541static int perf_event_check_period(struct perf_event *event, u64 value)
5542{
5543        return event->pmu->check_period(event, value);
5544}
5545
5546static int _perf_event_period(struct perf_event *event, u64 value)
5547{
5548        if (!is_sampling_event(event))
5549                return -EINVAL;
5550
5551        if (!value)
5552                return -EINVAL;
5553
5554        if (event->attr.freq && value > sysctl_perf_event_sample_rate)
5555                return -EINVAL;
5556
5557        if (perf_event_check_period(event, value))
5558                return -EINVAL;
5559
5560        if (!event->attr.freq && (value & (1ULL << 63)))
5561                return -EINVAL;
5562
5563        event_function_call(event, __perf_event_period, &value);
5564
5565        return 0;
5566}
5567
5568int perf_event_period(struct perf_event *event, u64 value)
5569{
5570        struct perf_event_context *ctx;
5571        int ret;
5572
5573        ctx = perf_event_ctx_lock(event);
5574        ret = _perf_event_period(event, value);
5575        perf_event_ctx_unlock(event, ctx);
5576
5577        return ret;
5578}
5579EXPORT_SYMBOL_GPL(perf_event_period);
5580
5581static const struct file_operations perf_fops;
5582
5583static inline int perf_fget_light(int fd, struct fd *p)
5584{
5585        struct fd f = fdget(fd);
5586        if (!f.file)
5587                return -EBADF;
5588
5589        if (f.file->f_op != &perf_fops) {
5590                fdput(f);
5591                return -EBADF;
5592        }
5593        *p = f;
5594        return 0;
5595}
5596
5597static int perf_event_set_output(struct perf_event *event,
5598                                 struct perf_event *output_event);
5599static int perf_event_set_filter(struct perf_event *event, void __user *arg);
5600static int perf_copy_attr(struct perf_event_attr __user *uattr,
5601                          struct perf_event_attr *attr);
5602
5603static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
5604{
5605        void (*func)(struct perf_event *);
5606        u32 flags = arg;
5607
5608        switch (cmd) {
5609        case PERF_EVENT_IOC_ENABLE:
5610                func = _perf_event_enable;
5611                break;
5612        case PERF_EVENT_IOC_DISABLE:
5613                func = _perf_event_disable;
5614                break;
5615        case PERF_EVENT_IOC_RESET:
5616                func = _perf_event_reset;
5617                break;
5618
5619        case PERF_EVENT_IOC_REFRESH:
5620                return _perf_event_refresh(event, arg);
5621
5622        case PERF_EVENT_IOC_PERIOD:
5623        {
5624                u64 value;
5625
5626                if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
5627                        return -EFAULT;
5628
5629                return _perf_event_period(event, value);
5630        }
5631        case PERF_EVENT_IOC_ID:
5632        {
5633                u64 id = primary_event_id(event);
5634
5635                if (copy_to_user((void __user *)arg, &id, sizeof(id)))
5636                        return -EFAULT;
5637                return 0;
5638        }
5639
5640        case PERF_EVENT_IOC_SET_OUTPUT:
5641        {
5642                int ret;
5643                if (arg != -1) {
5644                        struct perf_event *output_event;
5645                        struct fd output;
5646                        ret = perf_fget_light(arg, &output);
5647                        if (ret)
5648                                return ret;
5649                        output_event = output.file->private_data;
5650                        ret = perf_event_set_output(event, output_event);
5651                        fdput(output);
5652                } else {
5653                        ret = perf_event_set_output(event, NULL);
5654                }
5655                return ret;
5656        }
5657
5658        case PERF_EVENT_IOC_SET_FILTER:
5659                return perf_event_set_filter(event, (void __user *)arg);
5660
5661        case PERF_EVENT_IOC_SET_BPF:
5662        {
5663                struct bpf_prog *prog;
5664                int err;
5665
5666                prog = bpf_prog_get(arg);
5667                if (IS_ERR(prog))
5668                        return PTR_ERR(prog);
5669
5670                err = perf_event_set_bpf_prog(event, prog, 0);
5671                if (err) {
5672                        bpf_prog_put(prog);
5673                        return err;
5674                }
5675
5676                return 0;
5677        }
5678
5679        case PERF_EVENT_IOC_PAUSE_OUTPUT: {
5680                struct perf_buffer *rb;
5681
5682                rcu_read_lock();
5683                rb = rcu_dereference(event->rb);
5684                if (!rb || !rb->nr_pages) {
5685                        rcu_read_unlock();
5686                        return -EINVAL;
5687                }
5688                rb_toggle_paused(rb, !!arg);
5689                rcu_read_unlock();
5690                return 0;
5691        }
5692
5693        case PERF_EVENT_IOC_QUERY_BPF:
5694                return perf_event_query_prog_array(event, (void __user *)arg);
5695
5696        case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
5697                struct perf_event_attr new_attr;
5698                int err = perf_copy_attr((struct perf_event_attr __user *)arg,
5699                                         &new_attr);
5700
5701                if (err)
5702                        return err;
5703
5704                return perf_event_modify_attr(event,  &new_attr);
5705        }
5706        default:
5707                return -ENOTTY;
5708        }
5709
5710        if (flags & PERF_IOC_FLAG_GROUP)
5711                perf_event_for_each(event, func);
5712        else
5713                perf_event_for_each_child(event, func);
5714
5715        return 0;
5716}
5717
5718static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
5719{
5720        struct perf_event *event = file->private_data;
5721        struct perf_event_context *ctx;
5722        long ret;
5723
5724        /* Treat ioctl like writes as it is likely a mutating operation. */
5725        ret = security_perf_event_write(event);
5726        if (ret)
5727                return ret;
5728
5729        ctx = perf_event_ctx_lock(event);
5730        ret = _perf_ioctl(event, cmd, arg);
5731        perf_event_ctx_unlock(event, ctx);
5732
5733        return ret;
5734}
5735
5736#ifdef CONFIG_COMPAT
5737static long perf_compat_ioctl(struct file *file, unsigned int cmd,
5738                                unsigned long arg)
5739{
5740        switch (_IOC_NR(cmd)) {
5741        case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
5742        case _IOC_NR(PERF_EVENT_IOC_ID):
5743        case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
5744        case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
5745                /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
5746                if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
5747                        cmd &= ~IOCSIZE_MASK;
5748                        cmd |= sizeof(void *) << IOCSIZE_SHIFT;
5749                }
5750                break;
5751        }
5752        return perf_ioctl(file, cmd, arg);
5753}
5754#else
5755# define perf_compat_ioctl NULL
5756#endif
5757
5758int perf_event_task_enable(void)
5759{
5760        struct perf_event_context *ctx;
5761        struct perf_event *event;
5762
5763        mutex_lock(&current->perf_event_mutex);
5764        list_for_each_entry(event, &current->perf_event_list, owner_entry) {
5765                ctx = perf_event_ctx_lock(event);
5766                perf_event_for_each_child(event, _perf_event_enable);
5767                perf_event_ctx_unlock(event, ctx);
5768        }
5769        mutex_unlock(&current->perf_event_mutex);
5770
5771        return 0;
5772}
5773
5774int perf_event_task_disable(void)
5775{
5776        struct perf_event_context *ctx;
5777        struct perf_event *event;
5778
5779        mutex_lock(&current->perf_event_mutex);
5780        list_for_each_entry(event, &current->perf_event_list, owner_entry) {
5781                ctx = perf_event_ctx_lock(event);
5782                perf_event_for_each_child(event, _perf_event_disable);
5783                perf_event_ctx_unlock(event, ctx);
5784        }
5785        mutex_unlock(&current->perf_event_mutex);
5786
5787        return 0;
5788}
5789
5790static int perf_event_index(struct perf_event *event)
5791{
5792        if (event->hw.state & PERF_HES_STOPPED)
5793                return 0;
5794
5795        if (event->state != PERF_EVENT_STATE_ACTIVE)
5796                return 0;
5797
5798        return event->pmu->event_idx(event);
5799}
5800
5801static void calc_timer_values(struct perf_event *event,
5802                                u64 *now,
5803                                u64 *enabled,
5804                                u64 *running)
5805{
5806        u64 ctx_time;
5807
5808        *now = perf_clock();
5809        ctx_time = event->shadow_ctx_time + *now;
5810        __perf_update_times(event, ctx_time, enabled, running);
5811}
5812
5813static void perf_event_init_userpage(struct perf_event *event)
5814{
5815        struct perf_event_mmap_page *userpg;
5816        struct perf_buffer *rb;
5817
5818        rcu_read_lock();
5819        rb = rcu_dereference(event->rb);
5820        if (!rb)
5821                goto unlock;
5822
5823        userpg = rb->user_page;
5824
5825        /* Allow new userspace to detect that bit 0 is deprecated */
5826        userpg->cap_bit0_is_deprecated = 1;
5827        userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
5828        userpg->data_offset = PAGE_SIZE;
5829        userpg->data_size = perf_data_size(rb);
5830
5831unlock:
5832        rcu_read_unlock();
5833}
5834
5835void __weak arch_perf_update_userpage(
5836        struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
5837{
5838}
5839
5840/*
5841 * Callers need to ensure there can be no nesting of this function, otherwise
5842 * the seqlock logic goes bad. We can not serialize this because the arch
5843 * code calls this from NMI context.
5844 */
5845void perf_event_update_userpage(struct perf_event *event)
5846{
5847        struct perf_event_mmap_page *userpg;
5848        struct perf_buffer *rb;
5849        u64 enabled, running, now;
5850
5851        rcu_read_lock();
5852        rb = rcu_dereference(event->rb);
5853        if (!rb)
5854                goto unlock;
5855
5856        /*
5857         * compute total_time_enabled, total_time_running
5858         * based on snapshot values taken when the event
5859         * was last scheduled in.
5860         *
5861         * we cannot simply called update_context_time()
5862         * because of locking issue as we can be called in
5863         * NMI context
5864         */
5865        calc_timer_values(event, &now, &enabled, &running);
5866
5867        userpg = rb->user_page;
5868        /*
5869         * Disable preemption to guarantee consistent time stamps are stored to
5870         * the user page.
5871         */
5872        preempt_disable();
5873        ++userpg->lock;
5874        barrier();
5875        userpg->index = perf_event_index(event);
5876        userpg->offset = perf_event_count(event);
5877        if (userpg->index)
5878                userpg->offset -= local64_read(&event->hw.prev_count);
5879
5880        userpg->time_enabled = enabled +
5881                        atomic64_read(&event->child_total_time_enabled);
5882
5883        userpg->time_running = running +
5884                        atomic64_read(&event->child_total_time_running);
5885
5886        arch_perf_update_userpage(event, userpg, now);
5887
5888        barrier();
5889        ++userpg->lock;
5890        preempt_enable();
5891unlock:
5892        rcu_read_unlock();
5893}
5894EXPORT_SYMBOL_GPL(perf_event_update_userpage);
5895
5896static vm_fault_t perf_mmap_fault(struct vm_fault *vmf)
5897{
5898        struct perf_event *event = vmf->vma->vm_file->private_data;
5899        struct perf_buffer *rb;
5900        vm_fault_t ret = VM_FAULT_SIGBUS;
5901
5902        if (vmf->flags & FAULT_FLAG_MKWRITE) {
5903                if (vmf->pgoff == 0)
5904                        ret = 0;
5905                return ret;
5906        }
5907
5908        rcu_read_lock();
5909        rb = rcu_dereference(event->rb);
5910        if (!rb)
5911                goto unlock;
5912
5913        if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
5914                goto unlock;
5915
5916        vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
5917        if (!vmf->page)
5918                goto unlock;
5919
5920        get_page(vmf->page);
5921        vmf->page->mapping = vmf->vma->