1
2
3
4
5
6
7
8
9
10
11
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
16#include <linux/idr.h>
17#include <linux/file.h>
18#include <linux/poll.h>
19#include <linux/slab.h>
20#include <linux/hash.h>
21#include <linux/sysfs.h>
22#include <linux/dcache.h>
23#include <linux/percpu.h>
24#include <linux/ptrace.h>
25#include <linux/reboot.h>
26#include <linux/vmstat.h>
27#include <linux/device.h>
28#include <linux/vmalloc.h>
29#include <linux/hardirq.h>
30#include <linux/rculist.h>
31#include <linux/uaccess.h>
32#include <linux/syscalls.h>
33#include <linux/anon_inodes.h>
34#include <linux/kernel_stat.h>
35#include <linux/perf_event.h>
36#include <linux/ftrace_event.h>
37#include <linux/hw_breakpoint.h>
38
39#include <asm/irq_regs.h>
40
41enum event_type_t {
42 EVENT_FLEXIBLE = 0x1,
43 EVENT_PINNED = 0x2,
44 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
45};
46
47atomic_t perf_task_events __read_mostly;
48static atomic_t nr_mmap_events __read_mostly;
49static atomic_t nr_comm_events __read_mostly;
50static atomic_t nr_task_events __read_mostly;
51
52static LIST_HEAD(pmus);
53static DEFINE_MUTEX(pmus_lock);
54static struct srcu_struct pmus_srcu;
55
56
57
58
59
60
61
62
63int sysctl_perf_event_paranoid __read_mostly = 1;
64
65int sysctl_perf_event_mlock __read_mostly = 512;
66
67
68
69
70int sysctl_perf_event_sample_rate __read_mostly = 100000;
71
72static atomic64_t perf_event_id;
73
74static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
75 enum event_type_t event_type);
76
77static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
78 enum event_type_t event_type);
79
80void __weak perf_event_print_debug(void) { }
81
82extern __weak const char *perf_pmu_name(void)
83{
84 return "pmu";
85}
86
87static inline u64 perf_clock(void)
88{
89 return local_clock();
90}
91
92void perf_pmu_disable(struct pmu *pmu)
93{
94 int *count = this_cpu_ptr(pmu->pmu_disable_count);
95 if (!(*count)++)
96 pmu->pmu_disable(pmu);
97}
98
99void perf_pmu_enable(struct pmu *pmu)
100{
101 int *count = this_cpu_ptr(pmu->pmu_disable_count);
102 if (!--(*count))
103 pmu->pmu_enable(pmu);
104}
105
106static DEFINE_PER_CPU(struct list_head, rotation_list);
107
108
109
110
111
112
113static void perf_pmu_rotate_start(struct pmu *pmu)
114{
115 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
116 struct list_head *head = &__get_cpu_var(rotation_list);
117
118 WARN_ON(!irqs_disabled());
119
120 if (list_empty(&cpuctx->rotation_list))
121 list_add(&cpuctx->rotation_list, head);
122}
123
124static void get_ctx(struct perf_event_context *ctx)
125{
126 WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
127}
128
129static void free_ctx(struct rcu_head *head)
130{
131 struct perf_event_context *ctx;
132
133 ctx = container_of(head, struct perf_event_context, rcu_head);
134 kfree(ctx);
135}
136
137static void put_ctx(struct perf_event_context *ctx)
138{
139 if (atomic_dec_and_test(&ctx->refcount)) {
140 if (ctx->parent_ctx)
141 put_ctx(ctx->parent_ctx);
142 if (ctx->task)
143 put_task_struct(ctx->task);
144 call_rcu(&ctx->rcu_head, free_ctx);
145 }
146}
147
148static void unclone_ctx(struct perf_event_context *ctx)
149{
150 if (ctx->parent_ctx) {
151 put_ctx(ctx->parent_ctx);
152 ctx->parent_ctx = NULL;
153 }
154}
155
156static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
157{
158
159
160
161 if (event->parent)
162 event = event->parent;
163
164 return task_tgid_nr_ns(p, event->ns);
165}
166
167static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
168{
169
170
171
172 if (event->parent)
173 event = event->parent;
174
175 return task_pid_nr_ns(p, event->ns);
176}
177
178
179
180
181
182static u64 primary_event_id(struct perf_event *event)
183{
184 u64 id = event->id;
185
186 if (event->parent)
187 id = event->parent->id;
188
189 return id;
190}
191
192
193
194
195
196
197static struct perf_event_context *
198perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
199{
200 struct perf_event_context *ctx;
201
202 rcu_read_lock();
203retry:
204 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
205 if (ctx) {
206
207
208
209
210
211
212
213
214
215
216 raw_spin_lock_irqsave(&ctx->lock, *flags);
217 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
218 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
219 goto retry;
220 }
221
222 if (!atomic_inc_not_zero(&ctx->refcount)) {
223 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
224 ctx = NULL;
225 }
226 }
227 rcu_read_unlock();
228 return ctx;
229}
230
231
232
233
234
235
236static struct perf_event_context *
237perf_pin_task_context(struct task_struct *task, int ctxn)
238{
239 struct perf_event_context *ctx;
240 unsigned long flags;
241
242 ctx = perf_lock_task_context(task, ctxn, &flags);
243 if (ctx) {
244 ++ctx->pin_count;
245 raw_spin_unlock_irqrestore(&ctx->lock, flags);
246 }
247 return ctx;
248}
249
250static void perf_unpin_context(struct perf_event_context *ctx)
251{
252 unsigned long flags;
253
254 raw_spin_lock_irqsave(&ctx->lock, flags);
255 --ctx->pin_count;
256 raw_spin_unlock_irqrestore(&ctx->lock, flags);
257 put_ctx(ctx);
258}
259
260
261
262
263static void update_context_time(struct perf_event_context *ctx)
264{
265 u64 now = perf_clock();
266
267 ctx->time += now - ctx->timestamp;
268 ctx->timestamp = now;
269}
270
271static u64 perf_event_time(struct perf_event *event)
272{
273 struct perf_event_context *ctx = event->ctx;
274 return ctx ? ctx->time : 0;
275}
276
277
278
279
280static void update_event_times(struct perf_event *event)
281{
282 struct perf_event_context *ctx = event->ctx;
283 u64 run_end;
284
285 if (event->state < PERF_EVENT_STATE_INACTIVE ||
286 event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
287 return;
288
289 if (ctx->is_active)
290 run_end = perf_event_time(event);
291 else
292 run_end = event->tstamp_stopped;
293
294 event->total_time_enabled = run_end - event->tstamp_enabled;
295
296 if (event->state == PERF_EVENT_STATE_INACTIVE)
297 run_end = event->tstamp_stopped;
298 else
299 run_end = perf_event_time(event);
300
301 event->total_time_running = run_end - event->tstamp_running;
302}
303
304
305
306
307static void update_group_times(struct perf_event *leader)
308{
309 struct perf_event *event;
310
311 update_event_times(leader);
312 list_for_each_entry(event, &leader->sibling_list, group_entry)
313 update_event_times(event);
314}
315
316static struct list_head *
317ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
318{
319 if (event->attr.pinned)
320 return &ctx->pinned_groups;
321 else
322 return &ctx->flexible_groups;
323}
324
325
326
327
328
329static void
330list_add_event(struct perf_event *event, struct perf_event_context *ctx)
331{
332 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
333 event->attach_state |= PERF_ATTACH_CONTEXT;
334
335
336
337
338
339
340 if (event->group_leader == event) {
341 struct list_head *list;
342
343 if (is_software_event(event))
344 event->group_flags |= PERF_GROUP_SOFTWARE;
345
346 list = ctx_group_list(event, ctx);
347 list_add_tail(&event->group_entry, list);
348 }
349
350 list_add_rcu(&event->event_entry, &ctx->event_list);
351 if (!ctx->nr_events)
352 perf_pmu_rotate_start(ctx->pmu);
353 ctx->nr_events++;
354 if (event->attr.inherit_stat)
355 ctx->nr_stat++;
356}
357
358
359
360
361
362static void perf_event__read_size(struct perf_event *event)
363{
364 int entry = sizeof(u64);
365 int size = 0;
366 int nr = 1;
367
368 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
369 size += sizeof(u64);
370
371 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
372 size += sizeof(u64);
373
374 if (event->attr.read_format & PERF_FORMAT_ID)
375 entry += sizeof(u64);
376
377 if (event->attr.read_format & PERF_FORMAT_GROUP) {
378 nr += event->group_leader->nr_siblings;
379 size += sizeof(u64);
380 }
381
382 size += entry * nr;
383 event->read_size = size;
384}
385
386static void perf_event__header_size(struct perf_event *event)
387{
388 struct perf_sample_data *data;
389 u64 sample_type = event->attr.sample_type;
390 u16 size = 0;
391
392 perf_event__read_size(event);
393
394 if (sample_type & PERF_SAMPLE_IP)
395 size += sizeof(data->ip);
396
397 if (sample_type & PERF_SAMPLE_ADDR)
398 size += sizeof(data->addr);
399
400 if (sample_type & PERF_SAMPLE_PERIOD)
401 size += sizeof(data->period);
402
403 if (sample_type & PERF_SAMPLE_READ)
404 size += event->read_size;
405
406 event->header_size = size;
407}
408
409static void perf_event__id_header_size(struct perf_event *event)
410{
411 struct perf_sample_data *data;
412 u64 sample_type = event->attr.sample_type;
413 u16 size = 0;
414
415 if (sample_type & PERF_SAMPLE_TID)
416 size += sizeof(data->tid_entry);
417
418 if (sample_type & PERF_SAMPLE_TIME)
419 size += sizeof(data->time);
420
421 if (sample_type & PERF_SAMPLE_ID)
422 size += sizeof(data->id);
423
424 if (sample_type & PERF_SAMPLE_STREAM_ID)
425 size += sizeof(data->stream_id);
426
427 if (sample_type & PERF_SAMPLE_CPU)
428 size += sizeof(data->cpu_entry);
429
430 event->id_header_size = size;
431}
432
433static void perf_group_attach(struct perf_event *event)
434{
435 struct perf_event *group_leader = event->group_leader, *pos;
436
437
438
439
440 if (event->attach_state & PERF_ATTACH_GROUP)
441 return;
442
443 event->attach_state |= PERF_ATTACH_GROUP;
444
445 if (group_leader == event)
446 return;
447
448 if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
449 !is_software_event(event))
450 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
451
452 list_add_tail(&event->group_entry, &group_leader->sibling_list);
453 group_leader->nr_siblings++;
454
455 perf_event__header_size(group_leader);
456
457 list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
458 perf_event__header_size(pos);
459}
460
461
462
463
464
465static void
466list_del_event(struct perf_event *event, struct perf_event_context *ctx)
467{
468
469
470
471 if (!(event->attach_state & PERF_ATTACH_CONTEXT))
472 return;
473
474 event->attach_state &= ~PERF_ATTACH_CONTEXT;
475
476 ctx->nr_events--;
477 if (event->attr.inherit_stat)
478 ctx->nr_stat--;
479
480 list_del_rcu(&event->event_entry);
481
482 if (event->group_leader == event)
483 list_del_init(&event->group_entry);
484
485 update_group_times(event);
486
487
488
489
490
491
492
493
494 if (event->state > PERF_EVENT_STATE_OFF)
495 event->state = PERF_EVENT_STATE_OFF;
496}
497
498static void perf_group_detach(struct perf_event *event)
499{
500 struct perf_event *sibling, *tmp;
501 struct list_head *list = NULL;
502
503
504
505
506 if (!(event->attach_state & PERF_ATTACH_GROUP))
507 return;
508
509 event->attach_state &= ~PERF_ATTACH_GROUP;
510
511
512
513
514 if (event->group_leader != event) {
515 list_del_init(&event->group_entry);
516 event->group_leader->nr_siblings--;
517 goto out;
518 }
519
520 if (!list_empty(&event->group_entry))
521 list = &event->group_entry;
522
523
524
525
526
527
528 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
529 if (list)
530 list_move_tail(&sibling->group_entry, list);
531 sibling->group_leader = sibling;
532
533
534 sibling->group_flags = event->group_flags;
535 }
536
537out:
538 perf_event__header_size(event->group_leader);
539
540 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
541 perf_event__header_size(tmp);
542}
543
544static inline int
545event_filter_match(struct perf_event *event)
546{
547 return event->cpu == -1 || event->cpu == smp_processor_id();
548}
549
550static void
551event_sched_out(struct perf_event *event,
552 struct perf_cpu_context *cpuctx,
553 struct perf_event_context *ctx)
554{
555 u64 tstamp = perf_event_time(event);
556 u64 delta;
557
558
559
560
561
562
563 if (event->state == PERF_EVENT_STATE_INACTIVE
564 && !event_filter_match(event)) {
565 delta = ctx->time - event->tstamp_stopped;
566 event->tstamp_running += delta;
567 event->tstamp_stopped = tstamp;
568 }
569
570 if (event->state != PERF_EVENT_STATE_ACTIVE)
571 return;
572
573 event->state = PERF_EVENT_STATE_INACTIVE;
574 if (event->pending_disable) {
575 event->pending_disable = 0;
576 event->state = PERF_EVENT_STATE_OFF;
577 }
578 event->tstamp_stopped = tstamp;
579 event->pmu->del(event, 0);
580 event->oncpu = -1;
581
582 if (!is_software_event(event))
583 cpuctx->active_oncpu--;
584 ctx->nr_active--;
585 if (event->attr.exclusive || !cpuctx->active_oncpu)
586 cpuctx->exclusive = 0;
587}
588
589static void
590group_sched_out(struct perf_event *group_event,
591 struct perf_cpu_context *cpuctx,
592 struct perf_event_context *ctx)
593{
594 struct perf_event *event;
595 int state = group_event->state;
596
597 event_sched_out(group_event, cpuctx, ctx);
598
599
600
601
602 list_for_each_entry(event, &group_event->sibling_list, group_entry)
603 event_sched_out(event, cpuctx, ctx);
604
605 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
606 cpuctx->exclusive = 0;
607}
608
609static inline struct perf_cpu_context *
610__get_cpu_context(struct perf_event_context *ctx)
611{
612 return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
613}
614
615
616
617
618
619
620
621static void __perf_event_remove_from_context(void *info)
622{
623 struct perf_event *event = info;
624 struct perf_event_context *ctx = event->ctx;
625 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
626
627
628
629
630
631
632 if (ctx->task && cpuctx->task_ctx != ctx)
633 return;
634
635 raw_spin_lock(&ctx->lock);
636
637 event_sched_out(event, cpuctx, ctx);
638
639 list_del_event(event, ctx);
640
641 raw_spin_unlock(&ctx->lock);
642}
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660static void perf_event_remove_from_context(struct perf_event *event)
661{
662 struct perf_event_context *ctx = event->ctx;
663 struct task_struct *task = ctx->task;
664
665 if (!task) {
666
667
668
669
670 smp_call_function_single(event->cpu,
671 __perf_event_remove_from_context,
672 event, 1);
673 return;
674 }
675
676retry:
677 task_oncpu_function_call(task, __perf_event_remove_from_context,
678 event);
679
680 raw_spin_lock_irq(&ctx->lock);
681
682
683
684 if (ctx->nr_active && !list_empty(&event->group_entry)) {
685 raw_spin_unlock_irq(&ctx->lock);
686 goto retry;
687 }
688
689
690
691
692
693
694 if (!list_empty(&event->group_entry))
695 list_del_event(event, ctx);
696 raw_spin_unlock_irq(&ctx->lock);
697}
698
699
700
701
702static void __perf_event_disable(void *info)
703{
704 struct perf_event *event = info;
705 struct perf_event_context *ctx = event->ctx;
706 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
707
708
709
710
711
712 if (ctx->task && cpuctx->task_ctx != ctx)
713 return;
714
715 raw_spin_lock(&ctx->lock);
716
717
718
719
720
721 if (event->state >= PERF_EVENT_STATE_INACTIVE) {
722 update_context_time(ctx);
723 update_group_times(event);
724 if (event == event->group_leader)
725 group_sched_out(event, cpuctx, ctx);
726 else
727 event_sched_out(event, cpuctx, ctx);
728 event->state = PERF_EVENT_STATE_OFF;
729 }
730
731 raw_spin_unlock(&ctx->lock);
732}
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747void perf_event_disable(struct perf_event *event)
748{
749 struct perf_event_context *ctx = event->ctx;
750 struct task_struct *task = ctx->task;
751
752 if (!task) {
753
754
755
756 smp_call_function_single(event->cpu, __perf_event_disable,
757 event, 1);
758 return;
759 }
760
761retry:
762 task_oncpu_function_call(task, __perf_event_disable, event);
763
764 raw_spin_lock_irq(&ctx->lock);
765
766
767
768 if (event->state == PERF_EVENT_STATE_ACTIVE) {
769 raw_spin_unlock_irq(&ctx->lock);
770 goto retry;
771 }
772
773
774
775
776
777 if (event->state == PERF_EVENT_STATE_INACTIVE) {
778 update_group_times(event);
779 event->state = PERF_EVENT_STATE_OFF;
780 }
781
782 raw_spin_unlock_irq(&ctx->lock);
783}
784
785#define MAX_INTERRUPTS (~0ULL)
786
787static void perf_log_throttle(struct perf_event *event, int enable);
788
789static int
790event_sched_in(struct perf_event *event,
791 struct perf_cpu_context *cpuctx,
792 struct perf_event_context *ctx)
793{
794 u64 tstamp = perf_event_time(event);
795
796 if (event->state <= PERF_EVENT_STATE_OFF)
797 return 0;
798
799 event->state = PERF_EVENT_STATE_ACTIVE;
800 event->oncpu = smp_processor_id();
801
802
803
804
805
806
807 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
808 perf_log_throttle(event, 1);
809 event->hw.interrupts = 0;
810 }
811
812
813
814
815 smp_wmb();
816
817 if (event->pmu->add(event, PERF_EF_START)) {
818 event->state = PERF_EVENT_STATE_INACTIVE;
819 event->oncpu = -1;
820 return -EAGAIN;
821 }
822
823 event->tstamp_running += tstamp - event->tstamp_stopped;
824
825 event->shadow_ctx_time = tstamp - ctx->timestamp;
826
827 if (!is_software_event(event))
828 cpuctx->active_oncpu++;
829 ctx->nr_active++;
830
831 if (event->attr.exclusive)
832 cpuctx->exclusive = 1;
833
834 return 0;
835}
836
837static int
838group_sched_in(struct perf_event *group_event,
839 struct perf_cpu_context *cpuctx,
840 struct perf_event_context *ctx)
841{
842 struct perf_event *event, *partial_group = NULL;
843 struct pmu *pmu = group_event->pmu;
844 u64 now = ctx->time;
845 bool simulate = false;
846
847 if (group_event->state == PERF_EVENT_STATE_OFF)
848 return 0;
849
850 pmu->start_txn(pmu);
851
852 if (event_sched_in(group_event, cpuctx, ctx)) {
853 pmu->cancel_txn(pmu);
854 return -EAGAIN;
855 }
856
857
858
859
860 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
861 if (event_sched_in(event, cpuctx, ctx)) {
862 partial_group = event;
863 goto group_error;
864 }
865 }
866
867 if (!pmu->commit_txn(pmu))
868 return 0;
869
870group_error:
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
886 if (event == partial_group)
887 simulate = true;
888
889 if (simulate) {
890 event->tstamp_running += now - event->tstamp_stopped;
891 event->tstamp_stopped = now;
892 } else {
893 event_sched_out(event, cpuctx, ctx);
894 }
895 }
896 event_sched_out(group_event, cpuctx, ctx);
897
898 pmu->cancel_txn(pmu);
899
900 return -EAGAIN;
901}
902
903
904
905
906static int group_can_go_on(struct perf_event *event,
907 struct perf_cpu_context *cpuctx,
908 int can_add_hw)
909{
910
911
912
913 if (event->group_flags & PERF_GROUP_SOFTWARE)
914 return 1;
915
916
917
918
919 if (cpuctx->exclusive)
920 return 0;
921
922
923
924
925 if (event->attr.exclusive && cpuctx->active_oncpu)
926 return 0;
927
928
929
930
931 return can_add_hw;
932}
933
934static void add_event_to_ctx(struct perf_event *event,
935 struct perf_event_context *ctx)
936{
937 u64 tstamp = perf_event_time(event);
938
939 list_add_event(event, ctx);
940 perf_group_attach(event);
941 event->tstamp_enabled = tstamp;
942 event->tstamp_running = tstamp;
943 event->tstamp_stopped = tstamp;
944}
945
946
947
948
949
950
951static void __perf_install_in_context(void *info)
952{
953 struct perf_event *event = info;
954 struct perf_event_context *ctx = event->ctx;
955 struct perf_event *leader = event->group_leader;
956 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
957 int err;
958
959
960
961
962
963
964
965
966 if (ctx->task && cpuctx->task_ctx != ctx) {
967 if (cpuctx->task_ctx || ctx->task != current)
968 return;
969 cpuctx->task_ctx = ctx;
970 }
971
972 raw_spin_lock(&ctx->lock);
973 ctx->is_active = 1;
974 update_context_time(ctx);
975
976 add_event_to_ctx(event, ctx);
977
978 if (!event_filter_match(event))
979 goto unlock;
980
981
982
983
984
985 if (event->state != PERF_EVENT_STATE_INACTIVE ||
986 (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
987 goto unlock;
988
989
990
991
992
993
994 if (!group_can_go_on(event, cpuctx, 1))
995 err = -EEXIST;
996 else
997 err = event_sched_in(event, cpuctx, ctx);
998
999 if (err) {
1000
1001
1002
1003
1004
1005 if (leader != event)
1006 group_sched_out(leader, cpuctx, ctx);
1007 if (leader->attr.pinned) {
1008 update_group_times(leader);
1009 leader->state = PERF_EVENT_STATE_ERROR;
1010 }
1011 }
1012
1013unlock:
1014 raw_spin_unlock(&ctx->lock);
1015}
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029static void
1030perf_install_in_context(struct perf_event_context *ctx,
1031 struct perf_event *event,
1032 int cpu)
1033{
1034 struct task_struct *task = ctx->task;
1035
1036 event->ctx = ctx;
1037
1038 if (!task) {
1039
1040
1041
1042
1043 smp_call_function_single(cpu, __perf_install_in_context,
1044 event, 1);
1045 return;
1046 }
1047
1048retry:
1049 task_oncpu_function_call(task, __perf_install_in_context,
1050 event);
1051
1052 raw_spin_lock_irq(&ctx->lock);
1053
1054
1055
1056 if (ctx->is_active && list_empty(&event->group_entry)) {
1057 raw_spin_unlock_irq(&ctx->lock);
1058 goto retry;
1059 }
1060
1061
1062
1063
1064
1065
1066 if (list_empty(&event->group_entry))
1067 add_event_to_ctx(event, ctx);
1068 raw_spin_unlock_irq(&ctx->lock);
1069}
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079static void __perf_event_mark_enabled(struct perf_event *event,
1080 struct perf_event_context *ctx)
1081{
1082 struct perf_event *sub;
1083 u64 tstamp = perf_event_time(event);
1084
1085 event->state = PERF_EVENT_STATE_INACTIVE;
1086 event->tstamp_enabled = tstamp - event->total_time_enabled;
1087 list_for_each_entry(sub, &event->sibling_list, group_entry) {
1088 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1089 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1090 }
1091}
1092
1093
1094
1095
1096static void __perf_event_enable(void *info)
1097{
1098 struct perf_event *event = info;
1099 struct perf_event_context *ctx = event->ctx;
1100 struct perf_event *leader = event->group_leader;
1101 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1102 int err;
1103
1104
1105
1106
1107
1108 if (ctx->task && cpuctx->task_ctx != ctx) {
1109 if (cpuctx->task_ctx || ctx->task != current)
1110 return;
1111 cpuctx->task_ctx = ctx;
1112 }
1113
1114 raw_spin_lock(&ctx->lock);
1115 ctx->is_active = 1;
1116 update_context_time(ctx);
1117
1118 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1119 goto unlock;
1120 __perf_event_mark_enabled(event, ctx);
1121
1122 if (!event_filter_match(event))
1123 goto unlock;
1124
1125
1126
1127
1128
1129 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1130 goto unlock;
1131
1132 if (!group_can_go_on(event, cpuctx, 1)) {
1133 err = -EEXIST;
1134 } else {
1135 if (event == leader)
1136 err = group_sched_in(event, cpuctx, ctx);
1137 else
1138 err = event_sched_in(event, cpuctx, ctx);
1139 }
1140
1141 if (err) {
1142
1143
1144
1145
1146 if (leader != event)
1147 group_sched_out(leader, cpuctx, ctx);
1148 if (leader->attr.pinned) {
1149 update_group_times(leader);
1150 leader->state = PERF_EVENT_STATE_ERROR;
1151 }
1152 }
1153
1154unlock:
1155 raw_spin_unlock(&ctx->lock);
1156}
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167void perf_event_enable(struct perf_event *event)
1168{
1169 struct perf_event_context *ctx = event->ctx;
1170 struct task_struct *task = ctx->task;
1171
1172 if (!task) {
1173
1174
1175
1176 smp_call_function_single(event->cpu, __perf_event_enable,
1177 event, 1);
1178 return;
1179 }
1180
1181 raw_spin_lock_irq(&ctx->lock);
1182 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1183 goto out;
1184
1185
1186
1187
1188
1189
1190
1191
1192 if (event->state == PERF_EVENT_STATE_ERROR)
1193 event->state = PERF_EVENT_STATE_OFF;
1194
1195retry:
1196 raw_spin_unlock_irq(&ctx->lock);
1197 task_oncpu_function_call(task, __perf_event_enable, event);
1198
1199 raw_spin_lock_irq(&ctx->lock);
1200
1201
1202
1203
1204
1205 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1206 goto retry;
1207
1208
1209
1210
1211
1212 if (event->state == PERF_EVENT_STATE_OFF)
1213 __perf_event_mark_enabled(event, ctx);
1214
1215out:
1216 raw_spin_unlock_irq(&ctx->lock);
1217}
1218
1219static int perf_event_refresh(struct perf_event *event, int refresh)
1220{
1221
1222
1223
1224 if (event->attr.inherit || !is_sampling_event(event))
1225 return -EINVAL;
1226
1227 atomic_add(refresh, &event->event_limit);
1228 perf_event_enable(event);
1229
1230 return 0;
1231}
1232
1233static void ctx_sched_out(struct perf_event_context *ctx,
1234 struct perf_cpu_context *cpuctx,
1235 enum event_type_t event_type)
1236{
1237 struct perf_event *event;
1238
1239 raw_spin_lock(&ctx->lock);
1240 perf_pmu_disable(ctx->pmu);
1241 ctx->is_active = 0;
1242 if (likely(!ctx->nr_events))
1243 goto out;
1244 update_context_time(ctx);
1245
1246 if (!ctx->nr_active)
1247 goto out;
1248
1249 if (event_type & EVENT_PINNED) {
1250 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1251 group_sched_out(event, cpuctx, ctx);
1252 }
1253
1254 if (event_type & EVENT_FLEXIBLE) {
1255 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1256 group_sched_out(event, cpuctx, ctx);
1257 }
1258out:
1259 perf_pmu_enable(ctx->pmu);
1260 raw_spin_unlock(&ctx->lock);
1261}
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274static int context_equiv(struct perf_event_context *ctx1,
1275 struct perf_event_context *ctx2)
1276{
1277 return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1278 && ctx1->parent_gen == ctx2->parent_gen
1279 && !ctx1->pin_count && !ctx2->pin_count;
1280}
1281
1282static void __perf_event_sync_stat(struct perf_event *event,
1283 struct perf_event *next_event)
1284{
1285 u64 value;
1286
1287 if (!event->attr.inherit_stat)
1288 return;
1289
1290
1291
1292
1293
1294
1295
1296
1297 switch (event->state) {
1298 case PERF_EVENT_STATE_ACTIVE:
1299 event->pmu->read(event);
1300
1301
1302 case PERF_EVENT_STATE_INACTIVE:
1303 update_event_times(event);
1304 break;
1305
1306 default:
1307 break;
1308 }
1309
1310
1311
1312
1313
1314 value = local64_read(&next_event->count);
1315 value = local64_xchg(&event->count, value);
1316 local64_set(&next_event->count, value);
1317
1318 swap(event->total_time_enabled, next_event->total_time_enabled);
1319 swap(event->total_time_running, next_event->total_time_running);
1320
1321
1322
1323
1324 perf_event_update_userpage(event);
1325 perf_event_update_userpage(next_event);
1326}
1327
1328#define list_next_entry(pos, member) \
1329 list_entry(pos->member.next, typeof(*pos), member)
1330
1331static void perf_event_sync_stat(struct perf_event_context *ctx,
1332 struct perf_event_context *next_ctx)
1333{
1334 struct perf_event *event, *next_event;
1335
1336 if (!ctx->nr_stat)
1337 return;
1338
1339 update_context_time(ctx);
1340
1341 event = list_first_entry(&ctx->event_list,
1342 struct perf_event, event_entry);
1343
1344 next_event = list_first_entry(&next_ctx->event_list,
1345 struct perf_event, event_entry);
1346
1347 while (&event->event_entry != &ctx->event_list &&
1348 &next_event->event_entry != &next_ctx->event_list) {
1349
1350 __perf_event_sync_stat(event, next_event);
1351
1352 event = list_next_entry(event, event_entry);
1353 next_event = list_next_entry(next_event, event_entry);
1354 }
1355}
1356
1357void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1358 struct task_struct *next)
1359{
1360 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1361 struct perf_event_context *next_ctx;
1362 struct perf_event_context *parent;
1363 struct perf_cpu_context *cpuctx;
1364 int do_switch = 1;
1365
1366 if (likely(!ctx))
1367 return;
1368
1369 cpuctx = __get_cpu_context(ctx);
1370 if (!cpuctx->task_ctx)
1371 return;
1372
1373 rcu_read_lock();
1374 parent = rcu_dereference(ctx->parent_ctx);
1375 next_ctx = next->perf_event_ctxp[ctxn];
1376 if (parent && next_ctx &&
1377 rcu_dereference(next_ctx->parent_ctx) == parent) {
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387 raw_spin_lock(&ctx->lock);
1388 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1389 if (context_equiv(ctx, next_ctx)) {
1390
1391
1392
1393
1394 task->perf_event_ctxp[ctxn] = next_ctx;
1395 next->perf_event_ctxp[ctxn] = ctx;
1396 ctx->task = next;
1397 next_ctx->task = task;
1398 do_switch = 0;
1399
1400 perf_event_sync_stat(ctx, next_ctx);
1401 }
1402 raw_spin_unlock(&next_ctx->lock);
1403 raw_spin_unlock(&ctx->lock);
1404 }
1405 rcu_read_unlock();
1406
1407 if (do_switch) {
1408 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1409 cpuctx->task_ctx = NULL;
1410 }
1411}
1412
1413#define for_each_task_context_nr(ctxn) \
1414 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427void __perf_event_task_sched_out(struct task_struct *task,
1428 struct task_struct *next)
1429{
1430 int ctxn;
1431
1432 for_each_task_context_nr(ctxn)
1433 perf_event_context_sched_out(task, ctxn, next);
1434}
1435
1436static void task_ctx_sched_out(struct perf_event_context *ctx,
1437 enum event_type_t event_type)
1438{
1439 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1440
1441 if (!cpuctx->task_ctx)
1442 return;
1443
1444 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
1445 return;
1446
1447 ctx_sched_out(ctx, cpuctx, event_type);
1448 cpuctx->task_ctx = NULL;
1449}
1450
1451
1452
1453
1454static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
1455 enum event_type_t event_type)
1456{
1457 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1458}
1459
1460static void
1461ctx_pinned_sched_in(struct perf_event_context *ctx,
1462 struct perf_cpu_context *cpuctx)
1463{
1464 struct perf_event *event;
1465
1466 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1467 if (event->state <= PERF_EVENT_STATE_OFF)
1468 continue;
1469 if (!event_filter_match(event))
1470 continue;
1471
1472 if (group_can_go_on(event, cpuctx, 1))
1473 group_sched_in(event, cpuctx, ctx);
1474
1475
1476
1477
1478
1479 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1480 update_group_times(event);
1481 event->state = PERF_EVENT_STATE_ERROR;
1482 }
1483 }
1484}
1485
1486static void
1487ctx_flexible_sched_in(struct perf_event_context *ctx,
1488 struct perf_cpu_context *cpuctx)
1489{
1490 struct perf_event *event;
1491 int can_add_hw = 1;
1492
1493 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1494
1495 if (event->state <= PERF_EVENT_STATE_OFF)
1496 continue;
1497
1498
1499
1500
1501 if (!event_filter_match(event))
1502 continue;
1503
1504 if (group_can_go_on(event, cpuctx, can_add_hw)) {
1505 if (group_sched_in(event, cpuctx, ctx))
1506 can_add_hw = 0;
1507 }
1508 }
1509}
1510
1511static void
1512ctx_sched_in(struct perf_event_context *ctx,
1513 struct perf_cpu_context *cpuctx,
1514 enum event_type_t event_type)
1515{
1516 raw_spin_lock(&ctx->lock);
1517 ctx->is_active = 1;
1518 if (likely(!ctx->nr_events))
1519 goto out;
1520
1521 ctx->timestamp = perf_clock();
1522
1523
1524
1525
1526
1527 if (event_type & EVENT_PINNED)
1528 ctx_pinned_sched_in(ctx, cpuctx);
1529
1530
1531 if (event_type & EVENT_FLEXIBLE)
1532 ctx_flexible_sched_in(ctx, cpuctx);
1533
1534out:
1535 raw_spin_unlock(&ctx->lock);
1536}
1537
1538static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
1539 enum event_type_t event_type)
1540{
1541 struct perf_event_context *ctx = &cpuctx->ctx;
1542
1543 ctx_sched_in(ctx, cpuctx, event_type);
1544}
1545
1546static void task_ctx_sched_in(struct perf_event_context *ctx,
1547 enum event_type_t event_type)
1548{
1549 struct perf_cpu_context *cpuctx;
1550
1551 cpuctx = __get_cpu_context(ctx);
1552 if (cpuctx->task_ctx == ctx)
1553 return;
1554
1555 ctx_sched_in(ctx, cpuctx, event_type);
1556 cpuctx->task_ctx = ctx;
1557}
1558
1559void perf_event_context_sched_in(struct perf_event_context *ctx)
1560{
1561 struct perf_cpu_context *cpuctx;
1562
1563 cpuctx = __get_cpu_context(ctx);
1564 if (cpuctx->task_ctx == ctx)
1565 return;
1566
1567 perf_pmu_disable(ctx->pmu);
1568
1569
1570
1571
1572
1573 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1574
1575 ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
1576 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1577 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
1578
1579 cpuctx->task_ctx = ctx;
1580
1581
1582
1583
1584
1585 perf_pmu_rotate_start(ctx->pmu);
1586 perf_pmu_enable(ctx->pmu);
1587}
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600void __perf_event_task_sched_in(struct task_struct *task)
1601{
1602 struct perf_event_context *ctx;
1603 int ctxn;
1604
1605 for_each_task_context_nr(ctxn) {
1606 ctx = task->perf_event_ctxp[ctxn];
1607 if (likely(!ctx))
1608 continue;
1609
1610 perf_event_context_sched_in(ctx);
1611 }
1612}
1613
1614static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
1615{
1616 u64 frequency = event->attr.sample_freq;
1617 u64 sec = NSEC_PER_SEC;
1618 u64 divisor, dividend;
1619
1620 int count_fls, nsec_fls, frequency_fls, sec_fls;
1621
1622 count_fls = fls64(count);
1623 nsec_fls = fls64(nsec);
1624 frequency_fls = fls64(frequency);
1625 sec_fls = 30;
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641#define REDUCE_FLS(a, b) \
1642do { \
1643 if (a##_fls > b##_fls) { \
1644 a >>= 1; \
1645 a##_fls--; \
1646 } else { \
1647 b >>= 1; \
1648 b##_fls--; \
1649 } \
1650} while (0)
1651
1652
1653
1654
1655
1656 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
1657 REDUCE_FLS(nsec, frequency);
1658 REDUCE_FLS(sec, count);
1659 }
1660
1661 if (count_fls + sec_fls > 64) {
1662 divisor = nsec * frequency;
1663
1664 while (count_fls + sec_fls > 64) {
1665 REDUCE_FLS(count, sec);
1666 divisor >>= 1;
1667 }
1668
1669 dividend = count * sec;
1670 } else {
1671 dividend = count * sec;
1672
1673 while (nsec_fls + frequency_fls > 64) {
1674 REDUCE_FLS(nsec, frequency);
1675 dividend >>= 1;
1676 }
1677
1678 divisor = nsec * frequency;
1679 }
1680
1681 if (!divisor)
1682 return dividend;
1683
1684 return div64_u64(dividend, divisor);
1685}
1686
1687static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1688{
1689 struct hw_perf_event *hwc = &event->hw;
1690 s64 period, sample_period;
1691 s64 delta;
1692
1693 period = perf_calculate_period(event, nsec, count);
1694
1695 delta = (s64)(period - hwc->sample_period);
1696 delta = (delta + 7) / 8;
1697
1698 sample_period = hwc->sample_period + delta;
1699
1700 if (!sample_period)
1701 sample_period = 1;
1702
1703 hwc->sample_period = sample_period;
1704
1705 if (local64_read(&hwc->period_left) > 8*sample_period) {
1706 event->pmu->stop(event, PERF_EF_UPDATE);
1707 local64_set(&hwc->period_left, 0);
1708 event->pmu->start(event, PERF_EF_RELOAD);
1709 }
1710}
1711
1712static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
1713{
1714 struct perf_event *event;
1715 struct hw_perf_event *hwc;
1716 u64 interrupts, now;
1717 s64 delta;
1718
1719 raw_spin_lock(&ctx->lock);
1720 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1721 if (event->state != PERF_EVENT_STATE_ACTIVE)
1722 continue;
1723
1724 if (!event_filter_match(event))
1725 continue;
1726
1727 hwc = &event->hw;
1728
1729 interrupts = hwc->interrupts;
1730 hwc->interrupts = 0;
1731
1732
1733
1734
1735 if (interrupts == MAX_INTERRUPTS) {
1736 perf_log_throttle(event, 1);
1737 event->pmu->start(event, 0);
1738 }
1739
1740 if (!event->attr.freq || !event->attr.sample_freq)
1741 continue;
1742
1743 event->pmu->read(event);
1744 now = local64_read(&event->count);
1745 delta = now - hwc->freq_count_stamp;
1746 hwc->freq_count_stamp = now;
1747
1748 if (delta > 0)
1749 perf_adjust_period(event, period, delta);
1750 }
1751 raw_spin_unlock(&ctx->lock);
1752}
1753
1754
1755
1756
1757static void rotate_ctx(struct perf_event_context *ctx)
1758{
1759 raw_spin_lock(&ctx->lock);
1760
1761
1762
1763
1764
1765 if (!ctx->rotate_disable)
1766 list_rotate_left(&ctx->flexible_groups);
1767
1768 raw_spin_unlock(&ctx->lock);
1769}
1770
1771
1772
1773
1774
1775
1776static void perf_rotate_context(struct perf_cpu_context *cpuctx)
1777{
1778 u64 interval = (u64)cpuctx->jiffies_interval * TICK_NSEC;
1779 struct perf_event_context *ctx = NULL;
1780 int rotate = 0, remove = 1;
1781
1782 if (cpuctx->ctx.nr_events) {
1783 remove = 0;
1784 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
1785 rotate = 1;
1786 }
1787
1788 ctx = cpuctx->task_ctx;
1789 if (ctx && ctx->nr_events) {
1790 remove = 0;
1791 if (ctx->nr_events != ctx->nr_active)
1792 rotate = 1;
1793 }
1794
1795 perf_pmu_disable(cpuctx->ctx.pmu);
1796 perf_ctx_adjust_freq(&cpuctx->ctx, interval);
1797 if (ctx)
1798 perf_ctx_adjust_freq(ctx, interval);
1799
1800 if (!rotate)
1801 goto done;
1802
1803 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1804 if (ctx)
1805 task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
1806
1807 rotate_ctx(&cpuctx->ctx);
1808 if (ctx)
1809 rotate_ctx(ctx);
1810
1811 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1812 if (ctx)
1813 task_ctx_sched_in(ctx, EVENT_FLEXIBLE);
1814
1815done:
1816 if (remove)
1817 list_del_init(&cpuctx->rotation_list);
1818
1819 perf_pmu_enable(cpuctx->ctx.pmu);
1820}
1821
1822void perf_event_task_tick(void)
1823{
1824 struct list_head *head = &__get_cpu_var(rotation_list);
1825 struct perf_cpu_context *cpuctx, *tmp;
1826
1827 WARN_ON(!irqs_disabled());
1828
1829 list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
1830 if (cpuctx->jiffies_interval == 1 ||
1831 !(jiffies % cpuctx->jiffies_interval))
1832 perf_rotate_context(cpuctx);
1833 }
1834}
1835
1836static int event_enable_on_exec(struct perf_event *event,
1837 struct perf_event_context *ctx)
1838{
1839 if (!event->attr.enable_on_exec)
1840 return 0;
1841
1842 event->attr.enable_on_exec = 0;
1843 if (event->state >= PERF_EVENT_STATE_INACTIVE)
1844 return 0;
1845
1846 __perf_event_mark_enabled(event, ctx);
1847
1848 return 1;
1849}
1850
1851
1852
1853
1854
1855static void perf_event_enable_on_exec(struct perf_event_context *ctx)
1856{
1857 struct perf_event *event;
1858 unsigned long flags;
1859 int enabled = 0;
1860 int ret;
1861
1862 local_irq_save(flags);
1863 if (!ctx || !ctx->nr_events)
1864 goto out;
1865
1866 task_ctx_sched_out(ctx, EVENT_ALL);
1867
1868 raw_spin_lock(&ctx->lock);
1869
1870 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
1871 ret = event_enable_on_exec(event, ctx);
1872 if (ret)
1873 enabled = 1;
1874 }
1875
1876 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
1877 ret = event_enable_on_exec(event, ctx);
1878 if (ret)
1879 enabled = 1;
1880 }
1881
1882
1883
1884
1885 if (enabled)
1886 unclone_ctx(ctx);
1887
1888 raw_spin_unlock(&ctx->lock);
1889
1890 perf_event_context_sched_in(ctx);
1891out:
1892 local_irq_restore(flags);
1893}
1894
1895
1896
1897
1898static void __perf_event_read(void *info)
1899{
1900 struct perf_event *event = info;
1901 struct perf_event_context *ctx = event->ctx;
1902 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1903
1904
1905
1906
1907
1908
1909
1910
1911 if (ctx->task && cpuctx->task_ctx != ctx)
1912 return;
1913
1914 raw_spin_lock(&ctx->lock);
1915 if (ctx->is_active)
1916 update_context_time(ctx);
1917 update_event_times(event);
1918 if (event->state == PERF_EVENT_STATE_ACTIVE)
1919 event->pmu->read(event);
1920 raw_spin_unlock(&ctx->lock);
1921}
1922
1923static inline u64 perf_event_count(struct perf_event *event)
1924{
1925 return local64_read(&event->count) + atomic64_read(&event->child_count);
1926}
1927
1928static u64 perf_event_read(struct perf_event *event)
1929{
1930
1931
1932
1933
1934 if (event->state == PERF_EVENT_STATE_ACTIVE) {
1935 smp_call_function_single(event->oncpu,
1936 __perf_event_read, event, 1);
1937 } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
1938 struct perf_event_context *ctx = event->ctx;
1939 unsigned long flags;
1940
1941 raw_spin_lock_irqsave(&ctx->lock, flags);
1942
1943
1944
1945
1946
1947 if (ctx->is_active)
1948 update_context_time(ctx);
1949 update_event_times(event);
1950 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1951 }
1952
1953 return perf_event_count(event);
1954}
1955
1956
1957
1958
1959
1960struct callchain_cpus_entries {
1961 struct rcu_head rcu_head;
1962 struct perf_callchain_entry *cpu_entries[0];
1963};
1964
1965static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
1966static atomic_t nr_callchain_events;
1967static DEFINE_MUTEX(callchain_mutex);
1968struct callchain_cpus_entries *callchain_cpus_entries;
1969
1970
1971__weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
1972 struct pt_regs *regs)
1973{
1974}
1975
1976__weak void perf_callchain_user(struct perf_callchain_entry *entry,
1977 struct pt_regs *regs)
1978{
1979}
1980
1981static void release_callchain_buffers_rcu(struct rcu_head *head)
1982{
1983 struct callchain_cpus_entries *entries;
1984 int cpu;
1985
1986 entries = container_of(head, struct callchain_cpus_entries, rcu_head);
1987
1988 for_each_possible_cpu(cpu)
1989 kfree(entries->cpu_entries[cpu]);
1990
1991 kfree(entries);
1992}
1993
1994static void release_callchain_buffers(void)
1995{
1996 struct callchain_cpus_entries *entries;
1997
1998 entries = callchain_cpus_entries;
1999 rcu_assign_pointer(callchain_cpus_entries, NULL);
2000 call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
2001}
2002
2003static int alloc_callchain_buffers(void)
2004{
2005 int cpu;
2006 int size;
2007 struct callchain_cpus_entries *entries;
2008
2009
2010
2011
2012
2013
2014 size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
2015
2016 entries = kzalloc(size, GFP_KERNEL);
2017 if (!entries)
2018 return -ENOMEM;
2019
2020 size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
2021
2022 for_each_possible_cpu(cpu) {
2023 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
2024 cpu_to_node(cpu));
2025 if (!entries->cpu_entries[cpu])
2026 goto fail;
2027 }
2028
2029 rcu_assign_pointer(callchain_cpus_entries, entries);
2030
2031 return 0;
2032
2033fail:
2034 for_each_possible_cpu(cpu)
2035 kfree(entries->cpu_entries[cpu]);
2036 kfree(entries);
2037
2038 return -ENOMEM;
2039}
2040
2041static int get_callchain_buffers(void)
2042{
2043 int err = 0;
2044 int count;
2045
2046 mutex_lock(&callchain_mutex);
2047
2048 count = atomic_inc_return(&nr_callchain_events);
2049 if (WARN_ON_ONCE(count < 1)) {
2050 err = -EINVAL;
2051 goto exit;
2052 }
2053
2054 if (count > 1) {
2055
2056 if (!callchain_cpus_entries)
2057 err = -ENOMEM;
2058 goto exit;
2059 }
2060
2061 err = alloc_callchain_buffers();
2062 if (err)
2063 release_callchain_buffers();
2064exit:
2065 mutex_unlock(&callchain_mutex);
2066
2067 return err;
2068}
2069
2070static void put_callchain_buffers(void)
2071{
2072 if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
2073 release_callchain_buffers();
2074 mutex_unlock(&callchain_mutex);
2075 }
2076}
2077
2078static int get_recursion_context(int *recursion)
2079{
2080 int rctx;
2081
2082 if (in_nmi())
2083 rctx = 3;
2084 else if (in_irq())
2085 rctx = 2;
2086 else if (in_softirq())
2087 rctx = 1;
2088 else
2089 rctx = 0;
2090
2091 if (recursion[rctx])
2092 return -1;
2093
2094 recursion[rctx]++;
2095 barrier();
2096
2097 return rctx;
2098}
2099
2100static inline void put_recursion_context(int *recursion, int rctx)
2101{
2102 barrier();
2103 recursion[rctx]--;
2104}
2105
2106static struct perf_callchain_entry *get_callchain_entry(int *rctx)
2107{
2108 int cpu;
2109 struct callchain_cpus_entries *entries;
2110
2111 *rctx = get_recursion_context(__get_cpu_var(callchain_recursion));
2112 if (*rctx == -1)
2113 return NULL;
2114
2115 entries = rcu_dereference(callchain_cpus_entries);
2116 if (!entries)
2117 return NULL;
2118
2119 cpu = smp_processor_id();
2120
2121 return &entries->cpu_entries[cpu][*rctx];
2122}
2123
2124static void
2125put_callchain_entry(int rctx)
2126{
2127 put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
2128}
2129
2130static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2131{
2132 int rctx;
2133 struct perf_callchain_entry *entry;
2134
2135
2136 entry = get_callchain_entry(&rctx);
2137 if (rctx == -1)
2138 return NULL;
2139
2140 if (!entry)
2141 goto exit_put;
2142
2143 entry->nr = 0;
2144
2145 if (!user_mode(regs)) {
2146 perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
2147 perf_callchain_kernel(entry, regs);
2148 if (current->mm)
2149 regs = task_pt_regs(current);
2150 else
2151 regs = NULL;
2152 }
2153
2154 if (regs) {
2155 perf_callchain_store(entry, PERF_CONTEXT_USER);
2156 perf_callchain_user(entry, regs);
2157 }
2158
2159exit_put:
2160 put_callchain_entry(rctx);
2161
2162 return entry;
2163}
2164
2165
2166
2167
2168static void __perf_event_init_context(struct perf_event_context *ctx)
2169{
2170 raw_spin_lock_init(&ctx->lock);
2171 mutex_init(&ctx->mutex);
2172 INIT_LIST_HEAD(&ctx->pinned_groups);
2173 INIT_LIST_HEAD(&ctx->flexible_groups);
2174 INIT_LIST_HEAD(&ctx->event_list);
2175 atomic_set(&ctx->refcount, 1);
2176}
2177
2178static struct perf_event_context *
2179alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2180{
2181 struct perf_event_context *ctx;
2182
2183 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2184 if (!ctx)
2185 return NULL;
2186
2187 __perf_event_init_context(ctx);
2188 if (task) {
2189 ctx->task = task;
2190 get_task_struct(task);
2191 }
2192 ctx->pmu = pmu;
2193
2194 return ctx;
2195}
2196
2197static struct task_struct *
2198find_lively_task_by_vpid(pid_t vpid)
2199{
2200 struct task_struct *task;
2201 int err;
2202
2203 rcu_read_lock();
2204 if (!vpid)
2205 task = current;
2206 else
2207 task = find_task_by_vpid(vpid);
2208 if (task)
2209 get_task_struct(task);
2210 rcu_read_unlock();
2211
2212 if (!task)
2213 return ERR_PTR(-ESRCH);
2214
2215
2216 err = -EACCES;
2217 if (!ptrace_may_access(task, PTRACE_MODE_READ))
2218 goto errout;
2219
2220 return task;
2221errout:
2222 put_task_struct(task);
2223 return ERR_PTR(err);
2224
2225}
2226
2227static struct perf_event_context *
2228find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2229{
2230 struct perf_event_context *ctx;
2231 struct perf_cpu_context *cpuctx;
2232 unsigned long flags;
2233 int ctxn, err;
2234
2235 if (!task) {
2236
2237 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2238 return ERR_PTR(-EACCES);
2239
2240
2241
2242
2243
2244
2245 if (!cpu_online(cpu))
2246 return ERR_PTR(-ENODEV);
2247
2248 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2249 ctx = &cpuctx->ctx;
2250 get_ctx(ctx);
2251
2252 return ctx;
2253 }
2254
2255 err = -EINVAL;
2256 ctxn = pmu->task_ctx_nr;
2257 if (ctxn < 0)
2258 goto errout;
2259
2260retry:
2261 ctx = perf_lock_task_context(task, ctxn, &flags);
2262 if (ctx) {
2263 unclone_ctx(ctx);
2264 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2265 }
2266
2267 if (!ctx) {
2268 ctx = alloc_perf_context(pmu, task);
2269 err = -ENOMEM;
2270 if (!ctx)
2271 goto errout;
2272
2273 get_ctx(ctx);
2274
2275 err = 0;
2276 mutex_lock(&task->perf_event_mutex);
2277
2278
2279
2280
2281 if (task->flags & PF_EXITING)
2282 err = -ESRCH;
2283 else if (task->perf_event_ctxp[ctxn])
2284 err = -EAGAIN;
2285 else
2286 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2287 mutex_unlock(&task->perf_event_mutex);
2288
2289 if (unlikely(err)) {
2290 put_task_struct(task);
2291 kfree(ctx);
2292
2293 if (err == -EAGAIN)
2294 goto retry;
2295 goto errout;
2296 }
2297 }
2298
2299 return ctx;
2300
2301errout:
2302 return ERR_PTR(err);
2303}
2304
2305static void perf_event_free_filter(struct perf_event *event);
2306
2307static void free_event_rcu(struct rcu_head *head)
2308{
2309 struct perf_event *event;
2310
2311 event = container_of(head, struct perf_event, rcu_head);
2312 if (event->ns)
2313 put_pid_ns(event->ns);
2314 perf_event_free_filter(event);
2315 kfree(event);
2316}
2317
2318static void perf_buffer_put(struct perf_buffer *buffer);
2319
2320static void free_event(struct perf_event *event)
2321{
2322 irq_work_sync(&event->pending);
2323
2324 if (!event->parent) {
2325 if (event->attach_state & PERF_ATTACH_TASK)
2326 jump_label_dec(&perf_task_events);
2327 if (event->attr.mmap || event->attr.mmap_data)
2328 atomic_dec(&nr_mmap_events);
2329 if (event->attr.comm)
2330 atomic_dec(&nr_comm_events);
2331 if (event->attr.task)
2332 atomic_dec(&nr_task_events);
2333 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2334 put_callchain_buffers();
2335 }
2336
2337 if (event->buffer) {
2338 perf_buffer_put(event->buffer);
2339 event->buffer = NULL;
2340 }
2341
2342 if (event->destroy)
2343 event->destroy(event);
2344
2345 if (event->ctx)
2346 put_ctx(event->ctx);
2347
2348 call_rcu(&event->rcu_head, free_event_rcu);
2349}
2350
2351int perf_event_release_kernel(struct perf_event *event)
2352{
2353 struct perf_event_context *ctx = event->ctx;
2354
2355
2356
2357
2358
2359 perf_event_disable(event);
2360
2361 WARN_ON_ONCE(ctx->parent_ctx);
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2375 raw_spin_lock_irq(&ctx->lock);
2376 perf_group_detach(event);
2377 list_del_event(event, ctx);
2378 raw_spin_unlock_irq(&ctx->lock);
2379 mutex_unlock(&ctx->mutex);
2380
2381 free_event(event);
2382
2383 return 0;
2384}
2385EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2386
2387
2388
2389
2390static int perf_release(struct inode *inode, struct file *file)
2391{
2392 struct perf_event *event = file->private_data;
2393 struct task_struct *owner;
2394
2395 file->private_data = NULL;
2396
2397 rcu_read_lock();
2398 owner = ACCESS_ONCE(event->owner);
2399
2400
2401
2402
2403
2404
2405 smp_read_barrier_depends();
2406 if (owner) {
2407
2408
2409
2410
2411
2412 get_task_struct(owner);
2413 }
2414 rcu_read_unlock();
2415
2416 if (owner) {
2417 mutex_lock(&owner->perf_event_mutex);
2418
2419
2420
2421
2422
2423
2424 if (event->owner)
2425 list_del_init(&event->owner_entry);
2426 mutex_unlock(&owner->perf_event_mutex);
2427 put_task_struct(owner);
2428 }
2429
2430 return perf_event_release_kernel(event);
2431}
2432
2433u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2434{
2435 struct perf_event *child;
2436 u64 total = 0;
2437
2438 *enabled = 0;
2439 *running = 0;
2440
2441 mutex_lock(&event->child_mutex);
2442 total += perf_event_read(event);
2443 *enabled += event->total_time_enabled +
2444 atomic64_read(&event->child_total_time_enabled);
2445 *running += event->total_time_running +
2446 atomic64_read(&event->child_total_time_running);
2447
2448 list_for_each_entry(child, &event->child_list, child_list) {
2449 total += perf_event_read(child);
2450 *enabled += child->total_time_enabled;
2451 *running += child->total_time_running;
2452 }
2453 mutex_unlock(&event->child_mutex);
2454
2455 return total;
2456}
2457EXPORT_SYMBOL_GPL(perf_event_read_value);
2458
2459static int perf_event_read_group(struct perf_event *event,
2460 u64 read_format, char __user *buf)
2461{
2462 struct perf_event *leader = event->group_leader, *sub;
2463 int n = 0, size = 0, ret = -EFAULT;
2464 struct perf_event_context *ctx = leader->ctx;
2465 u64 values[5];
2466 u64 count, enabled, running;
2467
2468 mutex_lock(&ctx->mutex);
2469 count = perf_event_read_value(leader, &enabled, &running);
2470
2471 values[n++] = 1 + leader->nr_siblings;
2472 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2473 values[n++] = enabled;
2474 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2475 values[n++] = running;
2476 values[n++] = count;
2477 if (read_format & PERF_FORMAT_ID)
2478 values[n++] = primary_event_id(leader);
2479
2480 size = n * sizeof(u64);
2481
2482 if (copy_to_user(buf, values, size))
2483 goto unlock;
2484
2485 ret = size;
2486
2487 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2488 n = 0;
2489
2490 values[n++] = perf_event_read_value(sub, &enabled, &running);
2491 if (read_format & PERF_FORMAT_ID)
2492 values[n++] = primary_event_id(sub);
2493
2494 size = n * sizeof(u64);
2495
2496 if (copy_to_user(buf + ret, values, size)) {
2497 ret = -EFAULT;
2498 goto unlock;
2499 }
2500
2501 ret += size;
2502 }
2503unlock:
2504 mutex_unlock(&ctx->mutex);
2505
2506 return ret;
2507}
2508
2509static int perf_event_read_one(struct perf_event *event,
2510 u64 read_format, char __user *buf)
2511{
2512 u64 enabled, running;
2513 u64 values[4];
2514 int n = 0;
2515
2516 values[n++] = perf_event_read_value(event, &enabled, &running);
2517 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2518 values[n++] = enabled;
2519 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2520 values[n++] = running;
2521 if (read_format & PERF_FORMAT_ID)
2522 values[n++] = primary_event_id(event);
2523
2524 if (copy_to_user(buf, values, n * sizeof(u64)))
2525 return -EFAULT;
2526
2527 return n * sizeof(u64);
2528}
2529
2530
2531
2532
2533static ssize_t
2534perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2535{
2536 u64 read_format = event->attr.read_format;
2537 int ret;
2538
2539
2540
2541
2542
2543
2544 if (event->state == PERF_EVENT_STATE_ERROR)
2545 return 0;
2546
2547 if (count < event->read_size)
2548 return -ENOSPC;
2549
2550 WARN_ON_ONCE(event->ctx->parent_ctx);
2551 if (read_format & PERF_FORMAT_GROUP)
2552 ret = perf_event_read_group(event, read_format, buf);
2553 else
2554 ret = perf_event_read_one(event, read_format, buf);
2555
2556 return ret;
2557}
2558
2559static ssize_t
2560perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
2561{
2562 struct perf_event *event = file->private_data;
2563
2564 return perf_read_hw(event, buf, count);
2565}
2566
2567static unsigned int perf_poll(struct file *file, poll_table *wait)
2568{
2569 struct perf_event *event = file->private_data;
2570 struct perf_buffer *buffer;
2571 unsigned int events = POLL_HUP;
2572
2573 rcu_read_lock();
2574 buffer = rcu_dereference(event->buffer);
2575 if (buffer)
2576 events = atomic_xchg(&buffer->poll, 0);
2577 rcu_read_unlock();
2578
2579 poll_wait(file, &event->waitq, wait);
2580
2581 return events;
2582}
2583
2584static void perf_event_reset(struct perf_event *event)
2585{
2586 (void)perf_event_read(event);
2587 local64_set(&event->count, 0);
2588 perf_event_update_userpage(event);
2589}
2590
2591
2592
2593
2594
2595
2596
2597static void perf_event_for_each_child(struct perf_event *event,
2598 void (*func)(struct perf_event *))
2599{
2600 struct perf_event *child;
2601
2602 WARN_ON_ONCE(event->ctx->parent_ctx);
2603 mutex_lock(&event->child_mutex);
2604 func(event);
2605 list_for_each_entry(child, &event->child_list, child_list)
2606 func(child);
2607 mutex_unlock(&event->child_mutex);
2608}
2609
2610static void perf_event_for_each(struct perf_event *event,
2611 void (*func)(struct perf_event *))
2612{
2613 struct perf_event_context *ctx = event->ctx;
2614 struct perf_event *sibling;
2615
2616 WARN_ON_ONCE(ctx->parent_ctx);
2617 mutex_lock(&ctx->mutex);
2618 event = event->group_leader;
2619
2620 perf_event_for_each_child(event, func);
2621 func(event);
2622 list_for_each_entry(sibling, &event->sibling_list, group_entry)
2623 perf_event_for_each_child(event, func);
2624 mutex_unlock(&ctx->mutex);
2625}
2626
2627static int perf_event_period(struct perf_event *event, u64 __user *arg)
2628{
2629 struct perf_event_context *ctx = event->ctx;
2630 int ret = 0;
2631 u64 value;
2632
2633 if (!is_sampling_event(event))
2634 return -EINVAL;
2635
2636 if (copy_from_user(&value, arg, sizeof(value)))
2637 return -EFAULT;
2638
2639 if (!value)
2640 return -EINVAL;
2641
2642 raw_spin_lock_irq(&ctx->lock);
2643 if (event->attr.freq) {
2644 if (value > sysctl_perf_event_sample_rate) {
2645 ret = -EINVAL;
2646 goto unlock;
2647 }
2648
2649 event->attr.sample_freq = value;
2650 } else {
2651 event->attr.sample_period = value;
2652 event->hw.sample_period = value;
2653 }
2654unlock:
2655 raw_spin_unlock_irq(&ctx->lock);
2656
2657 return ret;
2658}
2659
2660static const struct file_operations perf_fops;
2661
2662static struct perf_event *perf_fget_light(int fd, int *fput_needed)
2663{
2664 struct file *file;
2665
2666 file = fget_light(fd, fput_needed);
2667 if (!file)
2668 return ERR_PTR(-EBADF);
2669
2670 if (file->f_op != &perf_fops) {
2671 fput_light(file, *fput_needed);
2672 *fput_needed = 0;
2673 return ERR_PTR(-EBADF);
2674 }
2675
2676 return file->private_data;
2677}
2678
2679static int perf_event_set_output(struct perf_event *event,
2680 struct perf_event *output_event);
2681static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2682
2683static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2684{
2685 struct perf_event *event = file->private_data;
2686 void (*func)(struct perf_event *);
2687 u32 flags = arg;
2688
2689 switch (cmd) {
2690 case PERF_EVENT_IOC_ENABLE:
2691 func = perf_event_enable;
2692 break;
2693 case PERF_EVENT_IOC_DISABLE:
2694 func = perf_event_disable;
2695 break;
2696 case PERF_EVENT_IOC_RESET:
2697 func = perf_event_reset;
2698 break;
2699
2700 case PERF_EVENT_IOC_REFRESH:
2701 return perf_event_refresh(event, arg);
2702
2703 case PERF_EVENT_IOC_PERIOD:
2704 return perf_event_period(event, (u64 __user *)arg);
2705
2706 case PERF_EVENT_IOC_SET_OUTPUT:
2707 {
2708 struct perf_event *output_event = NULL;
2709 int fput_needed = 0;
2710 int ret;
2711
2712 if (arg != -1) {
2713 output_event = perf_fget_light(arg, &fput_needed);
2714 if (IS_ERR(output_event))
2715 return PTR_ERR(output_event);
2716 }
2717
2718 ret = perf_event_set_output(event, output_event);
2719 if (output_event)
2720 fput_light(output_event->filp, fput_needed);
2721
2722 return ret;
2723 }
2724
2725 case PERF_EVENT_IOC_SET_FILTER:
2726 return perf_event_set_filter(event, (void __user *)arg);
2727
2728 default:
2729 return -ENOTTY;
2730 }
2731
2732 if (flags & PERF_IOC_FLAG_GROUP)
2733 perf_event_for_each(event, func);
2734 else
2735 perf_event_for_each_child(event, func);
2736
2737 return 0;
2738}
2739
2740int perf_event_task_enable(void)
2741{
2742 struct perf_event *event;
2743
2744 mutex_lock(¤t->perf_event_mutex);
2745 list_for_each_entry(event, ¤t->perf_event_list, owner_entry)
2746 perf_event_for_each_child(event, perf_event_enable);
2747 mutex_unlock(¤t->perf_event_mutex);
2748
2749 return 0;
2750}
2751
2752int perf_event_task_disable(void)
2753{
2754 struct perf_event *event;
2755
2756 mutex_lock(¤t->perf_event_mutex);
2757 list_for_each_entry(event, ¤t->perf_event_list, owner_entry)
2758 perf_event_for_each_child(event, perf_event_disable);
2759 mutex_unlock(¤t->perf_event_mutex);
2760
2761 return 0;
2762}
2763
2764#ifndef PERF_EVENT_INDEX_OFFSET
2765# define PERF_EVENT_INDEX_OFFSET 0
2766#endif
2767
2768static int perf_event_index(struct perf_event *event)
2769{
2770 if (event->hw.state & PERF_HES_STOPPED)
2771 return 0;
2772
2773 if (event->state != PERF_EVENT_STATE_ACTIVE)
2774 return 0;
2775
2776 return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2777}
2778
2779
2780
2781
2782
2783
2784void perf_event_update_userpage(struct perf_event *event)
2785{
2786 struct perf_event_mmap_page *userpg;
2787 struct perf_buffer *buffer;
2788
2789 rcu_read_lock();
2790 buffer = rcu_dereference(event->buffer);
2791 if (!buffer)
2792 goto unlock;
2793
2794 userpg = buffer->user_page;
2795
2796
2797
2798
2799
2800 preempt_disable();
2801 ++userpg->lock;
2802 barrier();
2803 userpg->index = perf_event_index(event);
2804 userpg->offset = perf_event_count(event);
2805 if (event->state == PERF_EVENT_STATE_ACTIVE)
2806 userpg->offset -= local64_read(&event->hw.prev_count);
2807
2808 userpg->time_enabled = event->total_time_enabled +
2809 atomic64_read(&event->child_total_time_enabled);
2810
2811 userpg->time_running = event->total_time_running +
2812 atomic64_read(&event->child_total_time_running);
2813
2814 barrier();
2815 ++userpg->lock;
2816 preempt_enable();
2817unlock:
2818 rcu_read_unlock();
2819}
2820
2821static unsigned long perf_data_size(struct perf_buffer *buffer);
2822
2823static void
2824perf_buffer_init(struct perf_buffer *buffer, long watermark, int flags)
2825{
2826 long max_size = perf_data_size(buffer);
2827
2828 if (watermark)
2829 buffer->watermark = min(max_size, watermark);
2830
2831 if (!buffer->watermark)
2832 buffer->watermark = max_size / 2;
2833
2834 if (flags & PERF_BUFFER_WRITABLE)
2835 buffer->writable = 1;
2836
2837 atomic_set(&buffer->refcount, 1);
2838}
2839
2840#ifndef CONFIG_PERF_USE_VMALLOC
2841
2842
2843
2844
2845
2846static struct page *
2847perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2848{
2849 if (pgoff > buffer->nr_pages)
2850 return NULL;
2851
2852 if (pgoff == 0)
2853 return virt_to_page(buffer->user_page);
2854
2855 return virt_to_page(buffer->data_pages[pgoff - 1]);
2856}
2857
2858static void *perf_mmap_alloc_page(int cpu)
2859{
2860 struct page *page;
2861 int node;
2862
2863 node = (cpu == -1) ? cpu : cpu_to_node(cpu);
2864 page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2865 if (!page)
2866 return NULL;
2867
2868 return page_address(page);
2869}
2870
2871static struct perf_buffer *
2872perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2873{
2874 struct perf_buffer *buffer;
2875 unsigned long size;
2876 int i;
2877
2878 size = sizeof(struct perf_buffer);
2879 size += nr_pages * sizeof(void *);
2880
2881 buffer = kzalloc(size, GFP_KERNEL);
2882 if (!buffer)
2883 goto fail;
2884
2885 buffer->user_page = perf_mmap_alloc_page(cpu);
2886 if (!buffer->user_page)
2887 goto fail_user_page;
2888
2889 for (i = 0; i < nr_pages; i++) {
2890 buffer->data_pages[i] = perf_mmap_alloc_page(cpu);
2891 if (!buffer->data_pages[i])
2892 goto fail_data_pages;
2893 }
2894
2895 buffer->nr_pages = nr_pages;
2896
2897 perf_buffer_init(buffer, watermark, flags);
2898
2899 return buffer;
2900
2901fail_data_pages:
2902 for (i--; i >= 0; i--)
2903 free_page((unsigned long)buffer->data_pages[i]);
2904
2905 free_page((unsigned long)buffer->user_page);
2906
2907fail_user_page:
2908 kfree(buffer);
2909
2910fail:
2911 return NULL;
2912}
2913
2914static void perf_mmap_free_page(unsigned long addr)
2915{
2916 struct page *page = virt_to_page((void *)addr);
2917
2918 page->mapping = NULL;
2919 __free_page(page);
2920}
2921
2922static void perf_buffer_free(struct perf_buffer *buffer)
2923{
2924 int i;
2925
2926 perf_mmap_free_page((unsigned long)buffer->user_page);
2927 for (i = 0; i < buffer->nr_pages; i++)
2928 perf_mmap_free_page((unsigned long)buffer->data_pages[i]);
2929 kfree(buffer);
2930}
2931
2932static inline int page_order(struct perf_buffer *buffer)
2933{
2934 return 0;
2935}
2936
2937#else
2938
2939
2940
2941
2942
2943
2944
2945static inline int page_order(struct perf_buffer *buffer)
2946{
2947 return buffer->page_order;
2948}
2949
2950static struct page *
2951perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2952{
2953 if (pgoff > (1UL << page_order(buffer)))
2954 return NULL;
2955
2956 return vmalloc_to_page((void *)buffer->user_page + pgoff * PAGE_SIZE);
2957}
2958
2959static void perf_mmap_unmark_page(void *addr)
2960{
2961 struct page *page = vmalloc_to_page(addr);
2962
2963 page->mapping = NULL;
2964}
2965
2966static void perf_buffer_free_work(struct work_struct *work)
2967{
2968 struct perf_buffer *buffer;
2969 void *base;
2970 int i, nr;
2971
2972 buffer = container_of(work, struct perf_buffer, work);
2973 nr = 1 << page_order(buffer);
2974
2975 base = buffer->user_page;
2976 for (i = 0; i < nr + 1; i++)
2977 perf_mmap_unmark_page(base + (i * PAGE_SIZE));
2978
2979 vfree(base);
2980 kfree(buffer);
2981}
2982
2983static void perf_buffer_free(struct perf_buffer *buffer)
2984{
2985 schedule_work(&buffer->work);
2986}
2987
2988static struct perf_buffer *
2989perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2990{
2991 struct perf_buffer *buffer;
2992 unsigned long size;
2993 void *all_buf;
2994
2995 size = sizeof(struct perf_buffer);
2996 size += sizeof(void *);
2997
2998 buffer = kzalloc(size, GFP_KERNEL);
2999 if (!buffer)
3000 goto fail;
3001
3002 INIT_WORK(&buffer->work, perf_buffer_free_work);
3003
3004 all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
3005 if (!all_buf)
3006 goto fail_all_buf;
3007
3008 buffer->user_page = all_buf;
3009 buffer->data_pages[0] = all_buf + PAGE_SIZE;
3010 buffer->page_order = ilog2(nr_pages);
3011 buffer->nr_pages = 1;
3012
3013 perf_buffer_init(buffer, watermark, flags);
3014
3015 return buffer;
3016
3017fail_all_buf:
3018 kfree(buffer);
3019
3020fail:
3021 return NULL;
3022}
3023
3024#endif
3025
3026static unsigned long perf_data_size(struct perf_buffer *buffer)
3027{
3028 return buffer->nr_pages << (PAGE_SHIFT + page_order(buffer));
3029}
3030
3031static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3032{
3033 struct perf_event *event = vma->vm_file->private_data;
3034 struct perf_buffer *buffer;
3035 int ret = VM_FAULT_SIGBUS;
3036
3037 if (vmf->flags & FAULT_FLAG_MKWRITE) {
3038 if (vmf->pgoff == 0)
3039 ret = 0;
3040 return ret;
3041 }
3042
3043 rcu_read_lock();
3044 buffer = rcu_dereference(event->buffer);
3045 if (!buffer)
3046 goto unlock;
3047
3048 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3049 goto unlock;
3050
3051 vmf->page = perf_mmap_to_page(buffer, vmf->pgoff);
3052 if (!vmf->page)
3053 goto unlock;
3054
3055 get_page(vmf->page);
3056 vmf->page->mapping = vma->vm_file->f_mapping;
3057 vmf->page->index = vmf->pgoff;
3058
3059 ret = 0;
3060unlock:
3061 rcu_read_unlock();
3062
3063 return ret;
3064}
3065
3066static void perf_buffer_free_rcu(struct rcu_head *rcu_head)
3067{
3068 struct perf_buffer *buffer;
3069
3070 buffer = container_of(rcu_head, struct perf_buffer, rcu_head);
3071 perf_buffer_free(buffer);
3072}
3073
3074static struct perf_buffer *perf_buffer_get(struct perf_event *event)
3075{
3076 struct perf_buffer *buffer;
3077
3078 rcu_read_lock();
3079 buffer = rcu_dereference(event->buffer);
3080 if (buffer) {
3081 if (!atomic_inc_not_zero(&buffer->refcount))
3082 buffer = NULL;
3083 }
3084 rcu_read_unlock();
3085
3086 return buffer;
3087}
3088
3089static void perf_buffer_put(struct perf_buffer *buffer)
3090{
3091 if (!atomic_dec_and_test(&buffer->refcount))
3092 return;
3093
3094 call_rcu(&buffer->rcu_head, perf_buffer_free_rcu);
3095}
3096
3097static void perf_mmap_open(struct vm_area_struct *vma)
3098{
3099 struct perf_event *event = vma->vm_file->private_data;
3100
3101 atomic_inc(&event->mmap_count);
3102}
3103
3104static void perf_mmap_close(struct vm_area_struct *vma)
3105{
3106 struct perf_event *event = vma->vm_file->private_data;
3107
3108 if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3109 unsigned long size = perf_data_size(event->buffer);
3110 struct user_struct *user = event->mmap_user;
3111 struct perf_buffer *buffer = event->buffer;
3112
3113 atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3114 vma->vm_mm->locked_vm -= event->mmap_locked;
3115 rcu_assign_pointer(event->buffer, NULL);
3116 mutex_unlock(&event->mmap_mutex);
3117
3118 perf_buffer_put(buffer);
3119 free_uid(user);
3120 }
3121}
3122
3123static const struct vm_operations_struct perf_mmap_vmops = {
3124 .open = perf_mmap_open,
3125 .close = perf_mmap_close,
3126 .fault = perf_mmap_fault,
3127 .page_mkwrite = perf_mmap_fault,
3128};
3129
3130static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3131{
3132 struct perf_event *event = file->private_data;
3133 unsigned long user_locked, user_lock_limit;
3134 struct user_struct *user = current_user();
3135 unsigned long locked, lock_limit;
3136 struct perf_buffer *buffer;
3137 unsigned long vma_size;
3138 unsigned long nr_pages;
3139 long user_extra, extra;
3140 int ret = 0, flags = 0;
3141
3142
3143
3144
3145
3146
3147 if (event->cpu == -1 && event->attr.inherit)
3148 return -EINVAL;
3149
3150 if (!(vma->vm_flags & VM_SHARED))
3151 return -EINVAL;
3152
3153 vma_size = vma->vm_end - vma->vm_start;
3154 nr_pages = (vma_size / PAGE_SIZE) - 1;
3155
3156
3157
3158
3159
3160 if (nr_pages != 0 && !is_power_of_2(nr_pages))
3161 return -EINVAL;
3162
3163 if (vma_size != PAGE_SIZE * (1 + nr_pages))
3164 return -EINVAL;
3165
3166 if (vma->vm_pgoff != 0)
3167 return -EINVAL;
3168
3169 WARN_ON_ONCE(event->ctx->parent_ctx);
3170 mutex_lock(&event->mmap_mutex);
3171 if (event->buffer) {
3172 if (event->buffer->nr_pages == nr_pages)
3173 atomic_inc(&event->buffer->refcount);
3174 else
3175 ret = -EINVAL;
3176 goto unlock;
3177 }
3178
3179 user_extra = nr_pages + 1;
3180 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3181
3182
3183
3184
3185 user_lock_limit *= num_online_cpus();
3186
3187 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3188
3189 extra = 0;
3190 if (user_locked > user_lock_limit)
3191 extra = user_locked - user_lock_limit;
3192
3193 lock_limit = rlimit(RLIMIT_MEMLOCK);
3194 lock_limit >>= PAGE_SHIFT;
3195 locked = vma->vm_mm->locked_vm + extra;
3196
3197 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3198 !capable(CAP_IPC_LOCK)) {
3199 ret = -EPERM;
3200 goto unlock;
3201 }
3202
3203 WARN_ON(event->buffer);
3204
3205 if (vma->vm_flags & VM_WRITE)
3206 flags |= PERF_BUFFER_WRITABLE;
3207
3208 buffer = perf_buffer_alloc(nr_pages, event->attr.wakeup_watermark,
3209 event->cpu, flags);
3210 if (!buffer) {
3211 ret = -ENOMEM;
3212 goto unlock;
3213 }
3214 rcu_assign_pointer(event->buffer, buffer);
3215
3216 atomic_long_add(user_extra, &user->locked_vm);
3217 event->mmap_locked = extra;
3218 event->mmap_user = get_current_user();
3219 vma->vm_mm->locked_vm += event->mmap_locked;
3220
3221unlock:
3222 if (!ret)
3223 atomic_inc(&event->mmap_count);
3224 mutex_unlock(&event->mmap_mutex);
3225
3226 vma->vm_flags |= VM_RESERVED;
3227 vma->vm_ops = &perf_mmap_vmops;
3228
3229 return ret;
3230}
3231
3232static int perf_fasync(int fd, struct file *filp, int on)
3233{
3234 struct inode *inode = filp->f_path.dentry->d_inode;
3235 struct perf_event *event = filp->private_data;
3236 int retval;
3237
3238 mutex_lock(&inode->i_mutex);
3239 retval = fasync_helper(fd, filp, on, &event->fasync);
3240 mutex_unlock(&inode->i_mutex);
3241
3242 if (retval < 0)
3243 return retval;
3244
3245 return 0;
3246}
3247
3248static const struct file_operations perf_fops = {
3249 .llseek = no_llseek,
3250 .release = perf_release,
3251 .read = perf_read,
3252 .poll = perf_poll,
3253 .unlocked_ioctl = perf_ioctl,
3254 .compat_ioctl = perf_ioctl,
3255 .mmap = perf_mmap,
3256 .fasync = perf_fasync,
3257};
3258
3259
3260
3261
3262
3263
3264
3265
3266void perf_event_wakeup(struct perf_event *event)
3267{
3268 wake_up_all(&event->waitq);
3269
3270 if (event->pending_kill) {
3271 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3272 event->pending_kill = 0;
3273 }
3274}
3275
3276static void perf_pending_event(struct irq_work *entry)
3277{
3278 struct perf_event *event = container_of(entry,
3279 struct perf_event, pending);
3280
3281 if (event->pending_disable) {
3282 event->pending_disable = 0;
3283 __perf_event_disable(event);
3284 }
3285
3286 if (event->pending_wakeup) {
3287 event->pending_wakeup = 0;
3288 perf_event_wakeup(event);
3289 }
3290}
3291
3292
3293
3294
3295
3296
3297struct perf_guest_info_callbacks *perf_guest_cbs;
3298
3299int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3300{
3301 perf_guest_cbs = cbs;
3302 return 0;
3303}
3304EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
3305
3306int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
3307{
3308 perf_guest_cbs = NULL;
3309 return 0;
3310}
3311EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
3312
3313
3314
3315
3316static bool perf_output_space(struct perf_buffer *buffer, unsigned long tail,
3317 unsigned long offset, unsigned long head)
3318{
3319 unsigned long mask;
3320
3321 if (!buffer->writable)
3322 return true;
3323
3324 mask = perf_data_size(buffer) - 1;
3325
3326 offset = (offset - tail) & mask;
3327 head = (head - tail) & mask;
3328
3329 if ((int)(head - offset) < 0)
3330 return false;
3331
3332 return true;
3333}
3334
3335static void perf_output_wakeup(struct perf_output_handle *handle)
3336{
3337 atomic_set(&handle->buffer->poll, POLL_IN);
3338
3339 if (handle->nmi) {
3340 handle->event->pending_wakeup = 1;
3341 irq_work_queue(&handle->event->pending);
3342 } else
3343 perf_event_wakeup(handle->event);
3344}
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354static void perf_output_get_handle(struct perf_output_handle *handle)
3355{
3356 struct perf_buffer *buffer = handle->buffer;
3357
3358 preempt_disable();
3359 local_inc(&buffer->nest);
3360 handle->wakeup = local_read(&buffer->wakeup);
3361}
3362
3363static void perf_output_put_handle(struct perf_output_handle *handle)
3364{
3365 struct perf_buffer *buffer = handle->buffer;
3366 unsigned long head;
3367
3368again:
3369 head = local_read(&buffer->head);
3370
3371
3372
3373
3374
3375 if (!local_dec_and_test(&buffer->nest))
3376 goto out;
3377
3378
3379
3380
3381
3382
3383 buffer->user_page->data_head = head;
3384
3385
3386
3387
3388
3389 if (unlikely(head != local_read(&buffer->head))) {
3390 local_inc(&buffer->nest);
3391 goto again;
3392 }
3393
3394 if (handle->wakeup != local_read(&buffer->wakeup))
3395 perf_output_wakeup(handle);
3396
3397out:
3398 preempt_enable();
3399}
3400
3401__always_inline void perf_output_copy(struct perf_output_handle *handle,
3402 const void *buf, unsigned int len)
3403{
3404 do {
3405 unsigned long size = min_t(unsigned long, handle->size, len);
3406
3407 memcpy(handle->addr, buf, size);
3408
3409 len -= size;
3410 handle->addr += size;
3411 buf += size;
3412 handle->size -= size;
3413 if (!handle->size) {
3414 struct perf_buffer *buffer = handle->buffer;
3415
3416 handle->page++;
3417 handle->page &= buffer->nr_pages - 1;
3418 handle->addr = buffer->data_pages[handle->page];
3419 handle->size = PAGE_SIZE << page_order(buffer);
3420 }
3421 } while (len);
3422}
3423
3424static void __perf_event_header__init_id(struct perf_event_header *header,
3425 struct perf_sample_data *data,
3426 struct perf_event *event)
3427{
3428 u64 sample_type = event->attr.sample_type;
3429
3430 data->type = sample_type;
3431 header->size += event->id_header_size;
3432
3433 if (sample_type & PERF_SAMPLE_TID) {
3434
3435 data->tid_entry.pid = perf_event_pid(event, current);
3436 data->tid_entry.tid = perf_event_tid(event, current);
3437 }
3438
3439 if (sample_type & PERF_SAMPLE_TIME)
3440 data->time = perf_clock();
3441
3442 if (sample_type & PERF_SAMPLE_ID)
3443 data->id = primary_event_id(event);
3444
3445 if (sample_type & PERF_SAMPLE_STREAM_ID)
3446 data->stream_id = event->id;
3447
3448 if (sample_type & PERF_SAMPLE_CPU) {
3449 data->cpu_entry.cpu = raw_smp_processor_id();
3450 data->cpu_entry.reserved = 0;
3451 }
3452}
3453
3454static void perf_event_header__init_id(struct perf_event_header *header,
3455 struct perf_sample_data *data,
3456 struct perf_event *event)
3457{
3458 if (event->attr.sample_id_all)
3459 __perf_event_header__init_id(header, data, event);
3460}
3461
3462static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3463 struct perf_sample_data *data)
3464{
3465 u64 sample_type = data->type;
3466
3467 if (sample_type & PERF_SAMPLE_TID)
3468 perf_output_put(handle, data->tid_entry);
3469
3470 if (sample_type & PERF_SAMPLE_TIME)
3471 perf_output_put(handle, data->time);
3472
3473 if (sample_type & PERF_SAMPLE_ID)
3474 perf_output_put(handle, data->id);
3475
3476 if (sample_type & PERF_SAMPLE_STREAM_ID)
3477 perf_output_put(handle, data->stream_id);
3478
3479 if (sample_type & PERF_SAMPLE_CPU)
3480 perf_output_put(handle, data->cpu_entry);
3481}
3482
3483static void perf_event__output_id_sample(struct perf_event *event,
3484 struct perf_output_handle *handle,
3485 struct perf_sample_data *sample)
3486{
3487 if (event->attr.sample_id_all)
3488 __perf_event__output_id_sample(handle, sample);
3489}
3490
3491int perf_output_begin(struct perf_output_handle *handle,
3492 struct perf_event *event, unsigned int size,
3493 int nmi, int sample)
3494{
3495 struct perf_buffer *buffer;
3496 unsigned long tail, offset, head;
3497 int have_lost;
3498 struct perf_sample_data sample_data;
3499 struct {
3500 struct perf_event_header header;
3501 u64 id;
3502 u64 lost;
3503 } lost_event;
3504
3505 rcu_read_lock();
3506
3507
3508
3509 if (event->parent)
3510 event = event->parent;
3511
3512 buffer = rcu_dereference(event->buffer);
3513 if (!buffer)
3514 goto out;
3515
3516 handle->buffer = buffer;
3517 handle->event = event;
3518 handle->nmi = nmi;
3519 handle->sample = sample;
3520
3521 if (!buffer->nr_pages)
3522 goto out;
3523
3524 have_lost = local_read(&buffer->lost);
3525 if (have_lost) {
3526 lost_event.header.size = sizeof(lost_event);
3527 perf_event_header__init_id(&lost_event.header, &sample_data,
3528 event);
3529 size += lost_event.header.size;
3530 }
3531
3532 perf_output_get_handle(handle);
3533
3534 do {
3535
3536
3537
3538
3539
3540 tail = ACCESS_ONCE(buffer->user_page->data_tail);
3541 smp_rmb();
3542 offset = head = local_read(&buffer->head);
3543 head += size;
3544 if (unlikely(!perf_output_space(buffer, tail, offset, head)))
3545 goto fail;
3546 } while (local_cmpxchg(&buffer->head, offset, head) != offset);
3547
3548 if (head - local_read(&buffer->wakeup) > buffer->watermark)
3549 local_add(buffer->watermark, &buffer->wakeup);
3550
3551 handle->page = offset >> (PAGE_SHIFT + page_order(buffer));
3552 handle->page &= buffer->nr_pages - 1;
3553 handle->size = offset & ((PAGE_SIZE << page_order(buffer)) - 1);
3554 handle->addr = buffer->data_pages[handle->page];
3555 handle->addr += handle->size;
3556 handle->size = (PAGE_SIZE << page_order(buffer)) - handle->size;
3557
3558 if (have_lost) {
3559 lost_event.header.type = PERF_RECORD_LOST;
3560 lost_event.header.misc = 0;
3561 lost_event.id = event->id;
3562 lost_event.lost = local_xchg(&buffer->lost, 0);
3563
3564 perf_output_put(handle, lost_event);
3565 perf_event__output_id_sample(event, handle, &sample_data);
3566 }
3567
3568 return 0;
3569
3570fail:
3571 local_inc(&buffer->lost);
3572 perf_output_put_handle(handle);
3573out:
3574 rcu_read_unlock();
3575
3576 return -ENOSPC;
3577}
3578
3579void perf_output_end(struct perf_output_handle *handle)
3580{
3581 struct perf_event *event = handle->event;
3582 struct perf_buffer *buffer = handle->buffer;
3583
3584 int wakeup_events = event->attr.wakeup_events;
3585
3586 if (handle->sample && wakeup_events) {
3587 int events = local_inc_return(&buffer->events);
3588 if (events >= wakeup_events) {
3589 local_sub(wakeup_events, &buffer->events);
3590 local_inc(&buffer->wakeup);
3591 }
3592 }
3593
3594 perf_output_put_handle(handle);
3595 rcu_read_unlock();
3596}
3597
3598static void perf_output_read_one(struct perf_output_handle *handle,
3599 struct perf_event *event,
3600 u64 enabled, u64 running)
3601{
3602 u64 read_format = event->attr.read_format;
3603 u64 values[4];
3604 int n = 0;
3605
3606 values[n++] = perf_event_count(event);
3607 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3608 values[n++] = enabled +
3609 atomic64_read(&event->child_total_time_enabled);
3610 }
3611 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3612 values[n++] = running +
3613 atomic64_read(&event->child_total_time_running);
3614 }
3615 if (read_format & PERF_FORMAT_ID)
3616 values[n++] = primary_event_id(event);
3617
3618 perf_output_copy(handle, values, n * sizeof(u64));
3619}
3620
3621
3622
3623
3624static void perf_output_read_group(struct perf_output_handle *handle,
3625 struct perf_event *event,
3626 u64 enabled, u64 running)
3627{
3628 struct perf_event *leader = event->group_leader, *sub;
3629 u64 read_format = event->attr.read_format;
3630 u64 values[5];
3631 int n = 0;
3632
3633 values[n++] = 1 + leader->nr_siblings;
3634
3635 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3636 values[n++] = enabled;
3637
3638 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3639 values[n++] = running;
3640
3641 if (leader != event)
3642 leader->pmu->read(leader);
3643
3644 values[n++] = perf_event_count(leader);
3645 if (read_format & PERF_FORMAT_ID)
3646 values[n++] = primary_event_id(leader);
3647
3648 perf_output_copy(handle, values, n * sizeof(u64));
3649
3650 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3651 n = 0;
3652
3653 if (sub != event)
3654 sub->pmu->read(sub);
3655
3656 values[n++] = perf_event_count(sub);
3657 if (read_format & PERF_FORMAT_ID)
3658 values[n++] = primary_event_id(sub);
3659
3660 perf_output_copy(handle, values, n * sizeof(u64));
3661 }
3662}
3663
3664#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
3665 PERF_FORMAT_TOTAL_TIME_RUNNING)
3666
3667static void perf_output_read(struct perf_output_handle *handle,
3668 struct perf_event *event)
3669{
3670 u64 enabled = 0, running = 0, now, ctx_time;
3671 u64 read_format = event->attr.read_format;
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682 if (read_format & PERF_FORMAT_TOTAL_TIMES) {
3683 now = perf_clock();
3684 ctx_time = event->shadow_ctx_time + now;
3685 enabled = ctx_time - event->tstamp_enabled;
3686 running = ctx_time - event->tstamp_running;
3687 }
3688
3689 if (event->attr.read_format & PERF_FORMAT_GROUP)
3690 perf_output_read_group(handle, event, enabled, running);
3691 else
3692 perf_output_read_one(handle, event, enabled, running);
3693}
3694
3695void perf_output_sample(struct perf_output_handle *handle,
3696 struct perf_event_header *header,
3697 struct perf_sample_data *data,
3698 struct perf_event *event)
3699{
3700 u64 sample_type = data->type;
3701
3702 perf_output_put(handle, *header);
3703
3704 if (sample_type & PERF_SAMPLE_IP)
3705 perf_output_put(handle, data->ip);
3706
3707 if (sample_type & PERF_SAMPLE_TID)
3708 perf_output_put(handle, data->tid_entry);
3709
3710 if (sample_type & PERF_SAMPLE_TIME)
3711 perf_output_put(handle, data->time);
3712
3713 if (sample_type & PERF_SAMPLE_ADDR)
3714 perf_output_put(handle, data->addr);
3715
3716 if (sample_type & PERF_SAMPLE_ID)
3717 perf_output_put(handle, data->id);
3718
3719 if (sample_type & PERF_SAMPLE_STREAM_ID)
3720 perf_output_put(handle, data->stream_id);
3721
3722 if (sample_type & PERF_SAMPLE_CPU)
3723 perf_output_put(handle, data->cpu_entry);
3724
3725 if (sample_type & PERF_SAMPLE_PERIOD)
3726 perf_output_put(handle, data->period);
3727
3728 if (sample_type & PERF_SAMPLE_READ)
3729 perf_output_read(handle, event);
3730
3731 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3732 if (data->callchain) {
3733 int size = 1;
3734
3735 if (data->callchain)
3736 size += data->callchain->nr;
3737
3738 size *= sizeof(u64);
3739
3740 perf_output_copy(handle, data->callchain, size);
3741 } else {
3742 u64 nr = 0;
3743 perf_output_put(handle, nr);
3744 }
3745 }
3746
3747 if (sample_type & PERF_SAMPLE_RAW) {
3748 if (data->raw) {
3749 perf_output_put(handle, data->raw->size);
3750 perf_output_copy(handle, data->raw->data,
3751 data->raw->size);
3752 } else {
3753 struct {
3754 u32 size;
3755 u32 data;
3756 } raw = {
3757 .size = sizeof(u32),
3758 .data = 0,
3759 };
3760 perf_output_put(handle, raw);
3761 }
3762 }
3763}
3764
3765void perf_prepare_sample(struct perf_event_header *header,
3766 struct perf_sample_data *data,
3767 struct perf_event *event,
3768 struct pt_regs *regs)
3769{
3770 u64 sample_type = event->attr.sample_type;
3771
3772 header->type = PERF_RECORD_SAMPLE;
3773 header->size = sizeof(*header) + event->header_size;
3774
3775 header->misc = 0;
3776 header->misc |= perf_misc_flags(regs);
3777
3778 __perf_event_header__init_id(header, data, event);
3779
3780 if (sample_type & PERF_SAMPLE_IP)
3781 data->ip = perf_instruction_pointer(regs);
3782
3783 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3784 int size = 1;
3785
3786 data->callchain = perf_callchain(regs);
3787
3788 if (data->callchain)
3789 size += data->callchain->nr;
3790
3791 header->size += size * sizeof(u64);
3792 }
3793
3794 if (sample_type & PERF_SAMPLE_RAW) {
3795 int size = sizeof(u32);
3796
3797 if (data->raw)
3798 size += data->raw->size;
3799 else
3800 size += sizeof(u32);
3801
3802 WARN_ON_ONCE(size & (sizeof(u64)-1));
3803 header->size += size;
3804 }
3805}
3806
3807static void perf_event_output(struct perf_event *event, int nmi,
3808 struct perf_sample_data *data,
3809 struct pt_regs *regs)
3810{
3811 struct perf_output_handle handle;
3812 struct perf_event_header header;
3813
3814
3815 rcu_read_lock();
3816
3817 perf_prepare_sample(&header, data, event, regs);
3818
3819 if (perf_output_begin(&handle, event, header.size, nmi, 1))
3820 goto exit;
3821
3822 perf_output_sample(&handle, &header, data, event);
3823
3824 perf_output_end(&handle);
3825
3826exit:
3827 rcu_read_unlock();
3828}
3829
3830
3831
3832
3833
3834struct perf_read_event {
3835 struct perf_event_header header;
3836
3837 u32 pid;
3838 u32 tid;
3839};
3840
3841static void
3842perf_event_read_event(struct perf_event *event,
3843 struct task_struct *task)
3844{
3845 struct perf_output_handle handle;
3846 struct perf_sample_data sample;
3847 struct perf_read_event read_event = {
3848 .header = {
3849 .type = PERF_RECORD_READ,
3850 .misc = 0,
3851 .size = sizeof(read_event) + event->read_size,
3852 },
3853 .pid = perf_event_pid(event, task),
3854 .tid = perf_event_tid(event, task),
3855 };
3856 int ret;
3857
3858 perf_event_header__init_id(&read_event.header, &sample, event);
3859 ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3860 if (ret)
3861 return;
3862
3863 perf_output_put(&handle, read_event);
3864 perf_output_read(&handle, event);
3865 perf_event__output_id_sample(event, &handle, &sample);
3866
3867 perf_output_end(&handle);
3868}
3869
3870
3871
3872
3873
3874
3875
3876struct perf_task_event {
3877 struct task_struct *task;
3878 struct perf_event_context *task_ctx;
3879
3880 struct {
3881 struct perf_event_header header;
3882
3883 u32 pid;
3884 u32 ppid;
3885 u32 tid;
3886 u32 ptid;
3887 u64 time;
3888 } event_id;
3889};
3890
3891static void perf_event_task_output(struct perf_event *event,
3892 struct perf_task_event *task_event)
3893{
3894 struct perf_output_handle handle;
3895 struct perf_sample_data sample;
3896 struct task_struct *task = task_event->task;
3897 int ret, size = task_event->event_id.header.size;
3898
3899 perf_event_header__init_id(&task_event->event_id.header, &sample, event);
3900
3901 ret = perf_output_begin(&handle, event,
3902 task_event->event_id.header.size, 0, 0);
3903 if (ret)
3904 goto out;
3905
3906 task_event->event_id.pid = perf_event_pid(event, task);
3907 task_event->event_id.ppid = perf_event_pid(event, current);
3908
3909 task_event->event_id.tid = perf_event_tid(event, task);
3910 task_event->event_id.ptid = perf_event_tid(event, current);
3911
3912 perf_output_put(&handle, task_event->event_id);
3913
3914 perf_event__output_id_sample(event, &handle, &sample);
3915
3916 perf_output_end(&handle);
3917out:
3918 task_event->event_id.header.size = size;
3919}
3920
3921static int perf_event_task_match(struct perf_event *event)
3922{
3923 if (event->state < PERF_EVENT_STATE_INACTIVE)
3924 return 0;
3925
3926 if (!event_filter_match(event))
3927 return 0;
3928
3929 if (event->attr.comm || event->attr.mmap ||
3930 event->attr.mmap_data || event->attr.task)
3931 return 1;
3932
3933 return 0;
3934}
3935
3936static void perf_event_task_ctx(struct perf_event_context *ctx,
3937 struct perf_task_event *task_event)
3938{
3939 struct perf_event *event;
3940
3941 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3942 if (perf_event_task_match(event))
3943 perf_event_task_output(event, task_event);
3944 }
3945}
3946
3947static void perf_event_task_event(struct perf_task_event *task_event)
3948{
3949 struct perf_cpu_context *cpuctx;
3950 struct perf_event_context *ctx;
3951 struct pmu *pmu;
3952 int ctxn;
3953
3954 rcu_read_lock();
3955 list_for_each_entry_rcu(pmu, &pmus, entry) {
3956 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
3957 if (cpuctx->active_pmu != pmu)
3958 goto next;
3959 perf_event_task_ctx(&cpuctx->ctx, task_event);
3960
3961 ctx = task_event->task_ctx;
3962 if (!ctx) {
3963 ctxn = pmu->task_ctx_nr;
3964 if (ctxn < 0)
3965 goto next;
3966 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
3967 }
3968 if (ctx)
3969 perf_event_task_ctx(ctx, task_event);
3970next:
3971 put_cpu_ptr(pmu->pmu_cpu_context);
3972 }
3973 rcu_read_unlock();
3974}
3975
3976static void perf_event_task(struct task_struct *task,
3977 struct perf_event_context *task_ctx,
3978 int new)
3979{
3980 struct perf_task_event task_event;
3981
3982 if (!atomic_read(&nr_comm_events) &&
3983 !atomic_read(&nr_mmap_events) &&
3984 !atomic_read(&nr_task_events))
3985 return;
3986
3987 task_event = (struct perf_task_event){
3988 .task = task,
3989 .task_ctx = task_ctx,
3990 .event_id = {
3991 .header = {
3992 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3993 .misc = 0,
3994 .size = sizeof(task_event.event_id),
3995 },
3996
3997
3998
3999
4000 .time = perf_clock(),
4001 },
4002 };
4003
4004 perf_event_task_event(&task_event);
4005}
4006
4007void perf_event_fork(struct task_struct *task)
4008{
4009 perf_event_task(task, NULL, 1);
4010}
4011
4012
4013
4014
4015
4016struct perf_comm_event {
4017 struct task_struct *task;
4018 char *comm;
4019 int comm_size;
4020
4021 struct {
4022 struct perf_event_header header;
4023
4024 u32 pid;
4025 u32 tid;
4026 } event_id;
4027};
4028
4029static void perf_event_comm_output(struct perf_event *event,
4030 struct perf_comm_event *comm_event)
4031{
4032 struct perf_output_handle handle;
4033 struct perf_sample_data sample;
4034 int size = comm_event->event_id.header.size;
4035 int ret;
4036
4037 perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4038 ret = perf_output_begin(&handle, event,
4039 comm_event->event_id.header.size, 0, 0);
4040
4041 if (ret)
4042 goto out;
4043
4044 comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4045 comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4046
4047 perf_output_put(&handle, comm_event->event_id);
4048 perf_output_copy(&handle, comm_event->comm,
4049 comm_event->comm_size);
4050
4051 perf_event__output_id_sample(event, &handle, &sample);
4052
4053 perf_output_end(&handle);
4054out:
4055 comm_event->event_id.header.size = size;
4056}
4057
4058static int perf_event_comm_match(struct perf_event *event)
4059{
4060 if (event->state < PERF_EVENT_STATE_INACTIVE)
4061 return 0;
4062
4063 if (!event_filter_match(event))
4064 return 0;
4065
4066 if (event->attr.comm)
4067 return 1;
4068
4069 return 0;
4070}
4071
4072static void perf_event_comm_ctx(struct perf_event_context *ctx,
4073 struct perf_comm_event *comm_event)
4074{
4075 struct perf_event *event;
4076
4077 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4078 if (perf_event_comm_match(event))
4079 perf_event_comm_output(event, comm_event);
4080 }
4081}
4082
4083static void perf_event_comm_event(struct perf_comm_event *comm_event)
4084{
4085 struct perf_cpu_context *cpuctx;
4086 struct perf_event_context *ctx;
4087 char comm[TASK_COMM_LEN];
4088 unsigned int size;
4089 struct pmu *pmu;
4090 int ctxn;
4091
4092 memset(comm, 0, sizeof(comm));
4093 strlcpy(comm, comm_event->task->comm, sizeof(comm));
4094 size = ALIGN(strlen(comm)+1, sizeof(u64));
4095
4096 comm_event->comm = comm;
4097 comm_event->comm_size = size;
4098
4099 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4100 rcu_read_lock();
4101 list_for_each_entry_rcu(pmu, &pmus, entry) {
4102 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4103 if (cpuctx->active_pmu != pmu)
4104 goto next;
4105 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4106
4107 ctxn = pmu->task_ctx_nr;
4108 if (ctxn < 0)
4109 goto next;
4110
4111 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4112 if (ctx)
4113 perf_event_comm_ctx(ctx, comm_event);
4114next:
4115 put_cpu_ptr(pmu->pmu_cpu_context);
4116 }
4117 rcu_read_unlock();
4118}
4119
4120void perf_event_comm(struct task_struct *task)
4121{
4122 struct perf_comm_event comm_event;
4123 struct perf_event_context *ctx;
4124 int ctxn;
4125
4126 for_each_task_context_nr(ctxn) {
4127 ctx = task->perf_event_ctxp[ctxn];
4128 if (!ctx)
4129 continue;
4130
4131 perf_event_enable_on_exec(ctx);
4132 }
4133
4134 if (!atomic_read(&nr_comm_events))
4135 return;
4136
4137 comm_event = (struct perf_comm_event){
4138 .task = task,
4139
4140
4141 .event_id = {
4142 .header = {
4143 .type = PERF_RECORD_COMM,
4144 .misc = 0,
4145
4146 },
4147
4148
4149 },
4150 };
4151
4152 perf_event_comm_event(&comm_event);
4153}
4154
4155
4156
4157
4158
4159struct perf_mmap_event {
4160 struct vm_area_struct *vma;
4161
4162 const char *file_name;
4163 int file_size;
4164
4165 struct {
4166 struct perf_event_header header;
4167
4168 u32 pid;
4169 u32 tid;
4170 u64 start;
4171 u64 len;
4172 u64 pgoff;
4173 } event_id;
4174};
4175
4176static void perf_event_mmap_output(struct perf_event *event,
4177 struct perf_mmap_event *mmap_event)
4178{
4179 struct perf_output_handle handle;
4180 struct perf_sample_data sample;
4181 int size = mmap_event->event_id.header.size;
4182 int ret;
4183
4184 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4185 ret = perf_output_begin(&handle, event,
4186 mmap_event->event_id.header.size, 0, 0);
4187 if (ret)
4188 goto out;
4189
4190 mmap_event->event_id.pid = perf_event_pid(event, current);
4191 mmap_event->event_id.tid = perf_event_tid(event, current);
4192
4193 perf_output_put(&handle, mmap_event->event_id);
4194 perf_output_copy(&handle, mmap_event->file_name,
4195 mmap_event->file_size);
4196
4197 perf_event__output_id_sample(event, &handle, &sample);
4198
4199 perf_output_end(&handle);
4200out:
4201 mmap_event->event_id.header.size = size;
4202}
4203
4204static int perf_event_mmap_match(struct perf_event *event,
4205 struct perf_mmap_event *mmap_event,
4206 int executable)
4207{
4208 if (event->state < PERF_EVENT_STATE_INACTIVE)
4209 return 0;
4210
4211 if (!event_filter_match(event))
4212 return 0;
4213
4214 if ((!executable && event->attr.mmap_data) ||
4215 (executable && event->attr.mmap))
4216 return 1;
4217
4218 return 0;
4219}
4220
4221static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4222 struct perf_mmap_event *mmap_event,
4223 int executable)
4224{
4225 struct perf_event *event;
4226
4227 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4228 if (perf_event_mmap_match(event, mmap_event, executable))
4229 perf_event_mmap_output(event, mmap_event);
4230 }
4231}
4232
4233static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4234{
4235 struct perf_cpu_context *cpuctx;
4236 struct perf_event_context *ctx;
4237 struct vm_area_struct *vma = mmap_event->vma;
4238 struct file *file = vma->vm_file;
4239 unsigned int size;
4240 char tmp[16];
4241 char *buf = NULL;
4242 const char *name;
4243 struct pmu *pmu;
4244 int ctxn;
4245
4246 memset(tmp, 0, sizeof(tmp));
4247
4248 if (file) {
4249
4250
4251
4252
4253
4254 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4255 if (!buf) {
4256 name = strncpy(tmp, "//enomem", sizeof(tmp));
4257 goto got_name;
4258 }
4259 name = d_path(&file->f_path, buf, PATH_MAX);
4260 if (IS_ERR(name)) {
4261 name = strncpy(tmp, "//toolong", sizeof(tmp));
4262 goto got_name;
4263 }
4264 } else {
4265 if (arch_vma_name(mmap_event->vma)) {
4266 name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4267 sizeof(tmp));
4268 goto got_name;
4269 }
4270
4271 if (!vma->vm_mm) {
4272 name = strncpy(tmp, "[vdso]", sizeof(tmp));
4273 goto got_name;
4274 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4275 vma->vm_end >= vma->vm_mm->brk) {
4276 name = strncpy(tmp, "[heap]", sizeof(tmp));
4277 goto got_name;
4278 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4279 vma->vm_end >= vma->vm_mm->start_stack) {
4280 name = strncpy(tmp, "[stack]", sizeof(tmp));
4281 goto got_name;
4282 }
4283
4284 name = strncpy(tmp, "//anon", sizeof(tmp));
4285 goto got_name;
4286 }
4287
4288got_name:
4289 size = ALIGN(strlen(name)+1, sizeof(u64));
4290
4291 mmap_event->file_name = name;
4292 mmap_event->file_size = size;
4293
4294 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4295
4296 rcu_read_lock();
4297 list_for_each_entry_rcu(pmu, &pmus, entry) {
4298 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4299 if (cpuctx->active_pmu != pmu)
4300 goto next;
4301 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4302 vma->vm_flags & VM_EXEC);
4303
4304 ctxn = pmu->task_ctx_nr;
4305 if (ctxn < 0)
4306 goto next;
4307
4308 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4309 if (ctx) {
4310 perf_event_mmap_ctx(ctx, mmap_event,
4311 vma->vm_flags & VM_EXEC);
4312 }
4313next:
4314 put_cpu_ptr(pmu->pmu_cpu_context);
4315 }
4316 rcu_read_unlock();
4317
4318 kfree(buf);
4319}
4320
4321void perf_event_mmap(struct vm_area_struct *vma)
4322{
4323 struct perf_mmap_event mmap_event;
4324
4325 if (!atomic_read(&nr_mmap_events))
4326 return;
4327
4328 mmap_event = (struct perf_mmap_event){
4329 .vma = vma,
4330
4331
4332 .event_id = {
4333 .header = {
4334 .type = PERF_RECORD_MMAP,
4335 .misc = PERF_RECORD_MISC_USER,
4336
4337 },
4338
4339
4340 .start = vma->vm_start,
4341 .len = vma->vm_end - vma->vm_start,
4342 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
4343 },
4344 };
4345
4346 perf_event_mmap_event(&mmap_event);
4347}
4348
4349
4350
4351
4352
4353static void perf_log_throttle(struct perf_event *event, int enable)
4354{
4355 struct perf_output_handle handle;
4356 struct perf_sample_data sample;
4357 int ret;
4358
4359 struct {
4360 struct perf_event_header header;
4361 u64 time;
4362 u64 id;
4363 u64 stream_id;
4364 } throttle_event = {
4365 .header = {
4366 .type = PERF_RECORD_THROTTLE,
4367 .misc = 0,
4368 .size = sizeof(throttle_event),
4369 },
4370 .time = perf_clock(),
4371 .id = primary_event_id(event),
4372 .stream_id = event->id,
4373 };
4374
4375 if (enable)
4376 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4377
4378 perf_event_header__init_id(&throttle_event.header, &sample, event);
4379
4380 ret = perf_output_begin(&handle, event,
4381 throttle_event.header.size, 1, 0);
4382 if (ret)
4383 return;
4384
4385 perf_output_put(&handle, throttle_event);
4386 perf_event__output_id_sample(event, &handle, &sample);
4387 perf_output_end(&handle);
4388}
4389
4390
4391
4392
4393
4394static int __perf_event_overflow(struct perf_event *event, int nmi,
4395 int throttle, struct perf_sample_data *data,
4396 struct pt_regs *regs)
4397{
4398 int events = atomic_read(&event->event_limit);
4399 struct hw_perf_event *hwc = &event->hw;
4400 int ret = 0;
4401
4402
4403
4404
4405
4406 if (unlikely(!is_sampling_event(event)))
4407 return 0;
4408
4409 if (!throttle) {
4410 hwc->interrupts++;
4411 } else {
4412 if (hwc->interrupts != MAX_INTERRUPTS) {
4413 hwc->interrupts++;
4414 if (HZ * hwc->interrupts >
4415 (u64)sysctl_perf_event_sample_rate) {
4416 hwc->interrupts = MAX_INTERRUPTS;
4417 perf_log_throttle(event, 0);
4418 ret = 1;
4419 }
4420 } else {
4421
4422
4423
4424
4425
4426 ret = 1;
4427 }
4428 }
4429
4430 if (event->attr.freq) {
4431 u64 now = perf_clock();
4432 s64 delta = now - hwc->freq_time_stamp;
4433
4434 hwc->freq_time_stamp = now;
4435
4436 if (delta > 0 && delta < 2*TICK_NSEC)
4437 perf_adjust_period(event, delta, hwc->last_period);
4438 }
4439
4440
4441
4442
4443
4444
4445 event->pending_kill = POLL_IN;
4446 if (events && atomic_dec_and_test(&event->event_limit)) {
4447 ret = 1;
4448 event->pending_kill = POLL_HUP;
4449 if (nmi) {
4450 event->pending_disable = 1;
4451 irq_work_queue(&event->pending);
4452 } else
4453 perf_event_disable(event);
4454 }
4455
4456 if (event->overflow_handler)
4457 event->overflow_handler(event, nmi, data, regs);
4458 else
4459 perf_event_output(event, nmi, data, regs);
4460
4461 return ret;
4462}
4463
4464int perf_event_overflow(struct perf_event *event, int nmi,
4465 struct perf_sample_data *data,
4466 struct pt_regs *regs)
4467{
4468 return __perf_event_overflow(event, nmi, 1, data, regs);
4469}
4470
4471
4472
4473
4474
4475struct swevent_htable {
4476 struct swevent_hlist *swevent_hlist;
4477 struct mutex hlist_mutex;
4478 int hlist_refcount;
4479
4480
4481 int recursion[PERF_NR_CONTEXTS];
4482};
4483
4484static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
4485
4486
4487
4488
4489
4490
4491
4492
4493static u64 perf_swevent_set_period(struct perf_event *event)
4494{
4495 struct hw_perf_event *hwc = &event->hw;
4496 u64 period = hwc->last_period;
4497 u64 nr, offset;
4498 s64 old, val;
4499
4500 hwc->last_period = hwc->sample_period;
4501
4502again:
4503 old = val = local64_read(&hwc->period_left);
4504 if (val < 0)
4505 return 0;
4506
4507 nr = div64_u64(period + val, period);
4508 offset = nr * period;
4509 val -= offset;
4510 if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4511 goto again;
4512
4513 return nr;
4514}
4515
4516static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4517 int nmi, struct perf_sample_data *data,
4518 struct pt_regs *regs)
4519{
4520 struct hw_perf_event *hwc = &event->hw;
4521 int throttle = 0;
4522
4523 data->period = event->hw.last_period;
4524 if (!overflow)
4525 overflow = perf_swevent_set_period(event);
4526
4527 if (hwc->interrupts == MAX_INTERRUPTS)
4528 return;
4529
4530 for (; overflow; overflow--) {
4531 if (__perf_event_overflow(event, nmi, throttle,
4532 data, regs)) {
4533
4534
4535
4536
4537 break;
4538 }
4539 throttle = 1;
4540 }
4541}
4542
4543static void perf_swevent_event(struct perf_event *event, u64 nr,
4544 int nmi, struct perf_sample_data *data,
4545 struct pt_regs *regs)
4546{
4547 struct hw_perf_event *hwc = &event->hw;
4548
4549 local64_add(nr, &event->count);
4550
4551 if (!regs)
4552 return;
4553
4554 if (!is_sampling_event(event))
4555 return;
4556
4557 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4558 return perf_swevent_overflow(event, 1, nmi, data, regs);
4559
4560 if (local64_add_negative(nr, &hwc->period_left))
4561 return;
4562
4563 perf_swevent_overflow(event, 0, nmi, data, regs);
4564}
4565
4566static int perf_exclude_event(struct perf_event *event,
4567 struct pt_regs *regs)
4568{
4569 if (event->hw.state & PERF_HES_STOPPED)
4570 return 0;
4571
4572 if (regs) {
4573 if (event->attr.exclude_user && user_mode(regs))
4574 return 1;
4575
4576 if (event->attr.exclude_kernel && !user_mode(regs))
4577 return 1;
4578 }
4579
4580 return 0;
4581}
4582
4583static int perf_swevent_match(struct perf_event *event,
4584 enum perf_type_id type,
4585 u32 event_id,
4586 struct perf_sample_data *data,
4587 struct pt_regs *regs)
4588{
4589 if (event->attr.type != type)
4590 return 0;
4591
4592 if (event->attr.config != event_id)
4593 return 0;
4594
4595 if (perf_exclude_event(event, regs))
4596 return 0;
4597
4598 return 1;
4599}
4600
4601static inline u64 swevent_hash(u64 type, u32 event_id)
4602{
4603 u64 val = event_id | (type << 32);
4604
4605 return hash_64(val, SWEVENT_HLIST_BITS);
4606}
4607
4608static inline struct hlist_head *
4609__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4610{
4611 u64 hash = swevent_hash(type, event_id);
4612
4613 return &hlist->heads[hash];
4614}
4615
4616
4617static inline struct hlist_head *
4618find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4619{
4620 struct swevent_hlist *hlist;
4621
4622 hlist = rcu_dereference(swhash->swevent_hlist);
4623 if (!hlist)
4624 return NULL;
4625
4626 return __find_swevent_head(hlist, type, event_id);
4627}
4628
4629
4630static inline struct hlist_head *
4631find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4632{
4633 struct swevent_hlist *hlist;
4634 u32 event_id = event->attr.config;
4635 u64 type = event->attr.type;
4636
4637
4638
4639
4640
4641
4642 hlist = rcu_dereference_protected(swhash->swevent_hlist,
4643 lockdep_is_held(&event->ctx->lock));
4644 if (!hlist)
4645 return NULL;
4646
4647 return __find_swevent_head(hlist, type, event_id);
4648}
4649
4650static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4651 u64 nr, int nmi,
4652 struct perf_sample_data *data,
4653 struct pt_regs *regs)
4654{
4655 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4656 struct perf_event *event;
4657 struct hlist_node *node;
4658 struct hlist_head *head;
4659
4660 rcu_read_lock();
4661 head = find_swevent_head_rcu(swhash, type, event_id);
4662 if (!head)
4663 goto end;
4664
4665 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4666 if (perf_swevent_match(event, type, event_id, data, regs))
4667 perf_swevent_event(event, nr, nmi, data, regs);
4668 }
4669end:
4670 rcu_read_unlock();
4671}
4672
4673int perf_swevent_get_recursion_context(void)
4674{
4675 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4676
4677 return get_recursion_context(swhash->recursion);
4678}
4679EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4680
4681inline void perf_swevent_put_recursion_context(int rctx)
4682{
4683 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4684
4685 put_recursion_context(swhash->recursion, rctx);
4686}
4687
4688void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4689 struct pt_regs *regs, u64 addr)
4690{
4691 struct perf_sample_data data;
4692 int rctx;
4693
4694 preempt_disable_notrace();
4695 rctx = perf_swevent_get_recursion_context();
4696 if (rctx < 0)
4697 return;
4698
4699 perf_sample_data_init(&data, addr);
4700
4701 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4702
4703 perf_swevent_put_recursion_context(rctx);
4704 preempt_enable_notrace();
4705}
4706
4707static void perf_swevent_read(struct perf_event *event)
4708{
4709}
4710
4711static int perf_swevent_add(struct perf_event *event, int flags)
4712{
4713 struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4714 struct hw_perf_event *hwc = &event->hw;
4715 struct hlist_head *head;
4716
4717 if (is_sampling_event(event)) {
4718 hwc->last_period = hwc->sample_period;
4719 perf_swevent_set_period(event);
4720 }
4721
4722 hwc->state = !(flags & PERF_EF_START);
4723
4724 head = find_swevent_head(swhash, event);
4725 if (WARN_ON_ONCE(!head))
4726 return -EINVAL;
4727
4728 hlist_add_head_rcu(&event->hlist_entry, head);
4729
4730 return 0;
4731}
4732
4733static void perf_swevent_del(struct perf_event *event, int flags)
4734{
4735 hlist_del_rcu(&event->hlist_entry);
4736}
4737
4738static void perf_swevent_start(struct perf_event *event, int flags)
4739{
4740 event->hw.state = 0;
4741}
4742
4743static void perf_swevent_stop(struct perf_event *event, int flags)
4744{
4745 event->hw.state = PERF_HES_STOPPED;
4746}
4747
4748
4749static inline struct swevent_hlist *
4750swevent_hlist_deref(struct swevent_htable *swhash)
4751{
4752 return rcu_dereference_protected(swhash->swevent_hlist,
4753 lockdep_is_held(&swhash->hlist_mutex));
4754}
4755
4756static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
4757{
4758 struct swevent_hlist *hlist;
4759
4760 hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
4761 kfree(hlist);
4762}
4763
4764static void swevent_hlist_release(struct swevent_htable *swhash)
4765{
4766 struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
4767
4768 if (!hlist)
4769 return;
4770
4771 rcu_assign_pointer(swhash->swevent_hlist, NULL);
4772 call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
4773}
4774
4775static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
4776{
4777 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4778
4779 mutex_lock(&swhash->hlist_mutex);
4780
4781 if (!--swhash->hlist_refcount)
4782 swevent_hlist_release(swhash);
4783
4784 mutex_unlock(&swhash->hlist_mutex);
4785}
4786
4787static void swevent_hlist_put(struct perf_event *event)
4788{
4789 int cpu;
4790
4791 if (event->cpu != -1) {
4792 swevent_hlist_put_cpu(event, event->cpu);
4793 return;
4794 }
4795
4796 for_each_possible_cpu(cpu)
4797 swevent_hlist_put_cpu(event, cpu);
4798}
4799
4800static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
4801{
4802 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4803 int err = 0;
4804
4805 mutex_lock(&swhash->hlist_mutex);
4806
4807 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
4808 struct swevent_hlist *hlist;
4809
4810 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
4811 if (!hlist) {
4812 err = -ENOMEM;
4813 goto exit;
4814 }
4815 rcu_assign_pointer(swhash->swevent_hlist, hlist);
4816 }
4817 swhash->hlist_refcount++;
4818exit:
4819 mutex_unlock(&swhash->hlist_mutex);
4820
4821 return err;
4822}
4823
4824static int swevent_hlist_get(struct perf_event *event)
4825{
4826 int err;
4827 int cpu, failed_cpu;
4828
4829 if (event->cpu != -1)
4830 return swevent_hlist_get_cpu(event, event->cpu);
4831
4832 get_online_cpus();
4833 for_each_possible_cpu(cpu) {
4834 err = swevent_hlist_get_cpu(event, cpu);
4835 if (err) {
4836 failed_cpu = cpu;
4837 goto fail;
4838 }
4839 }
4840 put_online_cpus();
4841
4842 return 0;
4843fail:
4844 for_each_possible_cpu(cpu) {
4845 if (cpu == failed_cpu)
4846 break;
4847 swevent_hlist_put_cpu(event, cpu);
4848 }
4849
4850 put_online_cpus();
4851 return err;
4852}
4853
4854atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4855
4856static void sw_perf_event_destroy(struct perf_event *event)
4857{
4858 u64 event_id = event->attr.config;
4859
4860 WARN_ON(event->parent);
4861
4862 jump_label_dec(&perf_swevent_enabled[event_id]);
4863 swevent_hlist_put(event);
4864}
4865
4866static int perf_swevent_init(struct perf_event *event)
4867{
4868 int event_id = event->attr.config;
4869
4870 if (event->attr.type != PERF_TYPE_SOFTWARE)
4871 return -ENOENT;
4872
4873 switch (event_id) {
4874 case PERF_COUNT_SW_CPU_CLOCK:
4875 case PERF_COUNT_SW_TASK_CLOCK:
4876 return -ENOENT;
4877
4878 default:
4879 break;
4880 }
4881
4882 if (event_id >= PERF_COUNT_SW_MAX)
4883 return -ENOENT;
4884
4885 if (!event->parent) {
4886 int err;
4887
4888 err = swevent_hlist_get(event);
4889 if (err)
4890 return err;
4891
4892 jump_label_inc(&perf_swevent_enabled[event_id]);
4893 event->destroy = sw_perf_event_destroy;
4894 }
4895
4896 return 0;
4897}
4898
4899static struct pmu perf_swevent = {
4900 .task_ctx_nr = perf_sw_context,
4901
4902 .event_init = perf_swevent_init,
4903 .add = perf_swevent_add,
4904 .del = perf_swevent_del,
4905 .start = perf_swevent_start,
4906 .stop = perf_swevent_stop,
4907 .read = perf_swevent_read,
4908};
4909
4910#ifdef CONFIG_EVENT_TRACING
4911
4912static int perf_tp_filter_match(struct perf_event *event,
4913 struct perf_sample_data *data)
4914{
4915 void *record = data->raw->data;
4916
4917 if (likely(!event->filter) || filter_match_preds(event->filter, record))
4918 return 1;
4919 return 0;
4920}
4921
4922static int perf_tp_event_match(struct perf_event *event,
4923 struct perf_sample_data *data,
4924 struct pt_regs *regs)
4925{
4926
4927
4928
4929 if (event->attr.exclude_kernel)
4930 return 0;
4931
4932 if (!perf_tp_filter_match(event, data))
4933 return 0;
4934
4935 return 1;
4936}
4937
4938void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
4939 struct pt_regs *regs, struct hlist_head *head, int rctx)
4940{
4941 struct perf_sample_data data;
4942 struct perf_event *event;
4943 struct hlist_node *node;
4944
4945 struct perf_raw_record raw = {
4946 .size = entry_size,
4947 .data = record,
4948 };
4949
4950 perf_sample_data_init(&data, addr);
4951 data.raw = &raw;
4952
4953 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4954 if (perf_tp_event_match(event, &data, regs))
4955 perf_swevent_event(event, count, 1, &data, regs);
4956 }
4957
4958 perf_swevent_put_recursion_context(rctx);
4959}
4960EXPORT_SYMBOL_GPL(perf_tp_event);
4961
4962static void tp_perf_event_destroy(struct perf_event *event)
4963{
4964 perf_trace_destroy(event);
4965}
4966
4967static int perf_tp_event_init(struct perf_event *event)
4968{
4969 int err;
4970
4971 if (event->attr.type != PERF_TYPE_TRACEPOINT)
4972 return -ENOENT;
4973
4974 err = perf_trace_init(event);
4975 if (err)
4976 return err;
4977
4978 event->destroy = tp_perf_event_destroy;
4979
4980 return 0;
4981}
4982
4983static struct pmu perf_tracepoint = {
4984 .task_ctx_nr = perf_sw_context,
4985
4986 .event_init = perf_tp_event_init,
4987 .add = perf_trace_add,
4988 .del = perf_trace_del,
4989 .start = perf_swevent_start,
4990 .stop = perf_swevent_stop,
4991 .read = perf_swevent_read,
4992};
4993
4994static inline void perf_tp_register(void)
4995{
4996 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
4997}
4998
4999static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5000{
5001 char *filter_str;
5002 int ret;
5003
5004 if (event->attr.type != PERF_TYPE_TRACEPOINT)
5005 return -EINVAL;
5006
5007 filter_str = strndup_user(arg, PAGE_SIZE);
5008 if (IS_ERR(filter_str))
5009 return PTR_ERR(filter_str);
5010
5011 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5012
5013 kfree(filter_str);
5014 return ret;
5015}
5016
5017static void perf_event_free_filter(struct perf_event *event)
5018{
5019 ftrace_profile_free_filter(event);
5020}
5021
5022#else
5023
5024static inline void perf_tp_register(void)
5025{
5026}
5027
5028static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5029{
5030 return -ENOENT;
5031}
5032
5033static void perf_event_free_filter(struct perf_event *event)
5034{
5035}
5036
5037#endif
5038
5039#ifdef CONFIG_HAVE_HW_BREAKPOINT
5040void perf_bp_event(struct perf_event *bp, void *data)
5041{
5042 struct perf_sample_data sample;
5043 struct pt_regs *regs = data;
5044
5045 perf_sample_data_init(&sample, bp->attr.bp_addr);
5046
5047 if (!bp->hw.state && !perf_exclude_event(bp, regs))
5048 perf_swevent_event(bp, 1, 1, &sample, regs);
5049}
5050#endif
5051
5052
5053
5054
5055
5056static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5057{
5058 enum hrtimer_restart ret = HRTIMER_RESTART;
5059 struct perf_sample_data data;
5060 struct pt_regs *regs;
5061 struct perf_event *event;
5062 u64 period;
5063
5064 event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5065 event->pmu->read(event);
5066
5067 perf_sample_data_init(&data, 0);
5068 data.period = event->hw.last_period;
5069 regs = get_irq_regs();
5070
5071 if (regs && !perf_exclude_event(event, regs)) {
5072 if (!(event->attr.exclude_idle && current->pid == 0))
5073 if (perf_event_overflow(event, 0, &data, regs))
5074 ret = HRTIMER_NORESTART;
5075 }
5076
5077 period = max_t(u64, 10000, event->hw.sample_period);
5078 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5079
5080 return ret;
5081}
5082
5083static void perf_swevent_start_hrtimer(struct perf_event *event)
5084{
5085 struct hw_perf_event *hwc = &event->hw;
5086 s64 period;
5087
5088 if (!is_sampling_event(event))
5089 return;
5090
5091 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5092 hwc->hrtimer.function = perf_swevent_hrtimer;
5093
5094 period = local64_read(&hwc->period_left);
5095 if (period) {
5096 if (period < 0)
5097 period = 10000;
5098
5099 local64_set(&hwc->period_left, 0);
5100 } else {
5101 period = max_t(u64, 10000, hwc->sample_period);
5102 }
5103 __hrtimer_start_range_ns(&hwc->hrtimer,
5104 ns_to_ktime(period), 0,
5105 HRTIMER_MODE_REL_PINNED, 0);
5106}
5107
5108static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5109{
5110 struct hw_perf_event *hwc = &event->hw;
5111
5112 if (is_sampling_event(event)) {
5113 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5114 local64_set(&hwc->period_left, ktime_to_ns(remaining));
5115
5116 hrtimer_cancel(&hwc->hrtimer);
5117 }
5118}
5119
5120
5121
5122
5123
5124static void cpu_clock_event_update(struct perf_event *event)
5125{
5126 s64 prev;
5127 u64 now;
5128
5129 now = local_clock();
5130 prev = local64_xchg(&event->hw.prev_count, now);
5131 local64_add(now - prev, &event->count);
5132}
5133
5134static void cpu_clock_event_start(struct perf_event *event, int flags)
5135{
5136 local64_set(&event->hw.prev_count, local_clock());
5137 perf_swevent_start_hrtimer(event);
5138}
5139
5140static void cpu_clock_event_stop(struct perf_event *event, int flags)
5141{
5142 perf_swevent_cancel_hrtimer(event);
5143 cpu_clock_event_update(event);
5144}
5145
5146static int cpu_clock_event_add(struct perf_event *event, int flags)
5147{
5148 if (flags & PERF_EF_START)
5149 cpu_clock_event_start(event, flags);
5150
5151 return 0;
5152}
5153
5154static void cpu_clock_event_del(struct perf_event *event, int flags)
5155{
5156 cpu_clock_event_stop(event, flags);
5157}
5158
5159static void cpu_clock_event_read(struct perf_event *event)
5160{
5161 cpu_clock_event_update(event);
5162}
5163
5164static int cpu_clock_event_init(struct perf_event *event)
5165{
5166 if (event->attr.type != PERF_TYPE_SOFTWARE)
5167 return -ENOENT;
5168
5169 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5170 return -ENOENT;
5171
5172 return 0;
5173}
5174
5175static struct pmu perf_cpu_clock = {
5176 .task_ctx_nr = perf_sw_context,
5177
5178 .event_init = cpu_clock_event_init,
5179 .add = cpu_clock_event_add,
5180 .del = cpu_clock_event_del,
5181 .start = cpu_clock_event_start,
5182 .stop = cpu_clock_event_stop,
5183 .read = cpu_clock_event_read,
5184};
5185
5186
5187
5188
5189
5190static void task_clock_event_update(struct perf_event *event, u64 now)
5191{
5192 u64 prev;
5193 s64 delta;
5194
5195 prev = local64_xchg(&event->hw.prev_count, now);
5196 delta = now - prev;
5197 local64_add(delta, &event->count);
5198}
5199
5200static void task_clock_event_start(struct perf_event *event, int flags)
5201{
5202 local64_set(&event->hw.prev_count, event->ctx->time);
5203 perf_swevent_start_hrtimer(event);
5204}
5205
5206static void task_clock_event_stop(struct perf_event *event, int flags)
5207{
5208 perf_swevent_cancel_hrtimer(event);
5209 task_clock_event_update(event, event->ctx->time);
5210}
5211
5212static int task_clock_event_add(struct perf_event *event, int flags)
5213{
5214 if (flags & PERF_EF_START)
5215 task_clock_event_start(event, flags);
5216
5217 return 0;
5218}
5219
5220static void task_clock_event_del(struct perf_event *event, int flags)
5221{
5222 task_clock_event_stop(event, PERF_EF_UPDATE);
5223}
5224
5225static void task_clock_event_read(struct perf_event *event)
5226{
5227 u64 time;
5228
5229 if (!in_nmi()) {
5230 update_context_time(event->ctx);
5231 time = event->ctx->time;
5232 } else {
5233 u64 now = perf_clock();
5234 u64 delta = now - event->ctx->timestamp;
5235 time = event->ctx->time + delta;
5236 }
5237
5238 task_clock_event_update(event, time);
5239}
5240
5241static int task_clock_event_init(struct perf_event *event)
5242{
5243 if (event->attr.type != PERF_TYPE_SOFTWARE)
5244 return -ENOENT;
5245
5246 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5247 return -ENOENT;
5248
5249 return 0;
5250}
5251
5252static struct pmu perf_task_clock = {
5253 .task_ctx_nr = perf_sw_context,
5254
5255 .event_init = task_clock_event_init,
5256 .add = task_clock_event_add,
5257 .del = task_clock_event_del,
5258 .start = task_clock_event_start,
5259 .stop = task_clock_event_stop,
5260 .read = task_clock_event_read,
5261};
5262
5263static void perf_pmu_nop_void(struct pmu *pmu)
5264{
5265}
5266
5267static int perf_pmu_nop_int(struct pmu *pmu)
5268{
5269 return 0;
5270}
5271
5272static void perf_pmu_start_txn(struct pmu *pmu)
5273{
5274 perf_pmu_disable(pmu);
5275}
5276
5277static int perf_pmu_commit_txn(struct pmu *pmu)
5278{
5279 perf_pmu_enable(pmu);
5280 return 0;
5281}
5282
5283static void perf_pmu_cancel_txn(struct pmu *pmu)
5284{
5285 perf_pmu_enable(pmu);
5286}
5287
5288
5289
5290
5291
5292static void *find_pmu_context(int ctxn)
5293{
5294 struct pmu *pmu;
5295
5296 if (ctxn < 0)
5297 return NULL;
5298
5299 list_for_each_entry(pmu, &pmus, entry) {
5300 if (pmu->task_ctx_nr == ctxn)
5301 return pmu->pmu_cpu_context;
5302 }
5303
5304 return NULL;
5305}
5306
5307static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5308{
5309 int cpu;
5310
5311 for_each_possible_cpu(cpu) {
5312 struct perf_cpu_context *cpuctx;
5313
5314 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5315
5316 if (cpuctx->active_pmu == old_pmu)
5317 cpuctx->active_pmu = pmu;
5318 }
5319}
5320
5321static void free_pmu_context(struct pmu *pmu)
5322{
5323 struct pmu *i;
5324
5325 mutex_lock(&pmus_lock);
5326
5327
5328
5329 list_for_each_entry(i, &pmus, entry) {
5330 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5331 update_pmu_context(i, pmu);
5332 goto out;
5333 }
5334 }
5335
5336 free_percpu(pmu->pmu_cpu_context);
5337out:
5338 mutex_unlock(&pmus_lock);
5339}
5340static struct idr pmu_idr;
5341
5342static ssize_t
5343type_show(struct device *dev, struct device_attribute *attr, char *page)
5344{
5345 struct pmu *pmu = dev_get_drvdata(dev);
5346
5347 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5348}
5349
5350static struct device_attribute pmu_dev_attrs[] = {
5351 __ATTR_RO(type),
5352 __ATTR_NULL,
5353};
5354
5355static int pmu_bus_running;
5356static struct bus_type pmu_bus = {
5357 .name = "event_source",
5358 .dev_attrs = pmu_dev_attrs,
5359};
5360
5361static void pmu_dev_release(struct device *dev)
5362{
5363 kfree(dev);
5364}
5365
5366static int pmu_dev_alloc(struct pmu *pmu)
5367{
5368 int ret = -ENOMEM;
5369
5370 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5371 if (!pmu->dev)
5372 goto out;
5373
5374 device_initialize(pmu->dev);
5375 ret = dev_set_name(pmu->dev, "%s", pmu->name);
5376 if (ret)
5377 goto free_dev;
5378
5379 dev_set_drvdata(pmu->dev, pmu);
5380 pmu->dev->bus = &pmu_bus;
5381 pmu->dev->release = pmu_dev_release;
5382 ret = device_add(pmu->dev);
5383 if (ret)
5384 goto free_dev;
5385
5386out:
5387 return ret;
5388
5389free_dev:
5390 put_device(pmu->dev);
5391 goto out;
5392}
5393
5394static struct lock_class_key cpuctx_mutex;
5395
5396int perf_pmu_register(struct pmu *pmu, char *name, int type)
5397{
5398 int cpu, ret;
5399
5400 mutex_lock(&pmus_lock);
5401 ret = -ENOMEM;
5402 pmu->pmu_disable_count = alloc_percpu(int);
5403 if (!pmu->pmu_disable_count)
5404 goto unlock;
5405
5406 pmu->type = -1;
5407 if (!name)
5408 goto skip_type;
5409 pmu->name = name;
5410
5411 if (type < 0) {
5412 int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5413 if (!err)
5414 goto free_pdc;
5415
5416 err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5417 if (err) {
5418 ret = err;
5419 goto free_pdc;
5420 }
5421 }
5422 pmu->type = type;
5423
5424 if (pmu_bus_running) {
5425 ret = pmu_dev_alloc(pmu);
5426 if (ret)
5427 goto free_idr;
5428 }
5429
5430skip_type:
5431 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5432 if (pmu->pmu_cpu_context)
5433 goto got_cpu_context;
5434
5435 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
5436 if (!pmu->pmu_cpu_context)
5437 goto free_dev;
5438
5439 for_each_possible_cpu(cpu) {
5440 struct perf_cpu_context *cpuctx;
5441
5442 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5443 __perf_event_init_context(&cpuctx->ctx);
5444 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5445 cpuctx->ctx.type = cpu_context;
5446 cpuctx->ctx.pmu = pmu;
5447 cpuctx->jiffies_interval = 1;
5448 INIT_LIST_HEAD(&cpuctx->rotation_list);
5449 cpuctx->active_pmu = pmu;
5450 }
5451
5452got_cpu_context:
5453 if (!pmu->start_txn) {
5454 if (pmu->pmu_enable) {
5455
5456
5457
5458
5459
5460 pmu->start_txn = perf_pmu_start_txn;
5461 pmu->commit_txn = perf_pmu_commit_txn;
5462 pmu->cancel_txn = perf_pmu_cancel_txn;
5463 } else {
5464 pmu->start_txn = perf_pmu_nop_void;
5465 pmu->commit_txn = perf_pmu_nop_int;
5466 pmu->cancel_txn = perf_pmu_nop_void;
5467 }
5468 }
5469
5470 if (!pmu->pmu_enable) {
5471 pmu->pmu_enable = perf_pmu_nop_void;
5472 pmu->pmu_disable = perf_pmu_nop_void;
5473 }
5474
5475 list_add_rcu(&pmu->entry, &pmus);
5476 ret = 0;
5477unlock:
5478 mutex_unlock(&pmus_lock);
5479
5480 return ret;
5481
5482free_dev:
5483 device_del(pmu->dev);
5484 put_device(pmu->dev);
5485
5486free_idr:
5487 if (pmu->type >= PERF_TYPE_MAX)
5488 idr_remove(&pmu_idr, pmu->type);
5489
5490free_pdc:
5491 free_percpu(pmu->pmu_disable_count);
5492 goto unlock;
5493}
5494
5495void perf_pmu_unregister(struct pmu *pmu)
5496{
5497 mutex_lock(&pmus_lock);
5498 list_del_rcu(&pmu->entry);
5499 mutex_unlock(&pmus_lock);
5500
5501
5502
5503
5504
5505 synchronize_srcu(&pmus_srcu);
5506 synchronize_rcu();
5507
5508 free_percpu(pmu->pmu_disable_count);
5509 if (pmu->type >= PERF_TYPE_MAX)
5510 idr_remove(&pmu_idr, pmu->type);
5511 device_del(pmu->dev);
5512 put_device(pmu->dev);
5513 free_pmu_context(pmu);
5514}
5515
5516struct pmu *perf_init_event(struct perf_event *event)
5517{
5518 struct pmu *pmu = NULL;
5519 int idx;
5520
5521 idx = srcu_read_lock(&pmus_srcu);
5522
5523 rcu_read_lock();
5524 pmu = idr_find(&pmu_idr, event->attr.type);
5525 rcu_read_unlock();
5526 if (pmu)
5527 goto unlock;
5528
5529 list_for_each_entry_rcu(pmu, &pmus, entry) {
5530 int ret = pmu->event_init(event);
5531 if (!ret)
5532 goto unlock;
5533
5534 if (ret != -ENOENT) {
5535 pmu = ERR_PTR(ret);
5536 goto unlock;
5537 }
5538 }
5539 pmu = ERR_PTR(-ENOENT);
5540unlock:
5541 srcu_read_unlock(&pmus_srcu, idx);
5542
5543 return pmu;
5544}
5545
5546
5547
5548
5549static struct perf_event *
5550perf_event_alloc(struct perf_event_attr *attr, int cpu,
5551 struct task_struct *task,
5552 struct perf_event *group_leader,
5553 struct perf_event *parent_event,
5554 perf_overflow_handler_t overflow_handler)
5555{
5556 struct pmu *pmu;
5557 struct perf_event *event;
5558 struct hw_perf_event *hwc;
5559 long err;
5560
5561 if ((unsigned)cpu >= nr_cpu_ids) {
5562 if (!task || cpu != -1)
5563 return ERR_PTR(-EINVAL);
5564 }
5565
5566 event = kzalloc(sizeof(*event), GFP_KERNEL);
5567 if (!event)
5568 return ERR_PTR(-ENOMEM);
5569
5570
5571
5572
5573
5574 if (!group_leader)
5575 group_leader = event;
5576
5577 mutex_init(&event->child_mutex);
5578 INIT_LIST_HEAD(&event->child_list);
5579
5580 INIT_LIST_HEAD(&event->group_entry);
5581 INIT_LIST_HEAD(&event->event_entry);
5582 INIT_LIST_HEAD(&event->sibling_list);
5583 init_waitqueue_head(&event->waitq);
5584 init_irq_work(&event->pending, perf_pending_event);
5585
5586 mutex_init(&event->mmap_mutex);
5587
5588 event->cpu = cpu;
5589 event->attr = *attr;
5590 event->group_leader = group_leader;
5591 event->pmu = NULL;
5592 event->oncpu = -1;
5593
5594 event->parent = parent_event;
5595
5596 event->ns = get_pid_ns(current->nsproxy->pid_ns);
5597 event->id = atomic64_inc_return(&perf_event_id);
5598
5599 event->state = PERF_EVENT_STATE_INACTIVE;
5600
5601 if (task) {
5602 event->attach_state = PERF_ATTACH_TASK;
5603#ifdef CONFIG_HAVE_HW_BREAKPOINT
5604
5605
5606
5607 if (attr->type == PERF_TYPE_BREAKPOINT)
5608 event->hw.bp_target = task;
5609#endif
5610 }
5611
5612 if (!overflow_handler && parent_event)
5613 overflow_handler = parent_event->overflow_handler;
5614
5615 event->overflow_handler = overflow_handler;
5616
5617 if (attr->disabled)
5618 event->state = PERF_EVENT_STATE_OFF;
5619
5620 pmu = NULL;
5621
5622 hwc = &event->hw;
5623 hwc->sample_period = attr->sample_period;
5624 if (attr->freq && attr->sample_freq)
5625 hwc->sample_period = 1;
5626 hwc->last_period = hwc->sample_period;
5627
5628 local64_set(&hwc->period_left, hwc->sample_period);
5629
5630
5631
5632
5633 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5634 goto done;
5635
5636 pmu = perf_init_event(event);
5637
5638done:
5639 err = 0;
5640 if (!pmu)
5641 err = -EINVAL;
5642 else if (IS_ERR(pmu))
5643 err = PTR_ERR(pmu);
5644
5645 if (err) {
5646 if (event->ns)
5647 put_pid_ns(event->ns);
5648 kfree(event);
5649 return ERR_PTR(err);
5650 }
5651
5652 event->pmu = pmu;
5653
5654 if (!event->parent) {
5655 if (event->attach_state & PERF_ATTACH_TASK)
5656 jump_label_inc(&perf_task_events);
5657 if (event->attr.mmap || event->attr.mmap_data)
5658 atomic_inc(&nr_mmap_events);
5659 if (event->attr.comm)
5660 atomic_inc(&nr_comm_events);
5661 if (event->attr.task)
5662 atomic_inc(&nr_task_events);
5663 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
5664 err = get_callchain_buffers();
5665 if (err) {
5666 free_event(event);
5667 return ERR_PTR(err);
5668 }
5669 }
5670 }
5671
5672 return event;
5673}
5674
5675static int perf_copy_attr(struct perf_event_attr __user *uattr,
5676 struct perf_event_attr *attr)
5677{
5678 u32 size;
5679 int ret;
5680
5681 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
5682 return -EFAULT;
5683
5684
5685
5686
5687 memset(attr, 0, sizeof(*attr));
5688
5689 ret = get_user(size, &uattr->size);
5690 if (ret)
5691 return ret;
5692
5693 if (size > PAGE_SIZE)
5694 goto err_size;
5695
5696 if (!size)
5697 size = PERF_ATTR_SIZE_VER0;
5698
5699 if (size < PERF_ATTR_SIZE_VER0)
5700 goto err_size;
5701
5702
5703
5704
5705
5706
5707
5708 if (size > sizeof(*attr)) {
5709 unsigned char __user *addr;
5710 unsigned char __user *end;
5711 unsigned char val;
5712
5713 addr = (void __user *)uattr + sizeof(*attr);
5714 end = (void __user *)uattr + size;
5715
5716 for (; addr < end; addr++) {
5717 ret = get_user(val, addr);
5718 if (ret)
5719 return ret;
5720 if (val)
5721 goto err_size;
5722 }
5723 size = sizeof(*attr);
5724 }
5725
5726 ret = copy_from_user(attr, uattr, size);
5727 if (ret)
5728 return -EFAULT;
5729
5730
5731
5732
5733
5734 if (attr->type >= PERF_TYPE_MAX)
5735 return -EINVAL;
5736
5737 if (attr->__reserved_1)
5738 return -EINVAL;
5739
5740 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
5741 return -EINVAL;
5742
5743 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
5744 return -EINVAL;
5745
5746out:
5747 return ret;
5748
5749err_size:
5750 put_user(sizeof(*attr), &uattr->size);
5751 ret = -E2BIG;
5752 goto out;
5753}
5754
5755static int
5756perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
5757{
5758 struct perf_buffer *buffer = NULL, *old_buffer = NULL;
5759 int ret = -EINVAL;
5760
5761 if (!output_event)
5762 goto set;
5763
5764
5765 if (event == output_event)
5766 goto out;
5767
5768
5769
5770
5771 if (output_event->cpu != event->cpu)
5772 goto out;
5773
5774
5775
5776
5777 if (output_event->cpu == -1 && output_event->ctx != event->ctx)
5778 goto out;
5779
5780set:
5781 mutex_lock(&event->mmap_mutex);
5782
5783 if (atomic_read(&event->mmap_count))
5784 goto unlock;
5785
5786 if (output_event) {
5787
5788 buffer = perf_buffer_get(output_event);
5789 if (!buffer)
5790 goto unlock;
5791 }
5792
5793 old_buffer = event->buffer;
5794 rcu_assign_pointer(event->buffer, buffer);
5795 ret = 0;
5796unlock:
5797 mutex_unlock(&event->mmap_mutex);
5798
5799 if (old_buffer)
5800 perf_buffer_put(old_buffer);
5801out:
5802 return ret;
5803}
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813SYSCALL_DEFINE5(perf_event_open,
5814 struct perf_event_attr __user *, attr_uptr,
5815 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
5816{
5817 struct perf_event *group_leader = NULL, *output_event = NULL;
5818 struct perf_event *event, *sibling;
5819 struct perf_event_attr attr;
5820 struct perf_event_context *ctx;
5821 struct file *event_file = NULL;
5822 struct file *group_file = NULL;
5823 struct task_struct *task = NULL;
5824 struct pmu *pmu;
5825 int event_fd;
5826 int move_group = 0;
5827 int fput_needed = 0;
5828 int err;
5829
5830
5831 if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
5832 return -EINVAL;
5833
5834 err = perf_copy_attr(attr_uptr, &attr);
5835 if (err)
5836 return err;
5837
5838 if (!attr.exclude_kernel) {
5839 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
5840 return -EACCES;
5841 }
5842
5843 if (attr.freq) {
5844 if (attr.sample_freq > sysctl_perf_event_sample_rate)
5845 return -EINVAL;
5846 }
5847
5848 event_fd = get_unused_fd_flags(O_RDWR);
5849 if (event_fd < 0)
5850 return event_fd;
5851
5852 if (group_fd != -1) {
5853 group_leader = perf_fget_light(group_fd, &fput_needed);
5854 if (IS_ERR(group_leader)) {
5855 err = PTR_ERR(group_leader);
5856 goto err_fd;
5857 }
5858 group_file = group_leader->filp;
5859 if (flags & PERF_FLAG_FD_OUTPUT)
5860 output_event = group_leader;
5861 if (flags & PERF_FLAG_FD_NO_GROUP)
5862 group_leader = NULL;
5863 }
5864
5865 if (pid != -1) {
5866 task = find_lively_task_by_vpid(pid);
5867 if (IS_ERR(task)) {
5868 err = PTR_ERR(task);
5869 goto err_group_fd;
5870 }
5871 }
5872
5873 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
5874 if (IS_ERR(event)) {
5875 err = PTR_ERR(event);
5876 goto err_task;
5877 }
5878
5879
5880
5881
5882
5883 pmu = event->pmu;
5884
5885 if (group_leader &&
5886 (is_software_event(event) != is_software_event(group_leader))) {
5887 if (is_software_event(event)) {
5888
5889
5890
5891
5892
5893
5894
5895
5896 pmu = group_leader->pmu;
5897 } else if (is_software_event(group_leader) &&
5898 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
5899
5900
5901
5902
5903
5904 move_group = 1;
5905 }
5906 }
5907
5908
5909
5910
5911 ctx = find_get_context(pmu, task, cpu);
5912 if (IS_ERR(ctx)) {
5913 err = PTR_ERR(ctx);
5914 goto err_alloc;
5915 }
5916
5917
5918
5919
5920 if (group_leader) {
5921 err = -EINVAL;
5922
5923
5924
5925
5926
5927 if (group_leader->group_leader != group_leader)
5928 goto err_context;
5929
5930
5931
5932
5933 if (move_group) {
5934 if (group_leader->ctx->type != ctx->type)
5935 goto err_context;
5936 } else {
5937 if (group_leader->ctx != ctx)
5938 goto err_context;
5939 }
5940
5941
5942
5943
5944 if (attr.exclusive || attr.pinned)
5945 goto err_context;
5946 }
5947
5948 if (output_event) {
5949 err = perf_event_set_output(event, output_event);
5950 if (err)
5951 goto err_context;
5952 }
5953
5954 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
5955 if (IS_ERR(event_file)) {
5956 err = PTR_ERR(event_file);
5957 goto err_context;
5958 }
5959
5960 if (move_group) {
5961 struct perf_event_context *gctx = group_leader->ctx;
5962
5963 mutex_lock(&gctx->mutex);
5964 perf_event_remove_from_context(group_leader);
5965 list_for_each_entry(sibling, &group_leader->sibling_list,
5966 group_entry) {
5967 perf_event_remove_from_context(sibling);
5968 put_ctx(gctx);
5969 }
5970 mutex_unlock(&gctx->mutex);
5971 put_ctx(gctx);
5972 }
5973
5974 event->filp = event_file;
5975 WARN_ON_ONCE(ctx->parent_ctx);
5976 mutex_lock(&ctx->mutex);
5977
5978 if (move_group) {
5979 perf_install_in_context(ctx, group_leader, cpu);
5980 get_ctx(ctx);
5981 list_for_each_entry(sibling, &group_leader->sibling_list,
5982 group_entry) {
5983 perf_install_in_context(ctx, sibling, cpu);
5984 get_ctx(ctx);
5985 }
5986 }
5987
5988 perf_install_in_context(ctx, event, cpu);
5989 ++ctx->generation;
5990 mutex_unlock(&ctx->mutex);
5991
5992 event->owner = current;
5993
5994 mutex_lock(¤t->perf_event_mutex);
5995 list_add_tail(&event->owner_entry, ¤t->perf_event_list);
5996 mutex_unlock(¤t->perf_event_mutex);
5997
5998
5999
6000
6001 perf_event__header_size(event);
6002 perf_event__id_header_size(event);
6003
6004
6005
6006
6007
6008
6009
6010 fput_light(group_file, fput_needed);
6011 fd_install(event_fd, event_file);
6012 return event_fd;
6013
6014err_context:
6015 put_ctx(ctx);
6016err_alloc:
6017 free_event(event);
6018err_task:
6019 if (task)
6020 put_task_struct(task);
6021err_group_fd:
6022 fput_light(group_file, fput_needed);
6023err_fd:
6024 put_unused_fd(event_fd);
6025 return err;
6026}
6027
6028
6029
6030
6031
6032
6033
6034
6035struct perf_event *
6036perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6037 struct task_struct *task,
6038 perf_overflow_handler_t overflow_handler)
6039{
6040 struct perf_event_context *ctx;
6041 struct perf_event *event;
6042 int err;
6043
6044
6045
6046
6047
6048 event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
6049 if (IS_ERR(event)) {
6050 err = PTR_ERR(event);
6051 goto err;
6052 }
6053
6054 ctx = find_get_context(event->pmu, task, cpu);
6055 if (IS_ERR(ctx)) {
6056 err = PTR_ERR(ctx);
6057 goto err_free;
6058 }
6059
6060 event->filp = NULL;
6061 WARN_ON_ONCE(ctx->parent_ctx);
6062 mutex_lock(&ctx->mutex);
6063 perf_install_in_context(ctx, event, cpu);
6064 ++ctx->generation;
6065 mutex_unlock(&ctx->mutex);
6066
6067 return event;
6068
6069err_free:
6070 free_event(event);
6071err:
6072 return ERR_PTR(err);
6073}
6074EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6075
6076static void sync_child_event(struct perf_event *child_event,
6077 struct task_struct *child)
6078{
6079 struct perf_event *parent_event = child_event->parent;
6080 u64 child_val;
6081
6082 if (child_event->attr.inherit_stat)
6083 perf_event_read_event(child_event, child);
6084
6085 child_val = perf_event_count(child_event);
6086
6087
6088
6089
6090 atomic64_add(child_val, &parent_event->child_count);
6091 atomic64_add(child_event->total_time_enabled,
6092 &parent_event->child_total_time_enabled);
6093 atomic64_add(child_event->total_time_running,
6094 &parent_event->child_total_time_running);
6095
6096
6097
6098
6099 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6100 mutex_lock(&parent_event->child_mutex);
6101 list_del_init(&child_event->child_list);
6102 mutex_unlock(&parent_event->child_mutex);
6103
6104
6105
6106
6107
6108 fput(parent_event->filp);
6109}
6110
6111static void
6112__perf_event_exit_task(struct perf_event *child_event,
6113 struct perf_event_context *child_ctx,
6114 struct task_struct *child)
6115{
6116 struct perf_event *parent_event;
6117
6118 perf_event_remove_from_context(child_event);
6119
6120 parent_event = child_event->parent;
6121
6122
6123
6124
6125
6126 if (parent_event) {
6127 sync_child_event(child_event, child);
6128 free_event(child_event);
6129 }
6130}
6131
6132static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6133{
6134 struct perf_event *child_event, *tmp;
6135 struct perf_event_context *child_ctx;
6136 unsigned long flags;
6137
6138 if (likely(!child->perf_event_ctxp[ctxn])) {
6139 perf_event_task(child, NULL, 0);
6140 return;
6141 }
6142
6143 local_irq_save(flags);
6144
6145
6146
6147
6148
6149
6150 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6151 task_ctx_sched_out(child_ctx, EVENT_ALL);
6152
6153
6154
6155
6156
6157
6158 raw_spin_lock(&child_ctx->lock);
6159 child->perf_event_ctxp[ctxn] = NULL;
6160
6161
6162
6163
6164
6165 unclone_ctx(child_ctx);
6166 update_context_time(child_ctx);
6167 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6168
6169
6170
6171
6172
6173
6174 perf_event_task(child, child_ctx, 0);
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187 mutex_lock(&child_ctx->mutex);
6188
6189again:
6190 list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6191 group_entry)
6192 __perf_event_exit_task(child_event, child_ctx, child);
6193
6194 list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6195 group_entry)
6196 __perf_event_exit_task(child_event, child_ctx, child);
6197
6198
6199
6200
6201
6202
6203 if (!list_empty(&child_ctx->pinned_groups) ||
6204 !list_empty(&child_ctx->flexible_groups))
6205 goto again;
6206
6207 mutex_unlock(&child_ctx->mutex);
6208
6209 put_ctx(child_ctx);
6210}
6211
6212
6213
6214
6215void perf_event_exit_task(struct task_struct *child)
6216{
6217 struct perf_event *event, *tmp;
6218 int ctxn;
6219
6220 mutex_lock(&child->perf_event_mutex);
6221 list_for_each_entry_safe(event, tmp, &child->perf_event_list,
6222 owner_entry) {
6223 list_del_init(&event->owner_entry);
6224
6225
6226
6227
6228
6229
6230 smp_wmb();
6231 event->owner = NULL;
6232 }
6233 mutex_unlock(&child->perf_event_mutex);
6234
6235 for_each_task_context_nr(ctxn)
6236 perf_event_exit_task_context(child, ctxn);
6237}
6238
6239static void perf_free_event(struct perf_event *event,
6240 struct perf_event_context *ctx)
6241{
6242 struct perf_event *parent = event->parent;
6243
6244 if (WARN_ON_ONCE(!parent))
6245 return;
6246
6247 mutex_lock(&parent->child_mutex);
6248 list_del_init(&event->child_list);
6249 mutex_unlock(&parent->child_mutex);
6250
6251 fput(parent->filp);
6252
6253 perf_group_detach(event);
6254 list_del_event(event, ctx);
6255 free_event(event);
6256}
6257
6258
6259
6260
6261
6262void perf_event_free_task(struct task_struct *task)
6263{
6264 struct perf_event_context *ctx;
6265 struct perf_event *event, *tmp;
6266 int ctxn;
6267
6268 for_each_task_context_nr(ctxn) {
6269 ctx = task->perf_event_ctxp[ctxn];
6270 if (!ctx)
6271 continue;
6272
6273 mutex_lock(&ctx->mutex);
6274again:
6275 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6276 group_entry)
6277 perf_free_event(event, ctx);
6278
6279 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6280 group_entry)
6281 perf_free_event(event, ctx);
6282
6283 if (!list_empty(&ctx->pinned_groups) ||
6284 !list_empty(&ctx->flexible_groups))
6285 goto again;
6286
6287 mutex_unlock(&ctx->mutex);
6288
6289 put_ctx(ctx);
6290 }
6291}
6292
6293void perf_event_delayed_put(struct task_struct *task)
6294{
6295 int ctxn;
6296
6297 for_each_task_context_nr(ctxn)
6298 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6299}
6300
6301
6302
6303
6304static struct perf_event *
6305inherit_event(struct perf_event *parent_event,
6306 struct task_struct *parent,
6307 struct perf_event_context *parent_ctx,
6308 struct task_struct *child,
6309 struct perf_event *group_leader,
6310 struct perf_event_context *child_ctx)
6311{
6312 struct perf_event *child_event;
6313 unsigned long flags;
6314
6315
6316
6317
6318
6319
6320
6321 if (parent_event->parent)
6322 parent_event = parent_event->parent;
6323
6324 child_event = perf_event_alloc(&parent_event->attr,
6325 parent_event->cpu,
6326 child,
6327 group_leader, parent_event,
6328 NULL);
6329 if (IS_ERR(child_event))
6330 return child_event;
6331 get_ctx(child_ctx);
6332
6333
6334
6335
6336
6337
6338 if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
6339 child_event->state = PERF_EVENT_STATE_INACTIVE;
6340 else
6341 child_event->state = PERF_EVENT_STATE_OFF;
6342
6343 if (parent_event->attr.freq) {
6344 u64 sample_period = parent_event->hw.sample_period;
6345 struct hw_perf_event *hwc = &child_event->hw;
6346
6347 hwc->sample_period = sample_period;
6348 hwc->last_period = sample_period;
6349
6350 local64_set(&hwc->period_left, sample_period);
6351 }
6352
6353 child_event->ctx = child_ctx;
6354 child_event->overflow_handler = parent_event->overflow_handler;
6355
6356
6357
6358
6359 perf_event__header_size(child_event);
6360 perf_event__id_header_size(child_event);
6361
6362
6363
6364
6365 raw_spin_lock_irqsave(&child_ctx->lock, flags);
6366 add_event_to_ctx(child_event, child_ctx);
6367 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6368
6369
6370
6371
6372
6373
6374
6375 atomic_long_inc(&parent_event->filp->f_count);
6376
6377
6378
6379
6380 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6381 mutex_lock(&parent_event->child_mutex);
6382 list_add_tail(&child_event->child_list, &parent_event->child_list);
6383 mutex_unlock(&parent_event->child_mutex);
6384
6385 return child_event;
6386}
6387
6388static int inherit_group(struct perf_event *parent_event,
6389 struct task_struct *parent,
6390 struct perf_event_context *parent_ctx,
6391 struct task_struct *child,
6392 struct perf_event_context *child_ctx)
6393{
6394 struct perf_event *leader;
6395 struct perf_event *sub;
6396 struct perf_event *child_ctr;
6397
6398 leader = inherit_event(parent_event, parent, parent_ctx,
6399 child, NULL, child_ctx);
6400 if (IS_ERR(leader))
6401 return PTR_ERR(leader);
6402 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
6403 child_ctr = inherit_event(sub, parent, parent_ctx,
6404 child, leader, child_ctx);
6405 if (IS_ERR(child_ctr))
6406 return PTR_ERR(child_ctr);
6407 }
6408 return 0;
6409}
6410
6411static int
6412inherit_task_group(struct perf_event *event, struct task_struct *parent,
6413 struct perf_event_context *parent_ctx,
6414 struct task_struct *child, int ctxn,
6415 int *inherited_all)
6416{
6417 int ret;
6418 struct perf_event_context *child_ctx;
6419
6420 if (!event->attr.inherit) {
6421 *inherited_all = 0;
6422 return 0;
6423 }
6424
6425 child_ctx = child->perf_event_ctxp[ctxn];
6426 if (!child_ctx) {
6427
6428
6429
6430
6431
6432
6433
6434 child_ctx = alloc_perf_context(event->pmu, child);
6435 if (!child_ctx)
6436 return -ENOMEM;
6437
6438 child->perf_event_ctxp[ctxn] = child_ctx;
6439 }
6440
6441 ret = inherit_group(event, parent, parent_ctx,
6442 child, child_ctx);
6443
6444 if (ret)
6445 *inherited_all = 0;
6446
6447 return ret;
6448}
6449
6450
6451
6452
6453int perf_event_init_context(struct task_struct *child, int ctxn)
6454{
6455 struct perf_event_context *child_ctx, *parent_ctx;
6456 struct perf_event_context *cloned_ctx;
6457 struct perf_event *event;
6458 struct task_struct *parent = current;
6459 int inherited_all = 1;
6460 unsigned long flags;
6461 int ret = 0;
6462
6463 if (likely(!parent->perf_event_ctxp[ctxn]))
6464 return 0;
6465
6466
6467
6468
6469
6470 parent_ctx = perf_pin_task_context(parent, ctxn);
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483 mutex_lock(&parent_ctx->mutex);
6484
6485
6486
6487
6488
6489 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
6490 ret = inherit_task_group(event, parent, parent_ctx,
6491 child, ctxn, &inherited_all);
6492 if (ret)
6493 break;
6494 }
6495
6496
6497
6498
6499
6500
6501 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6502 parent_ctx->rotate_disable = 1;
6503 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6504
6505 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
6506 ret = inherit_task_group(event, parent, parent_ctx,
6507 child, ctxn, &inherited_all);
6508 if (ret)
6509 break;
6510 }
6511
6512 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
6513 parent_ctx->rotate_disable = 0;
6514
6515 child_ctx = child->perf_event_ctxp[ctxn];
6516
6517 if (child_ctx && inherited_all) {
6518
6519
6520
6521
6522
6523
6524
6525 cloned_ctx = parent_ctx->parent_ctx;
6526 if (cloned_ctx) {
6527 child_ctx->parent_ctx = cloned_ctx;
6528 child_ctx->parent_gen = parent_ctx->parent_gen;
6529 } else {
6530 child_ctx->parent_ctx = parent_ctx;
6531 child_ctx->parent_gen = parent_ctx->generation;
6532 }
6533 get_ctx(child_ctx->parent_ctx);
6534 }
6535
6536 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6537 mutex_unlock(&parent_ctx->mutex);
6538
6539 perf_unpin_context(parent_ctx);
6540
6541 return ret;
6542}
6543
6544
6545
6546
6547int perf_event_init_task(struct task_struct *child)
6548{
6549 int ctxn, ret;
6550
6551 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
6552 mutex_init(&child->perf_event_mutex);
6553 INIT_LIST_HEAD(&child->perf_event_list);
6554
6555 for_each_task_context_nr(ctxn) {
6556 ret = perf_event_init_context(child, ctxn);
6557 if (ret)
6558 return ret;
6559 }
6560
6561 return 0;
6562}
6563
6564static void __init perf_event_init_all_cpus(void)
6565{
6566 struct swevent_htable *swhash;
6567 int cpu;
6568
6569 for_each_possible_cpu(cpu) {
6570 swhash = &per_cpu(swevent_htable, cpu);
6571 mutex_init(&swhash->hlist_mutex);
6572 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
6573 }
6574}
6575
6576static void __cpuinit perf_event_init_cpu(int cpu)
6577{
6578 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6579
6580 mutex_lock(&swhash->hlist_mutex);
6581 if (swhash->hlist_refcount > 0) {
6582 struct swevent_hlist *hlist;
6583
6584 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
6585 WARN_ON(!hlist);
6586 rcu_assign_pointer(swhash->swevent_hlist, hlist);
6587 }
6588 mutex_unlock(&swhash->hlist_mutex);
6589}
6590
6591#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
6592static void perf_pmu_rotate_stop(struct pmu *pmu)
6593{
6594 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
6595
6596 WARN_ON(!irqs_disabled());
6597
6598 list_del_init(&cpuctx->rotation_list);
6599}
6600
6601static void __perf_event_exit_context(void *__info)
6602{
6603 struct perf_event_context *ctx = __info;
6604 struct perf_event *event, *tmp;
6605
6606 perf_pmu_rotate_stop(ctx->pmu);
6607
6608 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
6609 __perf_event_remove_from_context(event);
6610 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
6611 __perf_event_remove_from_context(event);
6612}
6613
6614static void perf_event_exit_cpu_context(int cpu)
6615{
6616 struct perf_event_context *ctx;
6617 struct pmu *pmu;
6618 int idx;
6619
6620 idx = srcu_read_lock(&pmus_srcu);
6621 list_for_each_entry_rcu(pmu, &pmus, entry) {
6622 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
6623
6624 mutex_lock(&ctx->mutex);
6625 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
6626 mutex_unlock(&ctx->mutex);
6627 }
6628 srcu_read_unlock(&pmus_srcu, idx);
6629}
6630
6631static void perf_event_exit_cpu(int cpu)
6632{
6633 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6634
6635 mutex_lock(&swhash->hlist_mutex);
6636 swevent_hlist_release(swhash);
6637 mutex_unlock(&swhash->hlist_mutex);
6638
6639 perf_event_exit_cpu_context(cpu);
6640}
6641#else
6642static inline void perf_event_exit_cpu(int cpu) { }
6643#endif
6644
6645static int
6646perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
6647{
6648 int cpu;
6649
6650 for_each_online_cpu(cpu)
6651 perf_event_exit_cpu(cpu);
6652
6653 return NOTIFY_OK;
6654}
6655
6656
6657
6658
6659
6660static struct notifier_block perf_reboot_notifier = {
6661 .notifier_call = perf_reboot,
6662 .priority = INT_MIN,
6663};
6664
6665static int __cpuinit
6666perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
6667{
6668 unsigned int cpu = (long)hcpu;
6669
6670 switch (action & ~CPU_TASKS_FROZEN) {
6671
6672 case CPU_UP_PREPARE:
6673 case CPU_DOWN_FAILED:
6674 perf_event_init_cpu(cpu);
6675 break;
6676
6677 case CPU_UP_CANCELED:
6678 case CPU_DOWN_PREPARE:
6679 perf_event_exit_cpu(cpu);
6680 break;
6681
6682 default:
6683 break;
6684 }
6685
6686 return NOTIFY_OK;
6687}
6688
6689void __init perf_event_init(void)
6690{
6691 int ret;
6692
6693 idr_init(&pmu_idr);
6694
6695 perf_event_init_all_cpus();
6696 init_srcu_struct(&pmus_srcu);
6697 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
6698 perf_pmu_register(&perf_cpu_clock, NULL, -1);
6699 perf_pmu_register(&perf_task_clock, NULL, -1);
6700 perf_tp_register();
6701 perf_cpu_notifier(perf_cpu_notify);
6702 register_reboot_notifier(&perf_reboot_notifier);
6703
6704 ret = init_hw_breakpoint();
6705 WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
6706}
6707
6708static int __init perf_event_sysfs_init(void)
6709{
6710 struct pmu *pmu;
6711 int ret;
6712
6713 mutex_lock(&pmus_lock);
6714
6715 ret = bus_register(&pmu_bus);
6716 if (ret)
6717 goto unlock;
6718
6719 list_for_each_entry(pmu, &pmus, entry) {
6720 if (!pmu->name || pmu->type < 0)
6721 continue;
6722
6723 ret = pmu_dev_alloc(pmu);
6724 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
6725 }
6726 pmu_bus_running = 1;
6727 ret = 0;
6728
6729unlock:
6730 mutex_unlock(&pmus_lock);
6731
6732 return ret;
6733}
6734device_initcall(perf_event_sysfs_init);
6735