1
2
3
4
5
6
7
8#define KMSG_COMPONENT "cpum_sf"
9#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/kernel_stat.h>
13#include <linux/perf_event.h>
14#include <linux/percpu.h>
15#include <linux/pid.h>
16#include <linux/notifier.h>
17#include <linux/export.h>
18#include <linux/slab.h>
19#include <linux/mm.h>
20#include <linux/moduleparam.h>
21#include <asm/cpu_mf.h>
22#include <asm/irq.h>
23#include <asm/debug.h>
24#include <asm/timex.h>
25
26
27
28
29
30#define CPUM_SF_MIN_SDBT 1
31
32
33
34
35
36#define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
37
38
39
40
41
42#define CPUM_SF_SDBT_TL_OFFSET (CPUM_SF_SDB_PER_TABLE * 8)
43static inline int require_table_link(const void *sdbt)
44{
45 return ((unsigned long) sdbt & ~PAGE_MASK) == CPUM_SF_SDBT_TL_OFFSET;
46}
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66static unsigned long __read_mostly CPUM_SF_MIN_SDB = 15;
67static unsigned long __read_mostly CPUM_SF_MAX_SDB = 8176;
68static unsigned long __read_mostly CPUM_SF_SDB_DIAG_FACTOR = 1;
69
70struct sf_buffer {
71 unsigned long *sdbt;
72
73 unsigned long num_sdb;
74 unsigned long num_sdbt;
75 unsigned long *tail;
76};
77
78struct aux_buffer {
79 struct sf_buffer sfb;
80 unsigned long head;
81 unsigned long alert_mark;
82 unsigned long empty_mark;
83 unsigned long *sdb_index;
84 unsigned long *sdbt_index;
85};
86
87struct cpu_hw_sf {
88
89 struct hws_qsi_info_block qsi;
90
91 struct hws_lsctl_request_block lsctl;
92 struct sf_buffer sfb;
93 unsigned int flags;
94 struct perf_event *event;
95 struct perf_output_handle handle;
96};
97static DEFINE_PER_CPU(struct cpu_hw_sf, cpu_hw_sf);
98
99
100static debug_info_t *sfdbg;
101
102
103
104
105static int sf_disable(void)
106{
107 struct hws_lsctl_request_block sreq;
108
109 memset(&sreq, 0, sizeof(sreq));
110 return lsctl(&sreq);
111}
112
113
114
115
116static int sf_buffer_available(struct cpu_hw_sf *cpuhw)
117{
118 return !!cpuhw->sfb.sdbt;
119}
120
121
122
123
124static void free_sampling_buffer(struct sf_buffer *sfb)
125{
126 unsigned long *sdbt, *curr;
127
128 if (!sfb->sdbt)
129 return;
130
131 sdbt = sfb->sdbt;
132 curr = sdbt;
133
134
135 while (1) {
136 if (!*curr || !sdbt)
137 break;
138
139
140 if (is_link_entry(curr)) {
141 curr = get_next_sdbt(curr);
142 if (sdbt)
143 free_page((unsigned long) sdbt);
144
145
146 if (curr == sfb->sdbt)
147 break;
148 else
149 sdbt = curr;
150 } else {
151
152 if (*curr) {
153 free_page(*curr);
154 curr++;
155 }
156 }
157 }
158
159 debug_sprintf_event(sfdbg, 5,
160 "free_sampling_buffer: freed sdbt=%p\n", sfb->sdbt);
161 memset(sfb, 0, sizeof(*sfb));
162}
163
164static int alloc_sample_data_block(unsigned long *sdbt, gfp_t gfp_flags)
165{
166 unsigned long sdb, *trailer;
167
168
169 sdb = get_zeroed_page(gfp_flags);
170 if (!sdb)
171 return -ENOMEM;
172 trailer = trailer_entry_ptr(sdb);
173 *trailer = SDB_TE_ALERT_REQ_MASK;
174
175
176 *sdbt = sdb;
177
178 return 0;
179}
180
181
182
183
184
185
186
187
188
189
190
191
192static int realloc_sampling_buffer(struct sf_buffer *sfb,
193 unsigned long num_sdb, gfp_t gfp_flags)
194{
195 int i, rc;
196 unsigned long *new, *tail;
197
198 if (!sfb->sdbt || !sfb->tail)
199 return -EINVAL;
200
201 if (!is_link_entry(sfb->tail))
202 return -EINVAL;
203
204
205
206
207
208
209 tail = sfb->tail;
210
211
212
213
214 if (sfb->sdbt != get_next_sdbt(tail)) {
215 debug_sprintf_event(sfdbg, 3, "realloc_sampling_buffer: "
216 "sampling buffer is not linked: origin=%p"
217 "tail=%p\n",
218 (void *) sfb->sdbt, (void *) tail);
219 return -EINVAL;
220 }
221
222
223 rc = 0;
224 for (i = 0; i < num_sdb; i++) {
225
226 if (require_table_link(tail)) {
227 new = (unsigned long *) get_zeroed_page(gfp_flags);
228 if (!new) {
229 rc = -ENOMEM;
230 break;
231 }
232 sfb->num_sdbt++;
233
234 *tail = (unsigned long)(void *) new + 1;
235 tail = new;
236 }
237
238
239
240
241
242
243 rc = alloc_sample_data_block(tail, gfp_flags);
244 if (rc)
245 break;
246 sfb->num_sdb++;
247 tail++;
248 }
249
250
251 *tail = (unsigned long) sfb->sdbt + 1;
252 sfb->tail = tail;
253
254 debug_sprintf_event(sfdbg, 4, "realloc_sampling_buffer: new buffer"
255 " settings: sdbt=%lu sdb=%lu\n",
256 sfb->num_sdbt, sfb->num_sdb);
257 return rc;
258}
259
260
261
262
263
264
265
266
267
268
269
270
271static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb)
272{
273 int rc;
274
275 if (sfb->sdbt)
276 return -EINVAL;
277
278
279 sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL);
280 if (!sfb->sdbt)
281 return -ENOMEM;
282 sfb->num_sdb = 0;
283 sfb->num_sdbt = 1;
284
285
286
287
288 sfb->tail = sfb->sdbt;
289 *sfb->tail = (unsigned long)(void *) sfb->sdbt + 1;
290
291
292 rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL);
293 if (rc) {
294 free_sampling_buffer(sfb);
295 debug_sprintf_event(sfdbg, 4, "alloc_sampling_buffer: "
296 "realloc_sampling_buffer failed with rc=%i\n", rc);
297 } else
298 debug_sprintf_event(sfdbg, 4,
299 "alloc_sampling_buffer: tear=%p dear=%p\n",
300 sfb->sdbt, (void *) *sfb->sdbt);
301 return rc;
302}
303
304static void sfb_set_limits(unsigned long min, unsigned long max)
305{
306 struct hws_qsi_info_block si;
307
308 CPUM_SF_MIN_SDB = min;
309 CPUM_SF_MAX_SDB = max;
310
311 memset(&si, 0, sizeof(si));
312 if (!qsi(&si))
313 CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes);
314}
315
316static unsigned long sfb_max_limit(struct hw_perf_event *hwc)
317{
318 return SAMPL_DIAG_MODE(hwc) ? CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR
319 : CPUM_SF_MAX_SDB;
320}
321
322static unsigned long sfb_pending_allocs(struct sf_buffer *sfb,
323 struct hw_perf_event *hwc)
324{
325 if (!sfb->sdbt)
326 return SFB_ALLOC_REG(hwc);
327 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)
328 return SFB_ALLOC_REG(hwc) - sfb->num_sdb;
329 return 0;
330}
331
332static int sfb_has_pending_allocs(struct sf_buffer *sfb,
333 struct hw_perf_event *hwc)
334{
335 return sfb_pending_allocs(sfb, hwc) > 0;
336}
337
338static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc)
339{
340
341 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc));
342 if (num)
343 SFB_ALLOC_REG(hwc) += num;
344}
345
346static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc)
347{
348 SFB_ALLOC_REG(hwc) = 0;
349 sfb_account_allocs(num, hwc);
350}
351
352static void deallocate_buffers(struct cpu_hw_sf *cpuhw)
353{
354 if (cpuhw->sfb.sdbt)
355 free_sampling_buffer(&cpuhw->sfb);
356}
357
358static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)
359{
360 unsigned long n_sdb, freq, factor;
361 size_t sample_size;
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387 sample_size = sizeof(struct hws_basic_entry);
388 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc));
389 factor = 1;
390 n_sdb = DIV_ROUND_UP(freq, factor * ((PAGE_SIZE-64) / sample_size));
391 if (n_sdb < CPUM_SF_MIN_SDB)
392 n_sdb = CPUM_SF_MIN_SDB;
393
394
395
396
397
398
399
400
401
402 sfb_init_allocs(n_sdb, hwc);
403 if (sf_buffer_available(cpuhw))
404 return 0;
405
406 debug_sprintf_event(sfdbg, 3,
407 "allocate_buffers: rate=%lu f=%lu sdb=%lu/%lu"
408 " sample_size=%lu cpuhw=%p\n",
409 SAMPL_RATE(hwc), freq, n_sdb, sfb_max_limit(hwc),
410 sample_size, cpuhw);
411
412 return alloc_sampling_buffer(&cpuhw->sfb,
413 sfb_pending_allocs(&cpuhw->sfb, hwc));
414}
415
416static unsigned long min_percent(unsigned int percent, unsigned long base,
417 unsigned long min)
418{
419 return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100));
420}
421
422static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base)
423{
424
425
426
427
428
429 if (ratio <= 5)
430 return 0;
431 if (ratio <= 25)
432 return min_percent(1, base, 1);
433 if (ratio <= 50)
434 return min_percent(1, base, 1);
435 if (ratio <= 75)
436 return min_percent(2, base, 2);
437 if (ratio <= 100)
438 return min_percent(3, base, 3);
439 if (ratio <= 250)
440 return min_percent(4, base, 4);
441
442 return min_percent(5, base, 8);
443}
444
445static void sfb_account_overflows(struct cpu_hw_sf *cpuhw,
446 struct hw_perf_event *hwc)
447{
448 unsigned long ratio, num;
449
450 if (!OVERFLOW_REG(hwc))
451 return;
452
453
454
455
456
457
458
459
460 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,
461 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));
462
463
464 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);
465 if (num)
466 sfb_account_allocs(num, hwc);
467
468 debug_sprintf_event(sfdbg, 5, "sfb: overflow: overflow=%llu ratio=%lu"
469 " num=%lu\n", OVERFLOW_REG(hwc), ratio, num);
470 OVERFLOW_REG(hwc) = 0;
471}
472
473
474
475
476
477
478
479
480
481
482
483
484static void extend_sampling_buffer(struct sf_buffer *sfb,
485 struct hw_perf_event *hwc)
486{
487 unsigned long num, num_old;
488 int rc;
489
490 num = sfb_pending_allocs(sfb, hwc);
491 if (!num)
492 return;
493 num_old = sfb->num_sdb;
494
495
496
497
498 sf_disable();
499
500
501
502
503
504
505 rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC);
506 if (rc)
507 debug_sprintf_event(sfdbg, 5, "sfb: extend: realloc "
508 "failed with rc=%i\n", rc);
509
510 if (sfb_has_pending_allocs(sfb, hwc))
511 debug_sprintf_event(sfdbg, 5, "sfb: extend: "
512 "req=%lu alloc=%lu remaining=%lu\n",
513 num, sfb->num_sdb - num_old,
514 sfb_pending_allocs(sfb, hwc));
515}
516
517
518
519static atomic_t num_events;
520
521static DEFINE_MUTEX(pmc_reserve_mutex);
522
523#define PMC_INIT 0
524#define PMC_RELEASE 1
525#define PMC_FAILURE 2
526static void setup_pmc_cpu(void *flags)
527{
528 int err;
529 struct cpu_hw_sf *cpusf = this_cpu_ptr(&cpu_hw_sf);
530
531 err = 0;
532 switch (*((int *) flags)) {
533 case PMC_INIT:
534 memset(cpusf, 0, sizeof(*cpusf));
535 err = qsi(&cpusf->qsi);
536 if (err)
537 break;
538 cpusf->flags |= PMU_F_RESERVED;
539 err = sf_disable();
540 if (err)
541 pr_err("Switching off the sampling facility failed "
542 "with rc=%i\n", err);
543 debug_sprintf_event(sfdbg, 5,
544 "setup_pmc_cpu: initialized: cpuhw=%p\n", cpusf);
545 break;
546 case PMC_RELEASE:
547 cpusf->flags &= ~PMU_F_RESERVED;
548 err = sf_disable();
549 if (err) {
550 pr_err("Switching off the sampling facility failed "
551 "with rc=%i\n", err);
552 } else
553 deallocate_buffers(cpusf);
554 debug_sprintf_event(sfdbg, 5,
555 "setup_pmc_cpu: released: cpuhw=%p\n", cpusf);
556 break;
557 }
558 if (err)
559 *((int *) flags) |= PMC_FAILURE;
560}
561
562static void release_pmc_hardware(void)
563{
564 int flags = PMC_RELEASE;
565
566 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
567 on_each_cpu(setup_pmc_cpu, &flags, 1);
568}
569
570static int reserve_pmc_hardware(void)
571{
572 int flags = PMC_INIT;
573
574 on_each_cpu(setup_pmc_cpu, &flags, 1);
575 if (flags & PMC_FAILURE) {
576 release_pmc_hardware();
577 return -ENODEV;
578 }
579 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
580
581 return 0;
582}
583
584static void hw_perf_event_destroy(struct perf_event *event)
585{
586
587 if (!atomic_add_unless(&num_events, -1, 1)) {
588 mutex_lock(&pmc_reserve_mutex);
589 if (atomic_dec_return(&num_events) == 0)
590 release_pmc_hardware();
591 mutex_unlock(&pmc_reserve_mutex);
592 }
593}
594
595static void hw_init_period(struct hw_perf_event *hwc, u64 period)
596{
597 hwc->sample_period = period;
598 hwc->last_period = hwc->sample_period;
599 local64_set(&hwc->period_left, hwc->sample_period);
600}
601
602static void hw_reset_registers(struct hw_perf_event *hwc,
603 unsigned long *sdbt_origin)
604{
605
606 TEAR_REG(hwc) = (unsigned long) sdbt_origin;
607}
608
609static unsigned long hw_limit_rate(const struct hws_qsi_info_block *si,
610 unsigned long rate)
611{
612 return clamp_t(unsigned long, rate,
613 si->min_sampl_rate, si->max_sampl_rate);
614}
615
616static u32 cpumsf_pid_type(struct perf_event *event,
617 u32 pid, enum pid_type type)
618{
619 struct task_struct *tsk;
620
621
622 if (!pid)
623 goto out;
624
625 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
626 pid = -1;
627 if (tsk) {
628
629
630
631
632 if (event->parent)
633 event = event->parent;
634 pid = __task_pid_nr_ns(tsk, type, event->ns);
635
636
637
638
639 if (!pid && !pid_alive(tsk))
640 pid = -1;
641 }
642out:
643 return pid;
644}
645
646static void cpumsf_output_event_pid(struct perf_event *event,
647 struct perf_sample_data *data,
648 struct pt_regs *regs)
649{
650 u32 pid;
651 struct perf_event_header header;
652 struct perf_output_handle handle;
653
654
655
656
657
658 pid = data->tid_entry.pid;
659
660
661 rcu_read_lock();
662
663 perf_prepare_sample(&header, data, event, regs);
664 if (perf_output_begin(&handle, event, header.size))
665 goto out;
666
667
668 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID);
669 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID);
670
671 perf_output_sample(&handle, &header, data, event);
672 perf_output_end(&handle);
673out:
674 rcu_read_unlock();
675}
676
677static int __hw_perf_event_init(struct perf_event *event)
678{
679 struct cpu_hw_sf *cpuhw;
680 struct hws_qsi_info_block si;
681 struct perf_event_attr *attr = &event->attr;
682 struct hw_perf_event *hwc = &event->hw;
683 unsigned long rate;
684 int cpu, err;
685
686
687 err = 0;
688 if (!atomic_inc_not_zero(&num_events)) {
689 mutex_lock(&pmc_reserve_mutex);
690 if (atomic_read(&num_events) == 0 && reserve_pmc_hardware())
691 err = -EBUSY;
692 else
693 atomic_inc(&num_events);
694 mutex_unlock(&pmc_reserve_mutex);
695 }
696 event->destroy = hw_perf_event_destroy;
697
698 if (err)
699 goto out;
700
701
702
703
704
705
706
707
708
709
710 memset(&si, 0, sizeof(si));
711 cpuhw = NULL;
712 if (event->cpu == -1)
713 qsi(&si);
714 else {
715
716
717
718 cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
719 si = cpuhw->qsi;
720 }
721
722
723
724
725
726 if (!si.as) {
727 err = -ENOENT;
728 goto out;
729 }
730
731
732 SAMPL_FLAGS(hwc) = PERF_CPUM_SF_BASIC_MODE;
733
734
735
736
737 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) {
738 if (!si.ad) {
739 err = -EPERM;
740 goto out;
741 }
742 SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_DIAG_MODE;
743 }
744
745
746 if (attr->config1 & PERF_CPUM_SF_FULL_BLOCKS)
747 SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_FULL_BLOCKS;
748
749
750
751
752
753
754 rate = 0;
755 if (attr->freq) {
756 if (!attr->sample_freq) {
757 err = -EINVAL;
758 goto out;
759 }
760 rate = freq_to_sample_rate(&si, attr->sample_freq);
761 rate = hw_limit_rate(&si, rate);
762 attr->freq = 0;
763 attr->sample_period = rate;
764 } else {
765
766
767
768
769 rate = hw_limit_rate(&si, hwc->sample_period);
770
771
772
773
774
775
776
777 if (sample_rate_to_freq(&si, rate) >
778 sysctl_perf_event_sample_rate) {
779 err = -EINVAL;
780 debug_sprintf_event(sfdbg, 1, "Sampling rate exceeds maximum perf sample rate\n");
781 goto out;
782 }
783 }
784 SAMPL_RATE(hwc) = rate;
785 hw_init_period(hwc, SAMPL_RATE(hwc));
786
787
788 hwc->extra_reg.reg = REG_OVERFLOW;
789 OVERFLOW_REG(hwc) = 0;
790
791
792 if (attr->config == PERF_EVENT_CPUM_SF_DIAG)
793 return 0;
794
795
796
797
798
799
800 if (cpuhw)
801
802 err = allocate_buffers(cpuhw, hwc);
803 else {
804
805
806
807 for_each_online_cpu(cpu) {
808 cpuhw = &per_cpu(cpu_hw_sf, cpu);
809 err = allocate_buffers(cpuhw, hwc);
810 if (err)
811 break;
812 }
813 }
814
815
816
817
818
819 if (event->attr.sample_type & PERF_SAMPLE_TID)
820 if (is_default_overflow_handler(event))
821 event->overflow_handler = cpumsf_output_event_pid;
822out:
823 return err;
824}
825
826static int cpumsf_pmu_event_init(struct perf_event *event)
827{
828 int err;
829
830
831 if (has_branch_stack(event))
832 return -EOPNOTSUPP;
833
834 switch (event->attr.type) {
835 case PERF_TYPE_RAW:
836 if ((event->attr.config != PERF_EVENT_CPUM_SF) &&
837 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG))
838 return -ENOENT;
839 break;
840 case PERF_TYPE_HARDWARE:
841
842
843
844
845
846 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES)
847 return -ENOENT;
848 if (!is_sampling_event(event))
849 return -ENOENT;
850 break;
851 default:
852 return -ENOENT;
853 }
854
855
856 if (event->cpu >= 0 && !cpu_online(event->cpu))
857 return -ENODEV;
858
859
860
861
862 if (event->attr.exclude_hv)
863 event->attr.exclude_hv = 0;
864 if (event->attr.exclude_idle)
865 event->attr.exclude_idle = 0;
866
867 err = __hw_perf_event_init(event);
868 if (unlikely(err))
869 if (event->destroy)
870 event->destroy(event);
871 return err;
872}
873
874static void cpumsf_pmu_enable(struct pmu *pmu)
875{
876 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
877 struct hw_perf_event *hwc;
878 int err;
879
880 if (cpuhw->flags & PMU_F_ENABLED)
881 return;
882
883 if (cpuhw->flags & PMU_F_ERR_MASK)
884 return;
885
886
887
888
889
890
891
892
893
894
895
896
897 if (cpuhw->event) {
898 hwc = &cpuhw->event->hw;
899 if (!(SAMPL_DIAG_MODE(hwc))) {
900
901
902
903
904 sfb_account_overflows(cpuhw, hwc);
905 if (sfb_has_pending_allocs(&cpuhw->sfb, hwc))
906 extend_sampling_buffer(&cpuhw->sfb, hwc);
907 }
908 }
909
910
911 cpuhw->flags |= PMU_F_ENABLED;
912 barrier();
913
914 err = lsctl(&cpuhw->lsctl);
915 if (err) {
916 cpuhw->flags &= ~PMU_F_ENABLED;
917 pr_err("Loading sampling controls failed: op=%i err=%i\n",
918 1, err);
919 return;
920 }
921
922
923 lpp(&S390_lowcore.lpp);
924
925 debug_sprintf_event(sfdbg, 6, "pmu_enable: es=%i cs=%i ed=%i cd=%i "
926 "tear=%p dear=%p\n", cpuhw->lsctl.es, cpuhw->lsctl.cs,
927 cpuhw->lsctl.ed, cpuhw->lsctl.cd,
928 (void *) cpuhw->lsctl.tear, (void *) cpuhw->lsctl.dear);
929}
930
931static void cpumsf_pmu_disable(struct pmu *pmu)
932{
933 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
934 struct hws_lsctl_request_block inactive;
935 struct hws_qsi_info_block si;
936 int err;
937
938 if (!(cpuhw->flags & PMU_F_ENABLED))
939 return;
940
941 if (cpuhw->flags & PMU_F_ERR_MASK)
942 return;
943
944
945 inactive = cpuhw->lsctl;
946 inactive.cs = 0;
947 inactive.cd = 0;
948
949 err = lsctl(&inactive);
950 if (err) {
951 pr_err("Loading sampling controls failed: op=%i err=%i\n",
952 2, err);
953 return;
954 }
955
956
957 if (!qsi(&si)) {
958
959
960
961
962
963 if (si.es) {
964 cpuhw->lsctl.tear = si.tear;
965 cpuhw->lsctl.dear = si.dear;
966 }
967 } else
968 debug_sprintf_event(sfdbg, 3, "cpumsf_pmu_disable: "
969 "qsi() failed with err=%i\n", err);
970
971 cpuhw->flags &= ~PMU_F_ENABLED;
972}
973
974
975
976
977
978
979
980
981
982
983static int perf_exclude_event(struct perf_event *event, struct pt_regs *regs,
984 struct perf_sf_sde_regs *sde_regs)
985{
986 if (event->attr.exclude_user && user_mode(regs))
987 return 1;
988 if (event->attr.exclude_kernel && !user_mode(regs))
989 return 1;
990 if (event->attr.exclude_guest && sde_regs->in_guest)
991 return 1;
992 if (event->attr.exclude_host && !sde_regs->in_guest)
993 return 1;
994 return 0;
995}
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008static int perf_push_sample(struct perf_event *event,
1009 struct hws_basic_entry *basic)
1010{
1011 int overflow;
1012 struct pt_regs regs;
1013 struct perf_sf_sde_regs *sde_regs;
1014 struct perf_sample_data data;
1015
1016
1017 perf_sample_data_init(&data, 0, event->hw.last_period);
1018
1019
1020
1021
1022
1023
1024 memset(®s, 0, sizeof(regs));
1025 regs.int_code = 0x1407;
1026 regs.int_parm = CPU_MF_INT_SF_PRA;
1027 sde_regs = (struct perf_sf_sde_regs *) ®s.int_parm_long;
1028
1029 psw_bits(regs.psw).ia = basic->ia;
1030 psw_bits(regs.psw).dat = basic->T;
1031 psw_bits(regs.psw).wait = basic->W;
1032 psw_bits(regs.psw).pstate = basic->P;
1033 psw_bits(regs.psw).as = basic->AS;
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046 switch (basic->CL) {
1047 case 1:
1048 sde_regs->in_guest = 0;
1049 break;
1050 case 2:
1051 sde_regs->in_guest = 1;
1052 break;
1053 default:
1054 if (basic->gpp || basic->prim_asn != 0xffff)
1055 sde_regs->in_guest = 1;
1056 break;
1057 }
1058
1059
1060
1061
1062
1063 data.tid_entry.pid = basic->hpp & LPP_PID_MASK;
1064
1065 overflow = 0;
1066 if (perf_exclude_event(event, ®s, sde_regs))
1067 goto out;
1068 if (perf_event_overflow(event, &data, ®s)) {
1069 overflow = 1;
1070 event->pmu->stop(event, 0);
1071 }
1072 perf_event_update_userpage(event);
1073out:
1074 return overflow;
1075}
1076
1077static void perf_event_count_update(struct perf_event *event, u64 count)
1078{
1079 local64_add(count, &event->count);
1080}
1081
1082static void debug_sample_entry(struct hws_basic_entry *sample,
1083 struct hws_trailer_entry *te)
1084{
1085 debug_sprintf_event(sfdbg, 4, "hw_collect_samples: Found unknown "
1086 "sampling data entry: te->f=%i basic.def=%04x (%p)\n",
1087 te->f, sample->def, sample);
1088}
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt,
1111 unsigned long long *overflow)
1112{
1113 struct hws_trailer_entry *te;
1114 struct hws_basic_entry *sample;
1115
1116 te = (struct hws_trailer_entry *) trailer_entry_ptr(*sdbt);
1117 sample = (struct hws_basic_entry *) *sdbt;
1118 while ((unsigned long *) sample < (unsigned long *) te) {
1119
1120 if (!sample->def)
1121 break;
1122
1123
1124 perf_event_count_update(event, SAMPL_RATE(&event->hw));
1125
1126
1127 if (sample->def == 0x0001) {
1128
1129
1130
1131
1132 if (!*overflow) {
1133
1134 if (sample->I == 0 && sample->W == 0) {
1135
1136 *overflow = perf_push_sample(event,
1137 sample);
1138 }
1139 } else
1140
1141 *overflow += 1;
1142 } else {
1143 debug_sample_entry(sample, te);
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154 if (!te->f)
1155 break;
1156 }
1157
1158
1159 sample->def = 0;
1160 sample++;
1161 }
1162}
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178static void hw_perf_event_update(struct perf_event *event, int flush_all)
1179{
1180 struct hw_perf_event *hwc = &event->hw;
1181 struct hws_trailer_entry *te;
1182 unsigned long *sdbt;
1183 unsigned long long event_overflow, sampl_overflow, num_sdb, te_flags;
1184 int done;
1185
1186
1187
1188
1189
1190 if (SAMPL_DIAG_MODE(&event->hw))
1191 return;
1192
1193 if (flush_all && SDB_FULL_BLOCKS(hwc))
1194 flush_all = 0;
1195
1196 sdbt = (unsigned long *) TEAR_REG(hwc);
1197 done = event_overflow = sampl_overflow = num_sdb = 0;
1198 while (!done) {
1199
1200 te = (struct hws_trailer_entry *) trailer_entry_ptr(*sdbt);
1201
1202
1203 if (!te->f) {
1204 done = 1;
1205 if (!flush_all)
1206 break;
1207 }
1208
1209
1210 if (te->overflow)
1211
1212
1213
1214
1215 sampl_overflow += te->overflow;
1216
1217
1218 debug_sprintf_event(sfdbg, 6, "hw_perf_event_update: sdbt=%p "
1219 "overflow=%llu timestamp=0x%llx\n",
1220 sdbt, te->overflow,
1221 (te->f) ? trailer_timestamp(te) : 0ULL);
1222
1223
1224
1225
1226
1227 hw_collect_samples(event, sdbt, &event_overflow);
1228 num_sdb++;
1229
1230
1231 do {
1232 te_flags = te->flags & ~SDB_TE_BUFFER_FULL_MASK;
1233 te_flags |= SDB_TE_ALERT_REQ_MASK;
1234 } while (!cmpxchg_double(&te->flags, &te->overflow,
1235 te->flags, te->overflow,
1236 te_flags, 0ULL));
1237
1238
1239 sdbt++;
1240 if (is_link_entry(sdbt))
1241 sdbt = get_next_sdbt(sdbt);
1242
1243
1244 TEAR_REG(hwc) = (unsigned long) sdbt;
1245
1246
1247
1248
1249 if (flush_all && done)
1250 break;
1251
1252
1253
1254
1255 if (event_overflow)
1256 flush_all = 1;
1257 }
1258
1259
1260 if (sampl_overflow)
1261 OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) +
1262 sampl_overflow, 1 + num_sdb);
1263 if (sampl_overflow || event_overflow)
1264 debug_sprintf_event(sfdbg, 4, "hw_perf_event_update: "
1265 "overflow stats: sample=%llu event=%llu\n",
1266 sampl_overflow, event_overflow);
1267}
1268
1269#define AUX_SDB_INDEX(aux, i) ((i) % aux->sfb.num_sdb)
1270#define AUX_SDB_NUM(aux, start, end) (end >= start ? end - start + 1 : 0)
1271#define AUX_SDB_NUM_ALERT(aux) AUX_SDB_NUM(aux, aux->head, aux->alert_mark)
1272#define AUX_SDB_NUM_EMPTY(aux) AUX_SDB_NUM(aux, aux->head, aux->empty_mark)
1273
1274
1275
1276
1277static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,
1278 unsigned long index)
1279{
1280 unsigned long sdb;
1281
1282 index = AUX_SDB_INDEX(aux, index);
1283 sdb = aux->sdb_index[index];
1284 return (struct hws_trailer_entry *)trailer_entry_ptr(sdb);
1285}
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296static void aux_output_end(struct perf_output_handle *handle)
1297{
1298 unsigned long i, range_scan, idx;
1299 struct aux_buffer *aux;
1300 struct hws_trailer_entry *te;
1301
1302 aux = perf_get_aux(handle);
1303 if (!aux)
1304 return;
1305
1306 range_scan = AUX_SDB_NUM_ALERT(aux);
1307 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {
1308 te = aux_sdb_trailer(aux, idx);
1309 if (!(te->flags & SDB_TE_BUFFER_FULL_MASK))
1310 break;
1311 }
1312
1313 perf_aux_output_end(handle, i << PAGE_SHIFT);
1314
1315
1316 te = aux_sdb_trailer(aux, aux->alert_mark);
1317 te->flags &= ~SDB_TE_ALERT_REQ_MASK;
1318
1319 debug_sprintf_event(sfdbg, 6, "aux_output_end: collect %lx SDBs\n", i);
1320}
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331static int aux_output_begin(struct perf_output_handle *handle,
1332 struct aux_buffer *aux,
1333 struct cpu_hw_sf *cpuhw)
1334{
1335 unsigned long range;
1336 unsigned long i, range_scan, idx;
1337 unsigned long head, base, offset;
1338 struct hws_trailer_entry *te;
1339
1340 if (WARN_ON_ONCE(handle->head & ~PAGE_MASK))
1341 return -EINVAL;
1342
1343 aux->head = handle->head >> PAGE_SHIFT;
1344 range = (handle->size + 1) >> PAGE_SHIFT;
1345 if (range <= 1)
1346 return -ENOMEM;
1347
1348
1349
1350
1351
1352 if (range > AUX_SDB_NUM_EMPTY(aux)) {
1353 range_scan = range - AUX_SDB_NUM_EMPTY(aux);
1354 idx = aux->empty_mark + 1;
1355 for (i = 0; i < range_scan; i++, idx++) {
1356 te = aux_sdb_trailer(aux, idx);
1357 te->flags = te->flags & ~SDB_TE_BUFFER_FULL_MASK;
1358 te->flags = te->flags & ~SDB_TE_ALERT_REQ_MASK;
1359 te->overflow = 0;
1360 }
1361
1362 aux->empty_mark = aux->head + range - 1;
1363 }
1364
1365
1366 aux->alert_mark = aux->head + range/2 - 1;
1367 te = aux_sdb_trailer(aux, aux->alert_mark);
1368 te->flags = te->flags | SDB_TE_ALERT_REQ_MASK;
1369
1370
1371 head = AUX_SDB_INDEX(aux, aux->head);
1372 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];
1373 offset = head % CPUM_SF_SDB_PER_TABLE;
1374 cpuhw->lsctl.tear = base + offset * sizeof(unsigned long);
1375 cpuhw->lsctl.dear = aux->sdb_index[head];
1376
1377 debug_sprintf_event(sfdbg, 6, "aux_output_begin: "
1378 "head->alert_mark->empty_mark (num_alert, range)"
1379 "[%lx -> %lx -> %lx] (%lx, %lx) "
1380 "tear index %lx, tear %lx dear %lx\n",
1381 aux->head, aux->alert_mark, aux->empty_mark,
1382 AUX_SDB_NUM_ALERT(aux), range,
1383 head / CPUM_SF_SDB_PER_TABLE,
1384 cpuhw->lsctl.tear,
1385 cpuhw->lsctl.dear);
1386
1387 return 0;
1388}
1389
1390
1391
1392
1393
1394
1395
1396static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,
1397 unsigned long long *overflow)
1398{
1399 unsigned long long orig_overflow, orig_flags, new_flags;
1400 struct hws_trailer_entry *te;
1401
1402 te = aux_sdb_trailer(aux, alert_index);
1403 do {
1404 orig_flags = te->flags;
1405 orig_overflow = te->overflow;
1406 *overflow = orig_overflow;
1407 if (orig_flags & SDB_TE_BUFFER_FULL_MASK) {
1408
1409
1410
1411
1412
1413 return false;
1414 }
1415 new_flags = orig_flags | SDB_TE_ALERT_REQ_MASK;
1416 } while (!cmpxchg_double(&te->flags, &te->overflow,
1417 orig_flags, orig_overflow,
1418 new_flags, 0ULL));
1419 return true;
1420}
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,
1445 unsigned long long *overflow)
1446{
1447 unsigned long long orig_overflow, orig_flags, new_flags;
1448 unsigned long i, range_scan, idx;
1449 struct hws_trailer_entry *te;
1450
1451 if (range <= AUX_SDB_NUM_EMPTY(aux))
1452
1453
1454
1455
1456
1457 return aux_set_alert(aux, aux->alert_mark, overflow);
1458
1459 if (aux->alert_mark <= aux->empty_mark)
1460
1461
1462
1463
1464 if (!aux_set_alert(aux, aux->alert_mark, overflow))
1465 return false;
1466
1467
1468
1469
1470
1471
1472 range_scan = range - AUX_SDB_NUM_EMPTY(aux);
1473 idx = aux->empty_mark + 1;
1474 for (i = 0; i < range_scan; i++, idx++) {
1475 te = aux_sdb_trailer(aux, idx);
1476 do {
1477 orig_flags = te->flags;
1478 orig_overflow = te->overflow;
1479 new_flags = orig_flags & ~SDB_TE_BUFFER_FULL_MASK;
1480 if (idx == aux->alert_mark)
1481 new_flags |= SDB_TE_ALERT_REQ_MASK;
1482 else
1483 new_flags &= ~SDB_TE_ALERT_REQ_MASK;
1484 } while (!cmpxchg_double(&te->flags, &te->overflow,
1485 orig_flags, orig_overflow,
1486 new_flags, 0ULL));
1487 *overflow += orig_overflow;
1488 }
1489
1490
1491 aux->empty_mark = aux->head + range - 1;
1492
1493 return true;
1494}
1495
1496
1497
1498
1499static void hw_collect_aux(struct cpu_hw_sf *cpuhw)
1500{
1501 struct aux_buffer *aux;
1502 int done = 0;
1503 unsigned long range = 0, size;
1504 unsigned long long overflow = 0;
1505 struct perf_output_handle *handle = &cpuhw->handle;
1506 unsigned long num_sdb;
1507
1508 aux = perf_get_aux(handle);
1509 if (WARN_ON_ONCE(!aux))
1510 return;
1511
1512
1513 size = AUX_SDB_NUM_ALERT(aux) << PAGE_SHIFT;
1514 perf_aux_output_end(handle, size);
1515 num_sdb = aux->sfb.num_sdb;
1516
1517 while (!done) {
1518
1519 aux = perf_aux_output_begin(handle, cpuhw->event);
1520 if (handle->size == 0) {
1521 pr_err("The AUX buffer with %lu pages for the "
1522 "diagnostic-sampling mode is full\n",
1523 num_sdb);
1524 debug_sprintf_event(sfdbg, 1, "AUX buffer used up\n");
1525 break;
1526 }
1527 if (WARN_ON_ONCE(!aux))
1528 return;
1529
1530
1531 aux->head = handle->head >> PAGE_SHIFT;
1532 range = (handle->size + 1) >> PAGE_SHIFT;
1533 if (range == 1)
1534 aux->alert_mark = aux->head;
1535 else
1536 aux->alert_mark = aux->head + range/2 - 1;
1537
1538 if (aux_reset_buffer(aux, range, &overflow)) {
1539 if (!overflow) {
1540 done = 1;
1541 break;
1542 }
1543 size = range << PAGE_SHIFT;
1544 perf_aux_output_end(&cpuhw->handle, size);
1545 pr_err("Sample data caused the AUX buffer with %lu "
1546 "pages to overflow\n", num_sdb);
1547 debug_sprintf_event(sfdbg, 1, "head %lx range %lx "
1548 "overflow %llx\n",
1549 aux->head, range, overflow);
1550 } else {
1551 size = AUX_SDB_NUM_ALERT(aux) << PAGE_SHIFT;
1552 perf_aux_output_end(&cpuhw->handle, size);
1553 debug_sprintf_event(sfdbg, 6, "head %lx alert %lx "
1554 "already full, try another\n",
1555 aux->head, aux->alert_mark);
1556 }
1557 }
1558
1559 if (done)
1560 debug_sprintf_event(sfdbg, 6, "aux_reset_buffer: "
1561 "[%lx -> %lx -> %lx] (%lx, %lx)\n",
1562 aux->head, aux->alert_mark, aux->empty_mark,
1563 AUX_SDB_NUM_ALERT(aux), range);
1564}
1565
1566
1567
1568
1569static void aux_buffer_free(void *data)
1570{
1571 struct aux_buffer *aux = data;
1572 unsigned long i, num_sdbt;
1573
1574 if (!aux)
1575 return;
1576
1577
1578 num_sdbt = aux->sfb.num_sdbt;
1579 for (i = 0; i < num_sdbt; i++)
1580 free_page(aux->sdbt_index[i]);
1581
1582 kfree(aux->sdbt_index);
1583 kfree(aux->sdb_index);
1584 kfree(aux);
1585
1586 debug_sprintf_event(sfdbg, 4, "aux_buffer_free: free "
1587 "%lu SDBTs\n", num_sdbt);
1588}
1589
1590static void aux_sdb_init(unsigned long sdb)
1591{
1592 struct hws_trailer_entry *te;
1593
1594 te = (struct hws_trailer_entry *)trailer_entry_ptr(sdb);
1595
1596
1597 te->clock_base = 1;
1598 memcpy(&te->progusage2, &tod_clock_base[1], 8);
1599}
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615static void *aux_buffer_setup(struct perf_event *event, void **pages,
1616 int nr_pages, bool snapshot)
1617{
1618 struct sf_buffer *sfb;
1619 struct aux_buffer *aux;
1620 unsigned long *new, *tail;
1621 int i, n_sdbt;
1622
1623 if (!nr_pages || !pages)
1624 return NULL;
1625
1626 if (nr_pages > CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1627 pr_err("AUX buffer size (%i pages) is larger than the "
1628 "maximum sampling buffer limit\n",
1629 nr_pages);
1630 return NULL;
1631 } else if (nr_pages < CPUM_SF_MIN_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1632 pr_err("AUX buffer size (%i pages) is less than the "
1633 "minimum sampling buffer limit\n",
1634 nr_pages);
1635 return NULL;
1636 }
1637
1638
1639 aux = kmalloc(sizeof(struct aux_buffer), GFP_KERNEL);
1640 if (!aux)
1641 goto no_aux;
1642 sfb = &aux->sfb;
1643
1644
1645 n_sdbt = (nr_pages + CPUM_SF_SDB_PER_TABLE - 1) / CPUM_SF_SDB_PER_TABLE;
1646 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);
1647 if (!aux->sdbt_index)
1648 goto no_sdbt_index;
1649
1650
1651 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);
1652 if (!aux->sdb_index)
1653 goto no_sdb_index;
1654
1655
1656 sfb->num_sdbt = 0;
1657 sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL);
1658 if (!sfb->sdbt)
1659 goto no_sdbt;
1660 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;
1661 tail = sfb->tail = sfb->sdbt;
1662
1663
1664
1665
1666
1667 for (i = 0; i < nr_pages; i++, tail++) {
1668 if (require_table_link(tail)) {
1669 new = (unsigned long *) get_zeroed_page(GFP_KERNEL);
1670 if (!new)
1671 goto no_sdbt;
1672 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;
1673
1674 *tail = (unsigned long)(void *) new + 1;
1675 tail = new;
1676 }
1677
1678 *tail = (unsigned long)pages[i];
1679 aux->sdb_index[i] = (unsigned long)pages[i];
1680 aux_sdb_init((unsigned long)pages[i]);
1681 }
1682 sfb->num_sdb = nr_pages;
1683
1684
1685 *tail = (unsigned long) sfb->sdbt + 1;
1686 sfb->tail = tail;
1687
1688
1689
1690
1691
1692
1693 aux->empty_mark = sfb->num_sdb - 1;
1694
1695 debug_sprintf_event(sfdbg, 4, "aux_buffer_setup: setup %lu SDBTs"
1696 " and %lu SDBs\n",
1697 sfb->num_sdbt, sfb->num_sdb);
1698
1699 return aux;
1700
1701no_sdbt:
1702
1703 for (i = 0; i < sfb->num_sdbt; i++)
1704 free_page(aux->sdbt_index[i]);
1705 kfree(aux->sdb_index);
1706no_sdb_index:
1707 kfree(aux->sdbt_index);
1708no_sdbt_index:
1709 kfree(aux);
1710no_aux:
1711 return NULL;
1712}
1713
1714static void cpumsf_pmu_read(struct perf_event *event)
1715{
1716
1717}
1718
1719
1720
1721
1722static void cpumsf_pmu_start(struct perf_event *event, int flags)
1723{
1724 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1725
1726 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1727 return;
1728
1729 if (flags & PERF_EF_RELOAD)
1730 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1731
1732 perf_pmu_disable(event->pmu);
1733 event->hw.state = 0;
1734 cpuhw->lsctl.cs = 1;
1735 if (SAMPL_DIAG_MODE(&event->hw))
1736 cpuhw->lsctl.cd = 1;
1737 perf_pmu_enable(event->pmu);
1738}
1739
1740
1741
1742
1743static void cpumsf_pmu_stop(struct perf_event *event, int flags)
1744{
1745 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1746
1747 if (event->hw.state & PERF_HES_STOPPED)
1748 return;
1749
1750 perf_pmu_disable(event->pmu);
1751 cpuhw->lsctl.cs = 0;
1752 cpuhw->lsctl.cd = 0;
1753 event->hw.state |= PERF_HES_STOPPED;
1754
1755 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) {
1756 hw_perf_event_update(event, 1);
1757 event->hw.state |= PERF_HES_UPTODATE;
1758 }
1759 perf_pmu_enable(event->pmu);
1760}
1761
1762static int cpumsf_pmu_add(struct perf_event *event, int flags)
1763{
1764 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1765 struct aux_buffer *aux;
1766 int err;
1767
1768 if (cpuhw->flags & PMU_F_IN_USE)
1769 return -EAGAIN;
1770
1771 if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt)
1772 return -EINVAL;
1773
1774 err = 0;
1775 perf_pmu_disable(event->pmu);
1776
1777 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1778
1779
1780
1781
1782
1783
1784 cpuhw->lsctl.s = 0;
1785 cpuhw->lsctl.h = 1;
1786 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw);
1787 if (!SAMPL_DIAG_MODE(&event->hw)) {
1788 cpuhw->lsctl.tear = (unsigned long) cpuhw->sfb.sdbt;
1789 cpuhw->lsctl.dear = *(unsigned long *) cpuhw->sfb.sdbt;
1790 hw_reset_registers(&event->hw, cpuhw->sfb.sdbt);
1791 }
1792
1793
1794
1795 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) {
1796 err = -EAGAIN;
1797 goto out;
1798 }
1799 if (SAMPL_DIAG_MODE(&event->hw)) {
1800 aux = perf_aux_output_begin(&cpuhw->handle, event);
1801 if (!aux) {
1802 err = -EINVAL;
1803 goto out;
1804 }
1805 err = aux_output_begin(&cpuhw->handle, aux, cpuhw);
1806 if (err)
1807 goto out;
1808 cpuhw->lsctl.ed = 1;
1809 }
1810 cpuhw->lsctl.es = 1;
1811
1812
1813 cpuhw->event = event;
1814 cpuhw->flags |= PMU_F_IN_USE;
1815
1816 if (flags & PERF_EF_START)
1817 cpumsf_pmu_start(event, PERF_EF_RELOAD);
1818out:
1819 perf_event_update_userpage(event);
1820 perf_pmu_enable(event->pmu);
1821 return err;
1822}
1823
1824static void cpumsf_pmu_del(struct perf_event *event, int flags)
1825{
1826 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1827
1828 perf_pmu_disable(event->pmu);
1829 cpumsf_pmu_stop(event, PERF_EF_UPDATE);
1830
1831 cpuhw->lsctl.es = 0;
1832 cpuhw->lsctl.ed = 0;
1833 cpuhw->flags &= ~PMU_F_IN_USE;
1834 cpuhw->event = NULL;
1835
1836 if (SAMPL_DIAG_MODE(&event->hw))
1837 aux_output_end(&cpuhw->handle);
1838 perf_event_update_userpage(event);
1839 perf_pmu_enable(event->pmu);
1840}
1841
1842CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC, PERF_EVENT_CPUM_SF);
1843CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC_DIAG, PERF_EVENT_CPUM_SF_DIAG);
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861enum {
1862 SF_CYCLES_BASIC_ATTR_IDX = 0,
1863 SF_CYCLES_BASIC_DIAG_ATTR_IDX,
1864 SF_CYCLES_ATTR_MAX
1865};
1866
1867static struct attribute *cpumsf_pmu_events_attr[SF_CYCLES_ATTR_MAX + 1] = {
1868 [SF_CYCLES_BASIC_ATTR_IDX] = CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC)
1869};
1870
1871PMU_FORMAT_ATTR(event, "config:0-63");
1872
1873static struct attribute *cpumsf_pmu_format_attr[] = {
1874 &format_attr_event.attr,
1875 NULL,
1876};
1877
1878static struct attribute_group cpumsf_pmu_events_group = {
1879 .name = "events",
1880 .attrs = cpumsf_pmu_events_attr,
1881};
1882static struct attribute_group cpumsf_pmu_format_group = {
1883 .name = "format",
1884 .attrs = cpumsf_pmu_format_attr,
1885};
1886static const struct attribute_group *cpumsf_pmu_attr_groups[] = {
1887 &cpumsf_pmu_events_group,
1888 &cpumsf_pmu_format_group,
1889 NULL,
1890};
1891
1892static struct pmu cpumf_sampling = {
1893 .pmu_enable = cpumsf_pmu_enable,
1894 .pmu_disable = cpumsf_pmu_disable,
1895
1896 .event_init = cpumsf_pmu_event_init,
1897 .add = cpumsf_pmu_add,
1898 .del = cpumsf_pmu_del,
1899
1900 .start = cpumsf_pmu_start,
1901 .stop = cpumsf_pmu_stop,
1902 .read = cpumsf_pmu_read,
1903
1904 .attr_groups = cpumsf_pmu_attr_groups,
1905
1906 .setup_aux = aux_buffer_setup,
1907 .free_aux = aux_buffer_free,
1908};
1909
1910static void cpumf_measurement_alert(struct ext_code ext_code,
1911 unsigned int alert, unsigned long unused)
1912{
1913 struct cpu_hw_sf *cpuhw;
1914
1915 if (!(alert & CPU_MF_INT_SF_MASK))
1916 return;
1917 inc_irq_stat(IRQEXT_CMS);
1918 cpuhw = this_cpu_ptr(&cpu_hw_sf);
1919
1920
1921
1922 if (!(cpuhw->flags & PMU_F_RESERVED))
1923 return;
1924
1925
1926
1927
1928
1929 if (alert & CPU_MF_INT_SF_PRA) {
1930 if (cpuhw->flags & PMU_F_IN_USE)
1931 if (SAMPL_DIAG_MODE(&cpuhw->event->hw))
1932 hw_collect_aux(cpuhw);
1933 else
1934 hw_perf_event_update(cpuhw->event, 0);
1935 else
1936 WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE));
1937 }
1938
1939
1940 if (alert != CPU_MF_INT_SF_PRA)
1941 debug_sprintf_event(sfdbg, 6, "measurement alert: 0x%x\n", alert);
1942
1943
1944 if (alert & CPU_MF_INT_SF_SACA)
1945 qsi(&cpuhw->qsi);
1946
1947
1948 if (alert & CPU_MF_INT_SF_LSDA) {
1949 pr_err("Sample data was lost\n");
1950 cpuhw->flags |= PMU_F_ERR_LSDA;
1951 sf_disable();
1952 }
1953
1954
1955 if (alert & (CPU_MF_INT_SF_IAE|CPU_MF_INT_SF_ISE)) {
1956 pr_err("A sampling buffer entry is incorrect (alert=0x%x)\n",
1957 alert);
1958 cpuhw->flags |= PMU_F_ERR_IBE;
1959 sf_disable();
1960 }
1961}
1962static int cpusf_pmu_setup(unsigned int cpu, int flags)
1963{
1964
1965
1966
1967 if (!atomic_read(&num_events))
1968 return 0;
1969
1970 local_irq_disable();
1971 setup_pmc_cpu(&flags);
1972 local_irq_enable();
1973 return 0;
1974}
1975
1976static int s390_pmu_sf_online_cpu(unsigned int cpu)
1977{
1978 return cpusf_pmu_setup(cpu, PMC_INIT);
1979}
1980
1981static int s390_pmu_sf_offline_cpu(unsigned int cpu)
1982{
1983 return cpusf_pmu_setup(cpu, PMC_RELEASE);
1984}
1985
1986static int param_get_sfb_size(char *buffer, const struct kernel_param *kp)
1987{
1988 if (!cpum_sf_avail())
1989 return -ENODEV;
1990 return sprintf(buffer, "%lu,%lu", CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
1991}
1992
1993static int param_set_sfb_size(const char *val, const struct kernel_param *kp)
1994{
1995 int rc;
1996 unsigned long min, max;
1997
1998 if (!cpum_sf_avail())
1999 return -ENODEV;
2000 if (!val || !strlen(val))
2001 return -EINVAL;
2002
2003
2004 min = CPUM_SF_MIN_SDB;
2005 max = CPUM_SF_MAX_SDB;
2006 if (strchr(val, ','))
2007 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL;
2008 else
2009 rc = kstrtoul(val, 10, &max);
2010
2011 if (min < 2 || min >= max || max > get_num_physpages())
2012 rc = -EINVAL;
2013 if (rc)
2014 return rc;
2015
2016 sfb_set_limits(min, max);
2017 pr_info("The sampling buffer limits have changed to: "
2018 "min=%lu max=%lu (diag=x%lu)\n",
2019 CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB, CPUM_SF_SDB_DIAG_FACTOR);
2020 return 0;
2021}
2022
2023#define param_check_sfb_size(name, p) __param_check(name, p, void)
2024static const struct kernel_param_ops param_ops_sfb_size = {
2025 .set = param_set_sfb_size,
2026 .get = param_get_sfb_size,
2027};
2028
2029#define RS_INIT_FAILURE_QSI 0x0001
2030#define RS_INIT_FAILURE_BSDES 0x0002
2031#define RS_INIT_FAILURE_ALRT 0x0003
2032#define RS_INIT_FAILURE_PERF 0x0004
2033static void __init pr_cpumsf_err(unsigned int reason)
2034{
2035 pr_err("Sampling facility support for perf is not available: "
2036 "reason=%04x\n", reason);
2037}
2038
2039static int __init init_cpum_sampling_pmu(void)
2040{
2041 struct hws_qsi_info_block si;
2042 int err;
2043
2044 if (!cpum_sf_avail())
2045 return -ENODEV;
2046
2047 memset(&si, 0, sizeof(si));
2048 if (qsi(&si)) {
2049 pr_cpumsf_err(RS_INIT_FAILURE_QSI);
2050 return -ENODEV;
2051 }
2052
2053 if (!si.as && !si.ad)
2054 return -ENODEV;
2055
2056 if (si.bsdes != sizeof(struct hws_basic_entry)) {
2057 pr_cpumsf_err(RS_INIT_FAILURE_BSDES);
2058 return -EINVAL;
2059 }
2060
2061 if (si.ad) {
2062 sfb_set_limits(CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
2063
2064
2065
2066 cpumsf_pmu_events_attr[SF_CYCLES_BASIC_DIAG_ATTR_IDX] =
2067 CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC_DIAG);
2068 }
2069
2070 sfdbg = debug_register(KMSG_COMPONENT, 2, 1, 80);
2071 if (!sfdbg) {
2072 pr_err("Registering for s390dbf failed\n");
2073 return -ENOMEM;
2074 }
2075 debug_register_view(sfdbg, &debug_sprintf_view);
2076
2077 err = register_external_irq(EXT_IRQ_MEASURE_ALERT,
2078 cpumf_measurement_alert);
2079 if (err) {
2080 pr_cpumsf_err(RS_INIT_FAILURE_ALRT);
2081 debug_unregister(sfdbg);
2082 goto out;
2083 }
2084
2085 err = perf_pmu_register(&cpumf_sampling, "cpum_sf", PERF_TYPE_RAW);
2086 if (err) {
2087 pr_cpumsf_err(RS_INIT_FAILURE_PERF);
2088 unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
2089 cpumf_measurement_alert);
2090 debug_unregister(sfdbg);
2091 goto out;
2092 }
2093
2094 cpuhp_setup_state(CPUHP_AP_PERF_S390_SF_ONLINE, "perf/s390/sf:online",
2095 s390_pmu_sf_online_cpu, s390_pmu_sf_offline_cpu);
2096out:
2097 return err;
2098}
2099arch_initcall(init_cpum_sampling_pmu);
2100core_param(cpum_sfb_size, CPUM_SF_MAX_SDB, sfb_size, 0640);
2101