1
2
3
4
5
6
7
8
9#include <linux/perf_event.h>
10#include <linux/init.h>
11#include <linux/export.h>
12#include <linux/pci.h>
13#include <linux/ptrace.h>
14#include <linux/syscore_ops.h>
15#include <linux/sched/clock.h>
16
17#include <asm/apic.h>
18
19#include "../perf_event.h"
20
21static u32 ibs_caps;
22
23#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
24
25#include <linux/kprobes.h>
26#include <linux/hardirq.h>
27
28#include <asm/nmi.h>
29#include <asm/amd-ibs.h>
30
31#define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT)
32#define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69enum ibs_states {
70 IBS_ENABLED = 0,
71 IBS_STARTED = 1,
72 IBS_STOPPING = 2,
73 IBS_STOPPED = 3,
74
75 IBS_MAX_STATES,
76};
77
78struct cpu_perf_ibs {
79 struct perf_event *event;
80 unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)];
81};
82
83struct perf_ibs {
84 struct pmu pmu;
85 unsigned int msr;
86 u64 config_mask;
87 u64 cnt_mask;
88 u64 enable_mask;
89 u64 valid_mask;
90 u64 max_period;
91 unsigned long offset_mask[1];
92 int offset_max;
93 unsigned int fetch_count_reset_broken : 1;
94 unsigned int fetch_ignore_if_zero_rip : 1;
95 struct cpu_perf_ibs __percpu *pcpu;
96
97 struct attribute **format_attrs;
98 struct attribute_group format_group;
99 const struct attribute_group *attr_groups[2];
100
101 u64 (*get_count)(u64 config);
102};
103
104static int
105perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period)
106{
107 s64 left = local64_read(&hwc->period_left);
108 s64 period = hwc->sample_period;
109 int overflow = 0;
110
111
112
113
114 if (unlikely(left <= -period)) {
115 left = period;
116 local64_set(&hwc->period_left, left);
117 hwc->last_period = period;
118 overflow = 1;
119 }
120
121 if (unlikely(left < (s64)min)) {
122 left += period;
123 local64_set(&hwc->period_left, left);
124 hwc->last_period = period;
125 overflow = 1;
126 }
127
128
129
130
131
132
133
134 if (left > max) {
135 left -= max;
136 if (left > max)
137 left = max;
138 else if (left < min)
139 left = min;
140 }
141
142 *hw_period = (u64)left;
143
144 return overflow;
145}
146
147static int
148perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width)
149{
150 struct hw_perf_event *hwc = &event->hw;
151 int shift = 64 - width;
152 u64 prev_raw_count;
153 u64 delta;
154
155
156
157
158
159
160
161
162 prev_raw_count = local64_read(&hwc->prev_count);
163 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
164 new_raw_count) != prev_raw_count)
165 return 0;
166
167
168
169
170
171
172
173
174
175 delta = (new_raw_count << shift) - (prev_raw_count << shift);
176 delta >>= shift;
177
178 local64_add(delta, &event->count);
179 local64_sub(delta, &hwc->period_left);
180
181 return 1;
182}
183
184static struct perf_ibs perf_ibs_fetch;
185static struct perf_ibs perf_ibs_op;
186
187static struct perf_ibs *get_ibs_pmu(int type)
188{
189 if (perf_ibs_fetch.pmu.type == type)
190 return &perf_ibs_fetch;
191 if (perf_ibs_op.pmu.type == type)
192 return &perf_ibs_op;
193 return NULL;
194}
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213static int perf_ibs_precise_event(struct perf_event *event, u64 *config)
214{
215 switch (event->attr.precise_ip) {
216 case 0:
217 return -ENOENT;
218 case 1:
219 case 2:
220 break;
221 default:
222 return -EOPNOTSUPP;
223 }
224
225 switch (event->attr.type) {
226 case PERF_TYPE_HARDWARE:
227 switch (event->attr.config) {
228 case PERF_COUNT_HW_CPU_CYCLES:
229 *config = 0;
230 return 0;
231 }
232 break;
233 case PERF_TYPE_RAW:
234 switch (event->attr.config) {
235 case 0x0076:
236 *config = 0;
237 return 0;
238 case 0x00C1:
239 *config = IBS_OP_CNT_CTL;
240 return 0;
241 }
242 break;
243 default:
244 return -ENOENT;
245 }
246
247 return -EOPNOTSUPP;
248}
249
250static int perf_ibs_init(struct perf_event *event)
251{
252 struct hw_perf_event *hwc = &event->hw;
253 struct perf_ibs *perf_ibs;
254 u64 max_cnt, config;
255 int ret;
256
257 perf_ibs = get_ibs_pmu(event->attr.type);
258 if (perf_ibs) {
259 config = event->attr.config;
260 } else {
261 perf_ibs = &perf_ibs_op;
262 ret = perf_ibs_precise_event(event, &config);
263 if (ret)
264 return ret;
265 }
266
267 if (event->pmu != &perf_ibs->pmu)
268 return -ENOENT;
269
270 if (config & ~perf_ibs->config_mask)
271 return -EINVAL;
272
273 if (hwc->sample_period) {
274 if (config & perf_ibs->cnt_mask)
275
276 return -EINVAL;
277 if (!event->attr.sample_freq && hwc->sample_period & 0x0f)
278
279
280
281
282
283 return -EINVAL;
284 hwc->sample_period &= ~0x0FULL;
285 if (!hwc->sample_period)
286 hwc->sample_period = 0x10;
287 } else {
288 max_cnt = config & perf_ibs->cnt_mask;
289 config &= ~perf_ibs->cnt_mask;
290 event->attr.sample_period = max_cnt << 4;
291 hwc->sample_period = event->attr.sample_period;
292 }
293
294 if (!hwc->sample_period)
295 return -EINVAL;
296
297
298
299
300
301 hwc->last_period = hwc->sample_period;
302 local64_set(&hwc->period_left, hwc->sample_period);
303
304 hwc->config_base = perf_ibs->msr;
305 hwc->config = config;
306
307 return 0;
308}
309
310static int perf_ibs_set_period(struct perf_ibs *perf_ibs,
311 struct hw_perf_event *hwc, u64 *period)
312{
313 int overflow;
314
315
316 overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period);
317 local64_set(&hwc->prev_count, 0);
318
319 return overflow;
320}
321
322static u64 get_ibs_fetch_count(u64 config)
323{
324 union ibs_fetch_ctl fetch_ctl = (union ibs_fetch_ctl)config;
325
326 return fetch_ctl.fetch_cnt << 4;
327}
328
329static u64 get_ibs_op_count(u64 config)
330{
331 union ibs_op_ctl op_ctl = (union ibs_op_ctl)config;
332 u64 count = 0;
333
334
335
336
337
338
339 if (op_ctl.op_val) {
340 count = op_ctl.opmaxcnt << 4;
341 if (ibs_caps & IBS_CAPS_OPCNTEXT)
342 count += op_ctl.opmaxcnt_ext << 20;
343 } else if (ibs_caps & IBS_CAPS_RDWROPCNT) {
344 count = op_ctl.opcurcnt;
345 }
346
347 return count;
348}
349
350static void
351perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event,
352 u64 *config)
353{
354 u64 count = perf_ibs->get_count(*config);
355
356
357
358
359
360
361 while (!perf_event_try_update(event, count, 64)) {
362 rdmsrl(event->hw.config_base, *config);
363 count = perf_ibs->get_count(*config);
364 }
365}
366
367static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
368 struct hw_perf_event *hwc, u64 config)
369{
370 u64 tmp = hwc->config | config;
371
372 if (perf_ibs->fetch_count_reset_broken)
373 wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask);
374
375 wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask);
376}
377
378
379
380
381
382
383
384
385static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,
386 struct hw_perf_event *hwc, u64 config)
387{
388 config &= ~perf_ibs->cnt_mask;
389 if (boot_cpu_data.x86 == 0x10)
390 wrmsrl(hwc->config_base, config);
391 config &= ~perf_ibs->enable_mask;
392 wrmsrl(hwc->config_base, config);
393}
394
395
396
397
398
399
400
401static void perf_ibs_start(struct perf_event *event, int flags)
402{
403 struct hw_perf_event *hwc = &event->hw;
404 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
405 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
406 u64 period, config = 0;
407
408 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
409 return;
410
411 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
412 hwc->state = 0;
413
414 perf_ibs_set_period(perf_ibs, hwc, &period);
415 if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) {
416 config |= period & IBS_OP_MAX_CNT_EXT_MASK;
417 period &= ~IBS_OP_MAX_CNT_EXT_MASK;
418 }
419 config |= period >> 4;
420
421
422
423
424
425 set_bit(IBS_STARTED, pcpu->state);
426 clear_bit(IBS_STOPPING, pcpu->state);
427 perf_ibs_enable_event(perf_ibs, hwc, config);
428
429 perf_event_update_userpage(event);
430}
431
432static void perf_ibs_stop(struct perf_event *event, int flags)
433{
434 struct hw_perf_event *hwc = &event->hw;
435 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
436 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
437 u64 config;
438 int stopping;
439
440 if (test_and_set_bit(IBS_STOPPING, pcpu->state))
441 return;
442
443 stopping = test_bit(IBS_STARTED, pcpu->state);
444
445 if (!stopping && (hwc->state & PERF_HES_UPTODATE))
446 return;
447
448 rdmsrl(hwc->config_base, config);
449
450 if (stopping) {
451
452
453
454
455
456
457 set_bit(IBS_STOPPED, pcpu->state);
458 perf_ibs_disable_event(perf_ibs, hwc, config);
459
460
461
462
463
464
465
466
467
468 clear_bit(IBS_STARTED, pcpu->state);
469 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
470 hwc->state |= PERF_HES_STOPPED;
471 }
472
473 if (hwc->state & PERF_HES_UPTODATE)
474 return;
475
476
477
478
479
480 config &= ~perf_ibs->valid_mask;
481
482 perf_ibs_event_update(perf_ibs, event, &config);
483 hwc->state |= PERF_HES_UPTODATE;
484}
485
486static int perf_ibs_add(struct perf_event *event, int flags)
487{
488 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
489 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
490
491 if (test_and_set_bit(IBS_ENABLED, pcpu->state))
492 return -ENOSPC;
493
494 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
495
496 pcpu->event = event;
497
498 if (flags & PERF_EF_START)
499 perf_ibs_start(event, PERF_EF_RELOAD);
500
501 return 0;
502}
503
504static void perf_ibs_del(struct perf_event *event, int flags)
505{
506 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
507 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
508
509 if (!test_and_clear_bit(IBS_ENABLED, pcpu->state))
510 return;
511
512 perf_ibs_stop(event, PERF_EF_UPDATE);
513
514 pcpu->event = NULL;
515
516 perf_event_update_userpage(event);
517}
518
519static void perf_ibs_read(struct perf_event *event) { }
520
521PMU_FORMAT_ATTR(rand_en, "config:57");
522PMU_FORMAT_ATTR(cnt_ctl, "config:19");
523
524static struct attribute *ibs_fetch_format_attrs[] = {
525 &format_attr_rand_en.attr,
526 NULL,
527};
528
529static struct attribute *ibs_op_format_attrs[] = {
530 NULL,
531 NULL,
532};
533
534static struct perf_ibs perf_ibs_fetch = {
535 .pmu = {
536 .task_ctx_nr = perf_invalid_context,
537
538 .event_init = perf_ibs_init,
539 .add = perf_ibs_add,
540 .del = perf_ibs_del,
541 .start = perf_ibs_start,
542 .stop = perf_ibs_stop,
543 .read = perf_ibs_read,
544 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
545 },
546 .msr = MSR_AMD64_IBSFETCHCTL,
547 .config_mask = IBS_FETCH_CONFIG_MASK,
548 .cnt_mask = IBS_FETCH_MAX_CNT,
549 .enable_mask = IBS_FETCH_ENABLE,
550 .valid_mask = IBS_FETCH_VAL,
551 .max_period = IBS_FETCH_MAX_CNT << 4,
552 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK },
553 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT,
554 .format_attrs = ibs_fetch_format_attrs,
555
556 .get_count = get_ibs_fetch_count,
557};
558
559static struct perf_ibs perf_ibs_op = {
560 .pmu = {
561 .task_ctx_nr = perf_invalid_context,
562
563 .event_init = perf_ibs_init,
564 .add = perf_ibs_add,
565 .del = perf_ibs_del,
566 .start = perf_ibs_start,
567 .stop = perf_ibs_stop,
568 .read = perf_ibs_read,
569 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
570 },
571 .msr = MSR_AMD64_IBSOPCTL,
572 .config_mask = IBS_OP_CONFIG_MASK,
573 .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT |
574 IBS_OP_CUR_CNT_RAND,
575 .enable_mask = IBS_OP_ENABLE,
576 .valid_mask = IBS_OP_VAL,
577 .max_period = IBS_OP_MAX_CNT << 4,
578 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK },
579 .offset_max = MSR_AMD64_IBSOP_REG_COUNT,
580 .format_attrs = ibs_op_format_attrs,
581
582 .get_count = get_ibs_op_count,
583};
584
585static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
586{
587 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
588 struct perf_event *event = pcpu->event;
589 struct hw_perf_event *hwc;
590 struct perf_sample_data data;
591 struct perf_raw_record raw;
592 struct pt_regs regs;
593 struct perf_ibs_data ibs_data;
594 int offset, size, check_rip, offset_max, throttle = 0;
595 unsigned int msr;
596 u64 *buf, *config, period, new_config = 0;
597
598 if (!test_bit(IBS_STARTED, pcpu->state)) {
599fail:
600
601
602
603
604
605
606 if (test_and_clear_bit(IBS_STOPPED, pcpu->state))
607 return 1;
608
609 return 0;
610 }
611
612 if (WARN_ON_ONCE(!event))
613 goto fail;
614
615 hwc = &event->hw;
616 msr = hwc->config_base;
617 buf = ibs_data.regs;
618 rdmsrl(msr, *buf);
619 if (!(*buf++ & perf_ibs->valid_mask))
620 goto fail;
621
622 config = &ibs_data.regs[0];
623 perf_ibs_event_update(perf_ibs, event, config);
624 perf_sample_data_init(&data, 0, hwc->last_period);
625 if (!perf_ibs_set_period(perf_ibs, hwc, &period))
626 goto out;
627
628 ibs_data.caps = ibs_caps;
629 size = 1;
630 offset = 1;
631 check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
632 if (event->attr.sample_type & PERF_SAMPLE_RAW)
633 offset_max = perf_ibs->offset_max;
634 else if (check_rip)
635 offset_max = 3;
636 else
637 offset_max = 1;
638 do {
639 rdmsrl(msr + offset, *buf++);
640 size++;
641 offset = find_next_bit(perf_ibs->offset_mask,
642 perf_ibs->offset_max,
643 offset + 1);
644 } while (offset < offset_max);
645
646
647
648
649
650 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
651 if (perf_ibs == &perf_ibs_op) {
652 if (ibs_caps & IBS_CAPS_BRNTRGT) {
653 rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++);
654 size++;
655 }
656 if (ibs_caps & IBS_CAPS_OPDATA4) {
657 rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++);
658 size++;
659 }
660 }
661 if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) {
662 rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++);
663 size++;
664 }
665 }
666 ibs_data.size = sizeof(u64) * size;
667
668 regs = *iregs;
669 if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) {
670 regs.flags &= ~PERF_EFLAGS_EXACT;
671 } else {
672
673 if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1]))
674 goto out;
675
676 set_linear_ip(®s, ibs_data.regs[1]);
677 regs.flags |= PERF_EFLAGS_EXACT;
678 }
679
680 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
681 raw = (struct perf_raw_record){
682 .frag = {
683 .size = sizeof(u32) + ibs_data.size,
684 .data = ibs_data.data,
685 },
686 };
687 data.raw = &raw;
688 }
689
690 throttle = perf_event_overflow(event, &data, ®s);
691out:
692 if (throttle) {
693 perf_ibs_stop(event, 0);
694 } else {
695 if (perf_ibs == &perf_ibs_op) {
696 if (ibs_caps & IBS_CAPS_OPCNTEXT) {
697 new_config = period & IBS_OP_MAX_CNT_EXT_MASK;
698 period &= ~IBS_OP_MAX_CNT_EXT_MASK;
699 }
700 if ((ibs_caps & IBS_CAPS_RDWROPCNT) && (*config & IBS_OP_CNT_CTL))
701 new_config |= *config & IBS_OP_CUR_CNT_RAND;
702 }
703 new_config |= period >> 4;
704
705 perf_ibs_enable_event(perf_ibs, hwc, new_config);
706 }
707
708 perf_event_update_userpage(event);
709
710 return 1;
711}
712
713static int
714perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
715{
716 u64 stamp = sched_clock();
717 int handled = 0;
718
719 handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs);
720 handled += perf_ibs_handle_irq(&perf_ibs_op, regs);
721
722 if (handled)
723 inc_irq_stat(apic_perf_irqs);
724
725 perf_sample_event_took(sched_clock() - stamp);
726
727 return handled;
728}
729NOKPROBE_SYMBOL(perf_ibs_nmi_handler);
730
731static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
732{
733 struct cpu_perf_ibs __percpu *pcpu;
734 int ret;
735
736 pcpu = alloc_percpu(struct cpu_perf_ibs);
737 if (!pcpu)
738 return -ENOMEM;
739
740 perf_ibs->pcpu = pcpu;
741
742
743 if (perf_ibs->format_attrs[0]) {
744 memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group));
745 perf_ibs->format_group.name = "format";
746 perf_ibs->format_group.attrs = perf_ibs->format_attrs;
747
748 memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups));
749 perf_ibs->attr_groups[0] = &perf_ibs->format_group;
750 perf_ibs->pmu.attr_groups = perf_ibs->attr_groups;
751 }
752
753 ret = perf_pmu_register(&perf_ibs->pmu, name, -1);
754 if (ret) {
755 perf_ibs->pcpu = NULL;
756 free_percpu(pcpu);
757 }
758
759 return ret;
760}
761
762static __init void perf_event_ibs_init(void)
763{
764 struct attribute **attr = ibs_op_format_attrs;
765
766
767
768
769
770 if (boot_cpu_data.x86 >= 0x16 && boot_cpu_data.x86 <= 0x18)
771 perf_ibs_fetch.fetch_count_reset_broken = 1;
772
773 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model < 0x10)
774 perf_ibs_fetch.fetch_ignore_if_zero_rip = 1;
775
776 perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
777
778 if (ibs_caps & IBS_CAPS_OPCNT) {
779 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
780 *attr++ = &format_attr_cnt_ctl.attr;
781 }
782
783 if (ibs_caps & IBS_CAPS_OPCNTEXT) {
784 perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK;
785 perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK;
786 perf_ibs_op.cnt_mask |= IBS_OP_MAX_CNT_EXT_MASK;
787 }
788
789 perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
790
791 register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
792 pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps);
793}
794
795#else
796
797static __init void perf_event_ibs_init(void) { }
798
799#endif
800
801
802
803static __init u32 __get_ibs_caps(void)
804{
805 u32 caps;
806 unsigned int max_level;
807
808 if (!boot_cpu_has(X86_FEATURE_IBS))
809 return 0;
810
811
812 max_level = cpuid_eax(0x80000000);
813 if (max_level < IBS_CPUID_FEATURES)
814 return IBS_CAPS_DEFAULT;
815
816 caps = cpuid_eax(IBS_CPUID_FEATURES);
817 if (!(caps & IBS_CAPS_AVAIL))
818
819 return IBS_CAPS_DEFAULT;
820
821 return caps;
822}
823
824u32 get_ibs_caps(void)
825{
826 return ibs_caps;
827}
828
829EXPORT_SYMBOL(get_ibs_caps);
830
831static inline int get_eilvt(int offset)
832{
833 return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
834}
835
836static inline int put_eilvt(int offset)
837{
838 return !setup_APIC_eilvt(offset, 0, 0, 1);
839}
840
841
842
843
844static inline int ibs_eilvt_valid(void)
845{
846 int offset;
847 u64 val;
848 int valid = 0;
849
850 preempt_disable();
851
852 rdmsrl(MSR_AMD64_IBSCTL, val);
853 offset = val & IBSCTL_LVT_OFFSET_MASK;
854
855 if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
856 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
857 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
858 goto out;
859 }
860
861 if (!get_eilvt(offset)) {
862 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
863 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
864 goto out;
865 }
866
867 valid = 1;
868out:
869 preempt_enable();
870
871 return valid;
872}
873
874static int setup_ibs_ctl(int ibs_eilvt_off)
875{
876 struct pci_dev *cpu_cfg;
877 int nodes;
878 u32 value = 0;
879
880 nodes = 0;
881 cpu_cfg = NULL;
882 do {
883 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
884 PCI_DEVICE_ID_AMD_10H_NB_MISC,
885 cpu_cfg);
886 if (!cpu_cfg)
887 break;
888 ++nodes;
889 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
890 | IBSCTL_LVT_OFFSET_VALID);
891 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
892 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
893 pci_dev_put(cpu_cfg);
894 pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n",
895 value);
896 return -EINVAL;
897 }
898 } while (1);
899
900 if (!nodes) {
901 pr_debug("No CPU node configured for IBS\n");
902 return -ENODEV;
903 }
904
905 return 0;
906}
907
908
909
910
911
912
913
914
915
916static void force_ibs_eilvt_setup(void)
917{
918 int offset;
919 int ret;
920
921 preempt_disable();
922
923 for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) {
924 if (get_eilvt(offset))
925 break;
926 }
927 preempt_enable();
928
929 if (offset == APIC_EILVT_NR_MAX) {
930 pr_debug("No EILVT entry available\n");
931 return;
932 }
933
934 ret = setup_ibs_ctl(offset);
935 if (ret)
936 goto out;
937
938 if (!ibs_eilvt_valid())
939 goto out;
940
941 pr_info("LVT offset %d assigned\n", offset);
942
943 return;
944out:
945 preempt_disable();
946 put_eilvt(offset);
947 preempt_enable();
948 return;
949}
950
951static void ibs_eilvt_setup(void)
952{
953
954
955
956
957
958
959 if (boot_cpu_data.x86 == 0x10)
960 force_ibs_eilvt_setup();
961}
962
963static inline int get_ibs_lvt_offset(void)
964{
965 u64 val;
966
967 rdmsrl(MSR_AMD64_IBSCTL, val);
968 if (!(val & IBSCTL_LVT_OFFSET_VALID))
969 return -EINVAL;
970
971 return val & IBSCTL_LVT_OFFSET_MASK;
972}
973
974static void setup_APIC_ibs(void)
975{
976 int offset;
977
978 offset = get_ibs_lvt_offset();
979 if (offset < 0)
980 goto failed;
981
982 if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
983 return;
984failed:
985 pr_warn("perf: IBS APIC setup failed on cpu #%d\n",
986 smp_processor_id());
987}
988
989static void clear_APIC_ibs(void)
990{
991 int offset;
992
993 offset = get_ibs_lvt_offset();
994 if (offset >= 0)
995 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
996}
997
998static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu)
999{
1000 setup_APIC_ibs();
1001 return 0;
1002}
1003
1004#ifdef CONFIG_PM
1005
1006static int perf_ibs_suspend(void)
1007{
1008 clear_APIC_ibs();
1009 return 0;
1010}
1011
1012static void perf_ibs_resume(void)
1013{
1014 ibs_eilvt_setup();
1015 setup_APIC_ibs();
1016}
1017
1018static struct syscore_ops perf_ibs_syscore_ops = {
1019 .resume = perf_ibs_resume,
1020 .suspend = perf_ibs_suspend,
1021};
1022
1023static void perf_ibs_pm_init(void)
1024{
1025 register_syscore_ops(&perf_ibs_syscore_ops);
1026}
1027
1028#else
1029
1030static inline void perf_ibs_pm_init(void) { }
1031
1032#endif
1033
1034static int x86_pmu_amd_ibs_dying_cpu(unsigned int cpu)
1035{
1036 clear_APIC_ibs();
1037 return 0;
1038}
1039
1040static __init int amd_ibs_init(void)
1041{
1042 u32 caps;
1043
1044 caps = __get_ibs_caps();
1045 if (!caps)
1046 return -ENODEV;
1047
1048 ibs_eilvt_setup();
1049
1050 if (!ibs_eilvt_valid())
1051 return -EINVAL;
1052
1053 perf_ibs_pm_init();
1054
1055 ibs_caps = caps;
1056
1057 smp_mb();
1058
1059
1060
1061
1062 cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_IBS_STARTING,
1063 "perf/x86/amd/ibs:starting",
1064 x86_pmu_amd_ibs_starting_cpu,
1065 x86_pmu_amd_ibs_dying_cpu);
1066
1067 perf_event_ibs_init();
1068
1069 return 0;
1070}
1071
1072
1073device_initcall(amd_ibs_init);
1074