1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/perf_event.h>
14#include <linux/slab.h>
15#include <asm/opal.h>
16#include <asm/imc-pmu.h>
17#include <asm/cputhreads.h>
18#include <asm/smp.h>
19#include <linux/string.h>
20
21
22
23
24
25
26
27static DEFINE_MUTEX(nest_init_lock);
28static DEFINE_PER_CPU(struct imc_pmu_ref *, local_nest_imc_refc);
29static struct imc_pmu **per_nest_pmu_arr;
30static cpumask_t nest_imc_cpumask;
31static struct imc_pmu_ref *nest_imc_refc;
32static int nest_pmus;
33
34
35
36static cpumask_t core_imc_cpumask;
37static struct imc_pmu_ref *core_imc_refc;
38static struct imc_pmu *core_imc_pmu;
39
40
41
42static DEFINE_PER_CPU(u64 *, thread_imc_mem);
43static struct imc_pmu *thread_imc_pmu;
44static int thread_imc_mem_size;
45
46static struct imc_pmu *imc_event_to_pmu(struct perf_event *event)
47{
48 return container_of(event->pmu, struct imc_pmu, pmu);
49}
50
51PMU_FORMAT_ATTR(event, "config:0-40");
52PMU_FORMAT_ATTR(offset, "config:0-31");
53PMU_FORMAT_ATTR(rvalue, "config:32");
54PMU_FORMAT_ATTR(mode, "config:33-40");
55static struct attribute *imc_format_attrs[] = {
56 &format_attr_event.attr,
57 &format_attr_offset.attr,
58 &format_attr_rvalue.attr,
59 &format_attr_mode.attr,
60 NULL,
61};
62
63static struct attribute_group imc_format_group = {
64 .name = "format",
65 .attrs = imc_format_attrs,
66};
67
68
69static ssize_t imc_pmu_cpumask_get_attr(struct device *dev,
70 struct device_attribute *attr,
71 char *buf)
72{
73 struct pmu *pmu = dev_get_drvdata(dev);
74 struct imc_pmu *imc_pmu = container_of(pmu, struct imc_pmu, pmu);
75 cpumask_t *active_mask;
76
77 switch(imc_pmu->domain){
78 case IMC_DOMAIN_NEST:
79 active_mask = &nest_imc_cpumask;
80 break;
81 case IMC_DOMAIN_CORE:
82 active_mask = &core_imc_cpumask;
83 break;
84 default:
85 return 0;
86 }
87
88 return cpumap_print_to_pagebuf(true, buf, active_mask);
89}
90
91static DEVICE_ATTR(cpumask, S_IRUGO, imc_pmu_cpumask_get_attr, NULL);
92
93static struct attribute *imc_pmu_cpumask_attrs[] = {
94 &dev_attr_cpumask.attr,
95 NULL,
96};
97
98static struct attribute_group imc_pmu_cpumask_attr_group = {
99 .attrs = imc_pmu_cpumask_attrs,
100};
101
102
103static struct attribute *device_str_attr_create(const char *name, const char *str)
104{
105 struct perf_pmu_events_attr *attr;
106
107 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
108 if (!attr)
109 return NULL;
110 sysfs_attr_init(&attr->attr.attr);
111
112 attr->event_str = str;
113 attr->attr.attr.name = name;
114 attr->attr.attr.mode = 0444;
115 attr->attr.show = perf_event_sysfs_show;
116
117 return &attr->attr.attr;
118}
119
120static int imc_parse_event(struct device_node *np, const char *scale,
121 const char *unit, const char *prefix,
122 u32 base, struct imc_events *event)
123{
124 const char *s;
125 u32 reg;
126
127 if (of_property_read_u32(np, "reg", ®))
128 goto error;
129
130 event->value = base + reg;
131
132 if (of_property_read_string(np, "event-name", &s))
133 goto error;
134
135 event->name = kasprintf(GFP_KERNEL, "%s%s", prefix, s);
136 if (!event->name)
137 goto error;
138
139 if (of_property_read_string(np, "scale", &s))
140 s = scale;
141
142 if (s) {
143 event->scale = kstrdup(s, GFP_KERNEL);
144 if (!event->scale)
145 goto error;
146 }
147
148 if (of_property_read_string(np, "unit", &s))
149 s = unit;
150
151 if (s) {
152 event->unit = kstrdup(s, GFP_KERNEL);
153 if (!event->unit)
154 goto error;
155 }
156
157 return 0;
158error:
159 kfree(event->unit);
160 kfree(event->scale);
161 kfree(event->name);
162 return -EINVAL;
163}
164
165
166
167
168
169static void imc_free_events(struct imc_events *events, int nr_entries)
170{
171 int i;
172
173
174 if (!events)
175 return;
176 for (i = 0; i < nr_entries; i++) {
177 kfree(events[i].unit);
178 kfree(events[i].scale);
179 kfree(events[i].name);
180 }
181
182 kfree(events);
183}
184
185
186
187
188
189static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
190{
191 struct attribute_group *attr_group;
192 struct attribute **attrs, *dev_str;
193 struct device_node *np, *pmu_events;
194 u32 handle, base_reg;
195 int i = 0, j = 0, ct, ret;
196 const char *prefix, *g_scale, *g_unit;
197 const char *ev_val_str, *ev_scale_str, *ev_unit_str;
198
199 if (!of_property_read_u32(node, "events", &handle))
200 pmu_events = of_find_node_by_phandle(handle);
201 else
202 return 0;
203
204
205 if (!pmu_events)
206 return 0;
207
208
209 ct = of_get_child_count(pmu_events);
210
211
212 if (of_property_read_string(node, "events-prefix", &prefix))
213 return 0;
214
215
216 if (of_property_read_string(node, "scale", &g_scale))
217 g_scale = NULL;
218
219 if (of_property_read_string(node, "unit", &g_unit))
220 g_unit = NULL;
221
222
223 of_property_read_u32(node, "reg", &base_reg);
224
225
226 pmu->events = kcalloc(ct, sizeof(struct imc_events), GFP_KERNEL);
227 if (!pmu->events)
228 return -ENOMEM;
229
230 ct = 0;
231
232 for_each_child_of_node(pmu_events, np) {
233 ret = imc_parse_event(np, g_scale, g_unit, prefix, base_reg, &pmu->events[ct]);
234 if (!ret)
235 ct++;
236 }
237
238
239 attr_group = kzalloc(sizeof(*attr_group), GFP_KERNEL);
240 if (!attr_group) {
241 imc_free_events(pmu->events, ct);
242 return -ENOMEM;
243 }
244
245
246
247
248
249
250
251
252
253 attrs = kcalloc(((ct * 3) + 1), sizeof(struct attribute *), GFP_KERNEL);
254 if (!attrs) {
255 kfree(attr_group);
256 imc_free_events(pmu->events, ct);
257 return -ENOMEM;
258 }
259
260 attr_group->name = "events";
261 attr_group->attrs = attrs;
262 do {
263 ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i].value);
264 dev_str = device_str_attr_create(pmu->events[i].name, ev_val_str);
265 if (!dev_str)
266 continue;
267
268 attrs[j++] = dev_str;
269 if (pmu->events[i].scale) {
270 ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale", pmu->events[i].name);
271 dev_str = device_str_attr_create(ev_scale_str, pmu->events[i].scale);
272 if (!dev_str)
273 continue;
274
275 attrs[j++] = dev_str;
276 }
277
278 if (pmu->events[i].unit) {
279 ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit", pmu->events[i].name);
280 dev_str = device_str_attr_create(ev_unit_str, pmu->events[i].unit);
281 if (!dev_str)
282 continue;
283
284 attrs[j++] = dev_str;
285 }
286 } while (++i < ct);
287
288
289 pmu->attr_groups[IMC_EVENT_ATTR] = attr_group;
290
291 return 0;
292}
293
294
295static struct imc_pmu_ref *get_nest_pmu_ref(int cpu)
296{
297 return per_cpu(local_nest_imc_refc, cpu);
298}
299
300static void nest_change_cpu_context(int old_cpu, int new_cpu)
301{
302 struct imc_pmu **pn = per_nest_pmu_arr;
303
304 if (old_cpu < 0 || new_cpu < 0)
305 return;
306
307 while (*pn) {
308 perf_pmu_migrate_context(&(*pn)->pmu, old_cpu, new_cpu);
309 pn++;
310 }
311}
312
313static int ppc_nest_imc_cpu_offline(unsigned int cpu)
314{
315 int nid, target = -1;
316 const struct cpumask *l_cpumask;
317 struct imc_pmu_ref *ref;
318
319
320
321
322
323 if (!cpumask_test_and_clear_cpu(cpu, &nest_imc_cpumask))
324 return 0;
325
326
327
328
329
330
331
332
333
334
335
336 if (!nest_pmus)
337 return 0;
338
339
340
341
342
343 nid = cpu_to_node(cpu);
344 l_cpumask = cpumask_of_node(nid);
345 target = cpumask_any_but(l_cpumask, cpu);
346
347
348
349
350
351 if (target >= 0 && target < nr_cpu_ids) {
352 cpumask_set_cpu(target, &nest_imc_cpumask);
353 nest_change_cpu_context(cpu, target);
354 } else {
355 opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
356 get_hard_smp_processor_id(cpu));
357
358
359
360
361 ref = get_nest_pmu_ref(cpu);
362 if (!ref)
363 return -EINVAL;
364
365 ref->refc = 0;
366 }
367 return 0;
368}
369
370static int ppc_nest_imc_cpu_online(unsigned int cpu)
371{
372 const struct cpumask *l_cpumask;
373 static struct cpumask tmp_mask;
374 int res;
375
376
377 l_cpumask = cpumask_of_node(cpu_to_node(cpu));
378
379
380
381
382
383 if (cpumask_and(&tmp_mask, l_cpumask, &nest_imc_cpumask))
384 return 0;
385
386
387
388
389
390 res = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
391 get_hard_smp_processor_id(cpu));
392 if (res)
393 return res;
394
395
396 cpumask_set_cpu(cpu, &nest_imc_cpumask);
397 return 0;
398}
399
400static int nest_pmu_cpumask_init(void)
401{
402 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE,
403 "perf/powerpc/imc:online",
404 ppc_nest_imc_cpu_online,
405 ppc_nest_imc_cpu_offline);
406}
407
408static void nest_imc_counters_release(struct perf_event *event)
409{
410 int rc, node_id;
411 struct imc_pmu_ref *ref;
412
413 if (event->cpu < 0)
414 return;
415
416 node_id = cpu_to_node(event->cpu);
417
418
419
420
421
422
423
424 ref = get_nest_pmu_ref(event->cpu);
425 if (!ref)
426 return;
427
428
429 mutex_lock(&ref->lock);
430 if (ref->refc == 0) {
431
432
433
434
435
436
437
438
439
440
441 mutex_unlock(&ref->lock);
442 return;
443 }
444 ref->refc--;
445 if (ref->refc == 0) {
446 rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
447 get_hard_smp_processor_id(event->cpu));
448 if (rc) {
449 mutex_unlock(&ref->lock);
450 pr_err("nest-imc: Unable to stop the counters for core %d\n", node_id);
451 return;
452 }
453 } else if (ref->refc < 0) {
454 WARN(1, "nest-imc: Invalid event reference count\n");
455 ref->refc = 0;
456 }
457 mutex_unlock(&ref->lock);
458}
459
460static int nest_imc_event_init(struct perf_event *event)
461{
462 int chip_id, rc, node_id;
463 u32 l_config, config = event->attr.config;
464 struct imc_mem_info *pcni;
465 struct imc_pmu *pmu;
466 struct imc_pmu_ref *ref;
467 bool flag = false;
468
469 if (event->attr.type != event->pmu->type)
470 return -ENOENT;
471
472
473 if (event->hw.sample_period)
474 return -EINVAL;
475
476 if (event->cpu < 0)
477 return -EINVAL;
478
479 pmu = imc_event_to_pmu(event);
480
481
482 if ((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size)
483 return -EINVAL;
484
485
486
487
488
489 chip_id = cpu_to_chip_id(event->cpu);
490 pcni = pmu->mem_info;
491 do {
492 if (pcni->id == chip_id) {
493 flag = true;
494 break;
495 }
496 pcni++;
497 } while (pcni);
498
499 if (!flag)
500 return -ENODEV;
501
502
503
504
505 l_config = config & IMC_EVENT_OFFSET_MASK;
506 event->hw.event_base = (u64)pcni->vbase + l_config;
507 node_id = cpu_to_node(event->cpu);
508
509
510
511
512
513
514 ref = get_nest_pmu_ref(event->cpu);
515 if (!ref)
516 return -EINVAL;
517
518 mutex_lock(&ref->lock);
519 if (ref->refc == 0) {
520 rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_NEST,
521 get_hard_smp_processor_id(event->cpu));
522 if (rc) {
523 mutex_unlock(&ref->lock);
524 pr_err("nest-imc: Unable to start the counters for node %d\n",
525 node_id);
526 return rc;
527 }
528 }
529 ++ref->refc;
530 mutex_unlock(&ref->lock);
531
532 event->destroy = nest_imc_counters_release;
533 return 0;
534}
535
536
537
538
539
540
541
542
543
544static int core_imc_mem_init(int cpu, int size)
545{
546 int nid, rc = 0, core_id = (cpu / threads_per_core);
547 struct imc_mem_info *mem_info;
548
549
550
551
552
553 nid = cpu_to_node(cpu);
554 mem_info = &core_imc_pmu->mem_info[core_id];
555 mem_info->id = core_id;
556
557
558 mem_info->vbase = page_address(alloc_pages_node(nid,
559 GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE |
560 __GFP_NOWARN, get_order(size)));
561 if (!mem_info->vbase)
562 return -ENOMEM;
563
564
565 core_imc_refc[core_id].id = core_id;
566 mutex_init(&core_imc_refc[core_id].lock);
567
568 rc = opal_imc_counters_init(OPAL_IMC_COUNTERS_CORE,
569 __pa((void *)mem_info->vbase),
570 get_hard_smp_processor_id(cpu));
571 if (rc) {
572 free_pages((u64)mem_info->vbase, get_order(size));
573 mem_info->vbase = NULL;
574 }
575
576 return rc;
577}
578
579static bool is_core_imc_mem_inited(int cpu)
580{
581 struct imc_mem_info *mem_info;
582 int core_id = (cpu / threads_per_core);
583
584 mem_info = &core_imc_pmu->mem_info[core_id];
585 if (!mem_info->vbase)
586 return false;
587
588 return true;
589}
590
591static int ppc_core_imc_cpu_online(unsigned int cpu)
592{
593 const struct cpumask *l_cpumask;
594 static struct cpumask tmp_mask;
595 int ret = 0;
596
597
598 l_cpumask = cpu_sibling_mask(cpu);
599
600
601 if (cpumask_and(&tmp_mask, l_cpumask, &core_imc_cpumask))
602 return 0;
603
604 if (!is_core_imc_mem_inited(cpu)) {
605 ret = core_imc_mem_init(cpu, core_imc_pmu->counter_mem_size);
606 if (ret) {
607 pr_info("core_imc memory allocation for cpu %d failed\n", cpu);
608 return ret;
609 }
610 }
611
612
613 cpumask_set_cpu(cpu, &core_imc_cpumask);
614 return 0;
615}
616
617static int ppc_core_imc_cpu_offline(unsigned int cpu)
618{
619 unsigned int core_id;
620 int ncpu;
621 struct imc_pmu_ref *ref;
622
623
624
625
626
627 if (!cpumask_test_and_clear_cpu(cpu, &core_imc_cpumask))
628 return 0;
629
630
631
632
633
634
635
636
637
638
639
640
641 if (!core_imc_pmu->pmu.event_init)
642 return 0;
643
644
645 ncpu = cpumask_any_but(cpu_sibling_mask(cpu), cpu);
646
647 if (ncpu >= 0 && ncpu < nr_cpu_ids) {
648 cpumask_set_cpu(ncpu, &core_imc_cpumask);
649 perf_pmu_migrate_context(&core_imc_pmu->pmu, cpu, ncpu);
650 } else {
651
652
653
654
655
656 opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
657 get_hard_smp_processor_id(cpu));
658 core_id = cpu / threads_per_core;
659 ref = &core_imc_refc[core_id];
660 if (!ref)
661 return -EINVAL;
662
663 ref->refc = 0;
664 }
665 return 0;
666}
667
668static int core_imc_pmu_cpumask_init(void)
669{
670 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE,
671 "perf/powerpc/imc_core:online",
672 ppc_core_imc_cpu_online,
673 ppc_core_imc_cpu_offline);
674}
675
676static void core_imc_counters_release(struct perf_event *event)
677{
678 int rc, core_id;
679 struct imc_pmu_ref *ref;
680
681 if (event->cpu < 0)
682 return;
683
684
685
686
687
688
689 core_id = event->cpu / threads_per_core;
690
691
692 ref = &core_imc_refc[core_id];
693 if (!ref)
694 return;
695
696 mutex_lock(&ref->lock);
697 if (ref->refc == 0) {
698
699
700
701
702
703
704
705
706
707
708 mutex_unlock(&ref->lock);
709 return;
710 }
711 ref->refc--;
712 if (ref->refc == 0) {
713 rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
714 get_hard_smp_processor_id(event->cpu));
715 if (rc) {
716 mutex_unlock(&ref->lock);
717 pr_err("IMC: Unable to stop the counters for core %d\n", core_id);
718 return;
719 }
720 } else if (ref->refc < 0) {
721 WARN(1, "core-imc: Invalid event reference count\n");
722 ref->refc = 0;
723 }
724 mutex_unlock(&ref->lock);
725}
726
727static int core_imc_event_init(struct perf_event *event)
728{
729 int core_id, rc;
730 u64 config = event->attr.config;
731 struct imc_mem_info *pcmi;
732 struct imc_pmu *pmu;
733 struct imc_pmu_ref *ref;
734
735 if (event->attr.type != event->pmu->type)
736 return -ENOENT;
737
738
739 if (event->hw.sample_period)
740 return -EINVAL;
741
742 if (event->cpu < 0)
743 return -EINVAL;
744
745 event->hw.idx = -1;
746 pmu = imc_event_to_pmu(event);
747
748
749 if (((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size))
750 return -EINVAL;
751
752 if (!is_core_imc_mem_inited(event->cpu))
753 return -ENODEV;
754
755 core_id = event->cpu / threads_per_core;
756 pcmi = &core_imc_pmu->mem_info[core_id];
757 if ((!pcmi->vbase))
758 return -ENODEV;
759
760
761 ref = &core_imc_refc[core_id];
762 if (!ref)
763 return -EINVAL;
764
765
766
767
768
769
770
771 mutex_lock(&ref->lock);
772 if (ref->refc == 0) {
773 rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
774 get_hard_smp_processor_id(event->cpu));
775 if (rc) {
776 mutex_unlock(&ref->lock);
777 pr_err("core-imc: Unable to start the counters for core %d\n",
778 core_id);
779 return rc;
780 }
781 }
782 ++ref->refc;
783 mutex_unlock(&ref->lock);
784
785 event->hw.event_base = (u64)pcmi->vbase + (config & IMC_EVENT_OFFSET_MASK);
786 event->destroy = core_imc_counters_release;
787 return 0;
788}
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808static int thread_imc_mem_alloc(int cpu_id, int size)
809{
810 u64 ldbar_value, *local_mem = per_cpu(thread_imc_mem, cpu_id);
811 int nid = cpu_to_node(cpu_id);
812
813 if (!local_mem) {
814
815
816
817
818 local_mem = page_address(alloc_pages_node(nid,
819 GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE |
820 __GFP_NOWARN, get_order(size)));
821 if (!local_mem)
822 return -ENOMEM;
823
824 per_cpu(thread_imc_mem, cpu_id) = local_mem;
825 }
826
827 ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) | THREAD_IMC_ENABLE;
828
829 mtspr(SPRN_LDBAR, ldbar_value);
830 return 0;
831}
832
833static int ppc_thread_imc_cpu_online(unsigned int cpu)
834{
835 return thread_imc_mem_alloc(cpu, thread_imc_mem_size);
836}
837
838static int ppc_thread_imc_cpu_offline(unsigned int cpu)
839{
840 mtspr(SPRN_LDBAR, 0);
841 return 0;
842}
843
844static int thread_imc_cpu_init(void)
845{
846 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE,
847 "perf/powerpc/imc_thread:online",
848 ppc_thread_imc_cpu_online,
849 ppc_thread_imc_cpu_offline);
850}
851
852static int thread_imc_event_init(struct perf_event *event)
853{
854 u32 config = event->attr.config;
855 struct task_struct *target;
856 struct imc_pmu *pmu;
857
858 if (event->attr.type != event->pmu->type)
859 return -ENOENT;
860
861
862 if (event->hw.sample_period)
863 return -EINVAL;
864
865 event->hw.idx = -1;
866 pmu = imc_event_to_pmu(event);
867
868
869 if (((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size))
870 return -EINVAL;
871
872 target = event->hw.target;
873 if (!target)
874 return -EINVAL;
875
876 event->pmu->task_ctx_nr = perf_sw_context;
877 return 0;
878}
879
880static bool is_thread_imc_pmu(struct perf_event *event)
881{
882 if (!strncmp(event->pmu->name, "thread_imc", strlen("thread_imc")))
883 return true;
884
885 return false;
886}
887
888static u64 * get_event_base_addr(struct perf_event *event)
889{
890 u64 addr;
891
892 if (is_thread_imc_pmu(event)) {
893 addr = (u64)per_cpu(thread_imc_mem, smp_processor_id());
894 return (u64 *)(addr + (event->attr.config & IMC_EVENT_OFFSET_MASK));
895 }
896
897 return (u64 *)event->hw.event_base;
898}
899
900static void thread_imc_pmu_start_txn(struct pmu *pmu,
901 unsigned int txn_flags)
902{
903 if (txn_flags & ~PERF_PMU_TXN_ADD)
904 return;
905 perf_pmu_disable(pmu);
906}
907
908static void thread_imc_pmu_cancel_txn(struct pmu *pmu)
909{
910 perf_pmu_enable(pmu);
911}
912
913static int thread_imc_pmu_commit_txn(struct pmu *pmu)
914{
915 perf_pmu_enable(pmu);
916 return 0;
917}
918
919static u64 imc_read_counter(struct perf_event *event)
920{
921 u64 *addr, data;
922
923
924
925
926
927
928
929 addr = get_event_base_addr(event);
930 data = be64_to_cpu(READ_ONCE(*addr));
931 local64_set(&event->hw.prev_count, data);
932
933 return data;
934}
935
936static void imc_event_update(struct perf_event *event)
937{
938 u64 counter_prev, counter_new, final_count;
939
940 counter_prev = local64_read(&event->hw.prev_count);
941 counter_new = imc_read_counter(event);
942 final_count = counter_new - counter_prev;
943
944
945 local64_add(final_count, &event->count);
946}
947
948static void imc_event_start(struct perf_event *event, int flags)
949{
950
951
952
953
954
955
956 imc_read_counter(event);
957}
958
959static void imc_event_stop(struct perf_event *event, int flags)
960{
961
962
963
964
965 imc_event_update(event);
966}
967
968static int imc_event_add(struct perf_event *event, int flags)
969{
970 if (flags & PERF_EF_START)
971 imc_event_start(event, flags);
972
973 return 0;
974}
975
976static int thread_imc_event_add(struct perf_event *event, int flags)
977{
978 int core_id;
979 struct imc_pmu_ref *ref;
980
981 if (flags & PERF_EF_START)
982 imc_event_start(event, flags);
983
984 if (!is_core_imc_mem_inited(smp_processor_id()))
985 return -EINVAL;
986
987 core_id = smp_processor_id() / threads_per_core;
988
989
990
991
992
993
994 ref = &core_imc_refc[core_id];
995 if (!ref)
996 return -EINVAL;
997
998 mutex_lock(&ref->lock);
999 if (ref->refc == 0) {
1000 if (opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
1001 get_hard_smp_processor_id(smp_processor_id()))) {
1002 mutex_unlock(&ref->lock);
1003 pr_err("thread-imc: Unable to start the counter\
1004 for core %d\n", core_id);
1005 return -EINVAL;
1006 }
1007 }
1008 ++ref->refc;
1009 mutex_unlock(&ref->lock);
1010 return 0;
1011}
1012
1013static void thread_imc_event_del(struct perf_event *event, int flags)
1014{
1015
1016 int core_id;
1017 struct imc_pmu_ref *ref;
1018
1019
1020
1021
1022
1023 imc_event_update(event);
1024
1025 core_id = smp_processor_id() / threads_per_core;
1026 ref = &core_imc_refc[core_id];
1027
1028 mutex_lock(&ref->lock);
1029 ref->refc--;
1030 if (ref->refc == 0) {
1031 if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
1032 get_hard_smp_processor_id(smp_processor_id()))) {
1033 mutex_unlock(&ref->lock);
1034 pr_err("thread-imc: Unable to stop the counters\
1035 for core %d\n", core_id);
1036 return;
1037 }
1038 } else if (ref->refc < 0) {
1039 ref->refc = 0;
1040 }
1041 mutex_unlock(&ref->lock);
1042}
1043
1044
1045static int update_pmu_ops(struct imc_pmu *pmu)
1046{
1047 pmu->pmu.task_ctx_nr = perf_invalid_context;
1048 pmu->pmu.add = imc_event_add;
1049 pmu->pmu.del = imc_event_stop;
1050 pmu->pmu.start = imc_event_start;
1051 pmu->pmu.stop = imc_event_stop;
1052 pmu->pmu.read = imc_event_update;
1053 pmu->pmu.attr_groups = pmu->attr_groups;
1054 pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
1055 pmu->attr_groups[IMC_FORMAT_ATTR] = &imc_format_group;
1056
1057 switch (pmu->domain) {
1058 case IMC_DOMAIN_NEST:
1059 pmu->pmu.event_init = nest_imc_event_init;
1060 pmu->attr_groups[IMC_CPUMASK_ATTR] = &imc_pmu_cpumask_attr_group;
1061 break;
1062 case IMC_DOMAIN_CORE:
1063 pmu->pmu.event_init = core_imc_event_init;
1064 pmu->attr_groups[IMC_CPUMASK_ATTR] = &imc_pmu_cpumask_attr_group;
1065 break;
1066 case IMC_DOMAIN_THREAD:
1067 pmu->pmu.event_init = thread_imc_event_init;
1068 pmu->pmu.add = thread_imc_event_add;
1069 pmu->pmu.del = thread_imc_event_del;
1070 pmu->pmu.start_txn = thread_imc_pmu_start_txn;
1071 pmu->pmu.cancel_txn = thread_imc_pmu_cancel_txn;
1072 pmu->pmu.commit_txn = thread_imc_pmu_commit_txn;
1073 break;
1074 default:
1075 break;
1076 }
1077
1078 return 0;
1079}
1080
1081
1082static int init_nest_pmu_ref(void)
1083{
1084 int nid, i, cpu;
1085
1086 nest_imc_refc = kcalloc(num_possible_nodes(), sizeof(*nest_imc_refc),
1087 GFP_KERNEL);
1088
1089 if (!nest_imc_refc)
1090 return -ENOMEM;
1091
1092 i = 0;
1093 for_each_node(nid) {
1094
1095
1096
1097
1098 mutex_init(&nest_imc_refc[i].lock);
1099
1100
1101
1102
1103
1104
1105
1106 nest_imc_refc[i++].id = nid;
1107 }
1108
1109
1110
1111
1112
1113 for_each_possible_cpu(cpu) {
1114 nid = cpu_to_node(cpu);
1115 for (i = 0; i < num_possible_nodes(); i++) {
1116 if (nest_imc_refc[i].id == nid) {
1117 per_cpu(local_nest_imc_refc, cpu) = &nest_imc_refc[i];
1118 break;
1119 }
1120 }
1121 }
1122 return 0;
1123}
1124
1125static void cleanup_all_core_imc_memory(void)
1126{
1127 int i, nr_cores = DIV_ROUND_UP(num_possible_cpus(), threads_per_core);
1128 struct imc_mem_info *ptr = core_imc_pmu->mem_info;
1129 int size = core_imc_pmu->counter_mem_size;
1130
1131
1132 for (i = 0; i < nr_cores; i++) {
1133 if (ptr[i].vbase)
1134 free_pages((u64)ptr[i].vbase, get_order(size));
1135 }
1136
1137 kfree(ptr);
1138 kfree(core_imc_refc);
1139}
1140
1141static void thread_imc_ldbar_disable(void *dummy)
1142{
1143
1144
1145
1146
1147 mtspr(SPRN_LDBAR, 0);
1148}
1149
1150void thread_imc_disable(void)
1151{
1152 on_each_cpu(thread_imc_ldbar_disable, NULL, 1);
1153}
1154
1155static void cleanup_all_thread_imc_memory(void)
1156{
1157 int i, order = get_order(thread_imc_mem_size);
1158
1159 for_each_online_cpu(i) {
1160 if (per_cpu(thread_imc_mem, i))
1161 free_pages((u64)per_cpu(thread_imc_mem, i), order);
1162
1163 }
1164}
1165
1166
1167static void imc_common_mem_free(struct imc_pmu *pmu_ptr)
1168{
1169 if (pmu_ptr->attr_groups[IMC_EVENT_ATTR])
1170 kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]->attrs);
1171 kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]);
1172}
1173
1174
1175
1176
1177
1178
1179
1180static void imc_common_cpuhp_mem_free(struct imc_pmu *pmu_ptr)
1181{
1182 if (pmu_ptr->domain == IMC_DOMAIN_NEST) {
1183 mutex_lock(&nest_init_lock);
1184 if (nest_pmus == 1) {
1185 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE);
1186 kfree(nest_imc_refc);
1187 kfree(per_nest_pmu_arr);
1188 per_nest_pmu_arr = NULL;
1189 }
1190
1191 if (nest_pmus > 0)
1192 nest_pmus--;
1193 mutex_unlock(&nest_init_lock);
1194 }
1195
1196
1197 if (pmu_ptr->domain == IMC_DOMAIN_CORE) {
1198 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE);
1199 cleanup_all_core_imc_memory();
1200 }
1201
1202
1203 if (pmu_ptr->domain == IMC_DOMAIN_THREAD) {
1204 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE);
1205 cleanup_all_thread_imc_memory();
1206 }
1207}
1208
1209
1210
1211
1212
1213void unregister_thread_imc(void)
1214{
1215 imc_common_cpuhp_mem_free(thread_imc_pmu);
1216 imc_common_mem_free(thread_imc_pmu);
1217 perf_pmu_unregister(&thread_imc_pmu->pmu);
1218}
1219
1220
1221
1222
1223static int imc_mem_init(struct imc_pmu *pmu_ptr, struct device_node *parent,
1224 int pmu_index)
1225{
1226 const char *s;
1227 int nr_cores, cpu, res = -ENOMEM;
1228
1229 if (of_property_read_string(parent, "name", &s))
1230 return -ENODEV;
1231
1232 switch (pmu_ptr->domain) {
1233 case IMC_DOMAIN_NEST:
1234
1235 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s_imc", "nest_", s);
1236 if (!pmu_ptr->pmu.name)
1237 goto err;
1238
1239
1240 if (!per_nest_pmu_arr) {
1241 per_nest_pmu_arr = kcalloc(get_max_nest_dev() + 1,
1242 sizeof(struct imc_pmu *),
1243 GFP_KERNEL);
1244 if (!per_nest_pmu_arr)
1245 goto err;
1246 }
1247 per_nest_pmu_arr[pmu_index] = pmu_ptr;
1248 break;
1249 case IMC_DOMAIN_CORE:
1250
1251 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
1252 if (!pmu_ptr->pmu.name)
1253 goto err;
1254
1255 nr_cores = DIV_ROUND_UP(num_possible_cpus(), threads_per_core);
1256 pmu_ptr->mem_info = kcalloc(nr_cores, sizeof(struct imc_mem_info),
1257 GFP_KERNEL);
1258
1259 if (!pmu_ptr->mem_info)
1260 goto err;
1261
1262 core_imc_refc = kcalloc(nr_cores, sizeof(struct imc_pmu_ref),
1263 GFP_KERNEL);
1264
1265 if (!core_imc_refc) {
1266 kfree(pmu_ptr->mem_info);
1267 goto err;
1268 }
1269
1270 core_imc_pmu = pmu_ptr;
1271 break;
1272 case IMC_DOMAIN_THREAD:
1273
1274 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
1275 if (!pmu_ptr->pmu.name)
1276 goto err;
1277
1278 thread_imc_mem_size = pmu_ptr->counter_mem_size;
1279 for_each_online_cpu(cpu) {
1280 res = thread_imc_mem_alloc(cpu, pmu_ptr->counter_mem_size);
1281 if (res) {
1282 cleanup_all_thread_imc_memory();
1283 goto err;
1284 }
1285 }
1286
1287 thread_imc_pmu = pmu_ptr;
1288 break;
1289 default:
1290 return -EINVAL;
1291 }
1292
1293 return 0;
1294err:
1295 return res;
1296}
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308int init_imc_pmu(struct device_node *parent, struct imc_pmu *pmu_ptr, int pmu_idx)
1309{
1310 int ret;
1311
1312 ret = imc_mem_init(pmu_ptr, parent, pmu_idx);
1313 if (ret)
1314 goto err_free_mem;
1315
1316 switch (pmu_ptr->domain) {
1317 case IMC_DOMAIN_NEST:
1318
1319
1320
1321
1322
1323
1324 mutex_lock(&nest_init_lock);
1325 if (nest_pmus == 0) {
1326 ret = init_nest_pmu_ref();
1327 if (ret) {
1328 mutex_unlock(&nest_init_lock);
1329 kfree(per_nest_pmu_arr);
1330 per_nest_pmu_arr = NULL;
1331 goto err_free_mem;
1332 }
1333
1334 ret = nest_pmu_cpumask_init();
1335 if (ret) {
1336 mutex_unlock(&nest_init_lock);
1337 kfree(nest_imc_refc);
1338 kfree(per_nest_pmu_arr);
1339 per_nest_pmu_arr = NULL;
1340 goto err_free_mem;
1341 }
1342 }
1343 nest_pmus++;
1344 mutex_unlock(&nest_init_lock);
1345 break;
1346 case IMC_DOMAIN_CORE:
1347 ret = core_imc_pmu_cpumask_init();
1348 if (ret) {
1349 cleanup_all_core_imc_memory();
1350 goto err_free_mem;
1351 }
1352
1353 break;
1354 case IMC_DOMAIN_THREAD:
1355 ret = thread_imc_cpu_init();
1356 if (ret) {
1357 cleanup_all_thread_imc_memory();
1358 goto err_free_mem;
1359 }
1360
1361 break;
1362 default:
1363 return -EINVAL;
1364 }
1365
1366 ret = update_events_in_group(parent, pmu_ptr);
1367 if (ret)
1368 goto err_free_cpuhp_mem;
1369
1370 ret = update_pmu_ops(pmu_ptr);
1371 if (ret)
1372 goto err_free_cpuhp_mem;
1373
1374 ret = perf_pmu_register(&pmu_ptr->pmu, pmu_ptr->pmu.name, -1);
1375 if (ret)
1376 goto err_free_cpuhp_mem;
1377
1378 pr_debug("%s performance monitor hardware support registered\n",
1379 pmu_ptr->pmu.name);
1380
1381 return 0;
1382
1383err_free_cpuhp_mem:
1384 imc_common_cpuhp_mem_free(pmu_ptr);
1385err_free_mem:
1386 imc_common_mem_free(pmu_ptr);
1387 return ret;
1388}
1389