1
2
3
4#include <linux/sched/task.h>
5#include <linux/io-64-nonatomic-lo-hi.h>
6#include "idxd.h"
7#include "perfmon.h"
8
9static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
10 char *buf);
11
12static cpumask_t perfmon_dsa_cpu_mask;
13static bool cpuhp_set_up;
14static enum cpuhp_state cpuhp_slot;
15
16
17
18
19
20
21static DEVICE_ATTR_RO(cpumask);
22
23static struct attribute *perfmon_cpumask_attrs[] = {
24 &dev_attr_cpumask.attr,
25 NULL,
26};
27
28static struct attribute_group cpumask_attr_group = {
29 .attrs = perfmon_cpumask_attrs,
30};
31
32
33
34
35
36DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
37DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
38
39
40
41
42
43DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
44DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
45DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
46DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
47DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
48
49#define PERFMON_FILTERS_START 2
50#define PERFMON_FILTERS_MAX 5
51
52static struct attribute *perfmon_format_attrs[] = {
53 &format_attr_idxd_event_category.attr,
54 &format_attr_idxd_event.attr,
55 &format_attr_idxd_filter_wq.attr,
56 &format_attr_idxd_filter_tc.attr,
57 &format_attr_idxd_filter_pgsz.attr,
58 &format_attr_idxd_filter_sz.attr,
59 &format_attr_idxd_filter_eng.attr,
60 NULL,
61};
62
63static struct attribute_group perfmon_format_attr_group = {
64 .name = "format",
65 .attrs = perfmon_format_attrs,
66};
67
68static const struct attribute_group *perfmon_attr_groups[] = {
69 &perfmon_format_attr_group,
70 &cpumask_attr_group,
71 NULL,
72};
73
74static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
75 char *buf)
76{
77 return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask);
78}
79
80static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
81{
82 return &idxd_pmu->pmu == event->pmu;
83}
84
85static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
86 struct perf_event *leader,
87 bool do_grp)
88{
89 struct perf_event *event;
90 int n, max_count;
91
92 max_count = idxd_pmu->n_counters;
93 n = idxd_pmu->n_events;
94
95 if (n >= max_count)
96 return -EINVAL;
97
98 if (is_idxd_event(idxd_pmu, leader)) {
99 idxd_pmu->event_list[n] = leader;
100 idxd_pmu->event_list[n]->hw.idx = n;
101 n++;
102 }
103
104 if (!do_grp)
105 return n;
106
107 for_each_sibling_event(event, leader) {
108 if (!is_idxd_event(idxd_pmu, event) ||
109 event->state <= PERF_EVENT_STATE_OFF)
110 continue;
111
112 if (n >= max_count)
113 return -EINVAL;
114
115 idxd_pmu->event_list[n] = event;
116 idxd_pmu->event_list[n]->hw.idx = n;
117 n++;
118 }
119
120 return n;
121}
122
123static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
124 struct perf_event *event, int idx)
125{
126 struct idxd_device *idxd = idxd_pmu->idxd;
127 struct hw_perf_event *hwc = &event->hw;
128
129 hwc->idx = idx;
130 hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
131 hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
132}
133
134static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
135 struct perf_event *event)
136{
137 int i;
138
139 for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
140 if (!test_and_set_bit(i, idxd_pmu->used_mask))
141 return i;
142
143 return -EINVAL;
144}
145
146
147
148
149
150
151
152
153
154static int perfmon_validate_group(struct idxd_pmu *pmu,
155 struct perf_event *event)
156{
157 struct perf_event *leader = event->group_leader;
158 struct idxd_pmu *fake_pmu;
159 int i, ret = 0, n, idx;
160
161 fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
162 if (!fake_pmu)
163 return -ENOMEM;
164
165 fake_pmu->pmu.name = pmu->pmu.name;
166 fake_pmu->n_counters = pmu->n_counters;
167
168 n = perfmon_collect_events(fake_pmu, leader, true);
169 if (n < 0) {
170 ret = n;
171 goto out;
172 }
173
174 fake_pmu->n_events = n;
175 n = perfmon_collect_events(fake_pmu, event, false);
176 if (n < 0) {
177 ret = n;
178 goto out;
179 }
180
181 fake_pmu->n_events = n;
182
183 for (i = 0; i < n; i++) {
184 event = fake_pmu->event_list[i];
185
186 idx = perfmon_assign_event(fake_pmu, event);
187 if (idx < 0) {
188 ret = idx;
189 goto out;
190 }
191 }
192out:
193 kfree(fake_pmu);
194
195 return ret;
196}
197
198static int perfmon_pmu_event_init(struct perf_event *event)
199{
200 struct idxd_device *idxd;
201 int ret = 0;
202
203 idxd = event_to_idxd(event);
204 event->hw.idx = -1;
205
206 if (event->attr.type != event->pmu->type)
207 return -ENOENT;
208
209
210 if (event->attr.sample_period)
211 return -EINVAL;
212
213 if (event->cpu < 0)
214 return -EINVAL;
215
216 if (event->pmu != &idxd->idxd_pmu->pmu)
217 return -EINVAL;
218
219 event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
220 event->cpu = idxd->idxd_pmu->cpu;
221 event->hw.config = event->attr.config;
222
223 if (event->group_leader != event)
224
225 ret = perfmon_validate_group(idxd->idxd_pmu, event);
226
227 return ret;
228}
229
230static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
231{
232 struct hw_perf_event *hwc = &event->hw;
233 struct idxd_device *idxd;
234 int cntr = hwc->idx;
235
236 idxd = event_to_idxd(event);
237
238 return ioread64(CNTRDATA_REG(idxd, cntr));
239}
240
241static void perfmon_pmu_event_update(struct perf_event *event)
242{
243 struct idxd_device *idxd = event_to_idxd(event);
244 u64 prev_raw_count, new_raw_count, delta, p, n;
245 int shift = 64 - idxd->idxd_pmu->counter_width;
246 struct hw_perf_event *hwc = &event->hw;
247
248 do {
249 prev_raw_count = local64_read(&hwc->prev_count);
250 new_raw_count = perfmon_pmu_read_counter(event);
251 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
252 new_raw_count) != prev_raw_count);
253
254 n = (new_raw_count << shift);
255 p = (prev_raw_count << shift);
256
257 delta = ((n - p) >> shift);
258
259 local64_add(delta, &event->count);
260}
261
262void perfmon_counter_overflow(struct idxd_device *idxd)
263{
264 int i, n_counters, max_loop = OVERFLOW_SIZE;
265 struct perf_event *event;
266 unsigned long ovfstatus;
267
268 n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
269
270 ovfstatus = ioread32(OVFSTATUS_REG(idxd));
271
272
273
274
275
276
277
278
279
280 while (ovfstatus && max_loop--) {
281
282 for_each_set_bit(i, &ovfstatus, n_counters) {
283 unsigned long ovfstatus_clear = 0;
284
285
286 event = idxd->idxd_pmu->event_list[i];
287 perfmon_pmu_event_update(event);
288
289 set_bit(i, &ovfstatus_clear);
290 iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
291 }
292
293 ovfstatus = ioread32(OVFSTATUS_REG(idxd));
294 }
295
296
297
298
299
300 WARN_ON_ONCE(ovfstatus);
301}
302
303static inline void perfmon_reset_config(struct idxd_device *idxd)
304{
305 iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
306 iowrite32(0, OVFSTATUS_REG(idxd));
307 iowrite32(0, PERFFRZ_REG(idxd));
308}
309
310static inline void perfmon_reset_counters(struct idxd_device *idxd)
311{
312 iowrite32(CNTR_RESET, PERFRST_REG(idxd));
313}
314
315static inline void perfmon_reset(struct idxd_device *idxd)
316{
317 perfmon_reset_config(idxd);
318 perfmon_reset_counters(idxd);
319}
320
321static void perfmon_pmu_event_start(struct perf_event *event, int mode)
322{
323 u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
324 u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
325 struct hw_perf_event *hwc = &event->hw;
326 union filter_cfg flt_cfg;
327 union event_cfg event_cfg;
328 struct idxd_device *idxd;
329 int cntr;
330
331 idxd = event_to_idxd(event);
332
333 event->hw.idx = hwc->idx;
334 cntr = hwc->idx;
335
336
337 event_cfg.val = event->attr.config;
338 flt_cfg.val = event->attr.config1;
339 event_cat = event_cfg.event_cat;
340 event_enc = event_cfg.event_enc;
341
342
343 flt_wq = flt_cfg.wq;
344 flt_tc = flt_cfg.tc;
345 flt_pg_sz = flt_cfg.pg_sz;
346 flt_xfer_sz = flt_cfg.xfer_sz;
347 flt_eng = flt_cfg.eng;
348
349 if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
350 iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
351 if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
352 iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
353 if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
354 iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
355 if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
356 iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
357 if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
358 iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
359
360
361 cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
362 local64_set(&event->hw.prev_count, cntrdata);
363
364
365 cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
366 cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
367
368 cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
369
370 iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
371}
372
373static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
374{
375 struct hw_perf_event *hwc = &event->hw;
376 struct idxd_device *idxd;
377 int i, cntr = hwc->idx;
378 u64 cntr_cfg;
379
380 idxd = event_to_idxd(event);
381
382
383 for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
384 if (event != idxd->idxd_pmu->event_list[i])
385 continue;
386
387 for (++i; i < idxd->idxd_pmu->n_events; i++)
388 idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
389 --idxd->idxd_pmu->n_events;
390 break;
391 }
392
393 cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
394 cntr_cfg &= ~CNTRCFG_ENABLE;
395 iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
396
397 if (mode == PERF_EF_UPDATE)
398 perfmon_pmu_event_update(event);
399
400 event->hw.idx = -1;
401 clear_bit(cntr, idxd->idxd_pmu->used_mask);
402}
403
404static void perfmon_pmu_event_del(struct perf_event *event, int mode)
405{
406 perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
407}
408
409static int perfmon_pmu_event_add(struct perf_event *event, int flags)
410{
411 struct idxd_device *idxd = event_to_idxd(event);
412 struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
413 struct hw_perf_event *hwc = &event->hw;
414 int idx, n;
415
416 n = perfmon_collect_events(idxd_pmu, event, false);
417 if (n < 0)
418 return n;
419
420 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
421 if (!(flags & PERF_EF_START))
422 hwc->state |= PERF_HES_ARCH;
423
424 idx = perfmon_assign_event(idxd_pmu, event);
425 if (idx < 0)
426 return idx;
427
428 perfmon_assign_hw_event(idxd_pmu, event, idx);
429
430 if (flags & PERF_EF_START)
431 perfmon_pmu_event_start(event, 0);
432
433 idxd_pmu->n_events = n;
434
435 return 0;
436}
437
438static void enable_perfmon_pmu(struct idxd_device *idxd)
439{
440 iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
441}
442
443static void disable_perfmon_pmu(struct idxd_device *idxd)
444{
445 iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
446}
447
448static void perfmon_pmu_enable(struct pmu *pmu)
449{
450 struct idxd_device *idxd = pmu_to_idxd(pmu);
451
452 enable_perfmon_pmu(idxd);
453}
454
455static void perfmon_pmu_disable(struct pmu *pmu)
456{
457 struct idxd_device *idxd = pmu_to_idxd(pmu);
458
459 disable_perfmon_pmu(idxd);
460}
461
462static void skip_filter(int i)
463{
464 int j;
465
466 for (j = i; j < PERFMON_FILTERS_MAX; j++)
467 perfmon_format_attrs[PERFMON_FILTERS_START + j] =
468 perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
469}
470
471static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
472{
473 int i;
474
475 for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
476 if (!test_bit(i, &idxd_pmu->supported_filters))
477 skip_filter(i);
478 }
479
480 idxd_pmu->pmu.name = idxd_pmu->name;
481 idxd_pmu->pmu.attr_groups = perfmon_attr_groups;
482 idxd_pmu->pmu.task_ctx_nr = perf_invalid_context;
483 idxd_pmu->pmu.event_init = perfmon_pmu_event_init;
484 idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable,
485 idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable,
486 idxd_pmu->pmu.add = perfmon_pmu_event_add;
487 idxd_pmu->pmu.del = perfmon_pmu_event_del;
488 idxd_pmu->pmu.start = perfmon_pmu_event_start;
489 idxd_pmu->pmu.stop = perfmon_pmu_event_stop;
490 idxd_pmu->pmu.read = perfmon_pmu_event_update;
491 idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
492 idxd_pmu->pmu.module = THIS_MODULE;
493}
494
495void perfmon_pmu_remove(struct idxd_device *idxd)
496{
497 if (!idxd->idxd_pmu)
498 return;
499
500 cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node);
501 perf_pmu_unregister(&idxd->idxd_pmu->pmu);
502 kfree(idxd->idxd_pmu);
503 idxd->idxd_pmu = NULL;
504}
505
506static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node)
507{
508 struct idxd_pmu *idxd_pmu;
509
510 idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
511
512
513 if (cpumask_empty(&perfmon_dsa_cpu_mask)) {
514 cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask);
515 idxd_pmu->cpu = cpu;
516 }
517
518 return 0;
519}
520
521static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node)
522{
523 struct idxd_pmu *idxd_pmu;
524 unsigned int target;
525
526 idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
527
528 if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask))
529 return 0;
530
531 target = cpumask_any_but(cpu_online_mask, cpu);
532
533
534 if (target < nr_cpu_ids)
535 cpumask_set_cpu(target, &perfmon_dsa_cpu_mask);
536 else
537 target = -1;
538
539 perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target);
540
541 return 0;
542}
543
544int perfmon_pmu_init(struct idxd_device *idxd)
545{
546 union idxd_perfcap perfcap;
547 struct idxd_pmu *idxd_pmu;
548 int rc = -ENODEV;
549
550
551
552
553 if (!cpuhp_set_up)
554 return -ENODEV;
555
556
557
558
559
560 if (idxd->perfmon_offset == 0)
561 return -ENODEV;
562
563 idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
564 if (!idxd_pmu)
565 return -ENOMEM;
566
567 idxd_pmu->idxd = idxd;
568 idxd->idxd_pmu = idxd_pmu;
569
570 if (idxd->data->type == IDXD_TYPE_DSA) {
571 rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);
572 if (rc < 0)
573 goto free;
574 } else if (idxd->data->type == IDXD_TYPE_IAX) {
575 rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);
576 if (rc < 0)
577 goto free;
578 } else {
579 goto free;
580 }
581
582 perfmon_reset(idxd);
583
584 perfcap.bits = ioread64(PERFCAP_REG(idxd));
585
586
587
588
589
590
591 if (perfcap.num_perf_counter == 0)
592 goto free;
593
594
595 if (perfcap.counter_width == 0)
596 goto free;
597
598
599 if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
600 goto free;
601
602
603 if (perfcap.num_event_category == 0)
604 goto free;
605
606
607
608
609 if (perfcap.cap_per_counter)
610 goto free;
611
612 idxd_pmu->n_event_categories = perfcap.num_event_category;
613 idxd_pmu->supported_event_categories = perfcap.global_event_category;
614 idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
615
616
617 idxd_pmu->supported_filters = perfcap.filter;
618 if (perfcap.filter)
619 idxd_pmu->n_filters = hweight8(perfcap.filter);
620
621
622 idxd_pmu->n_counters = perfcap.num_perf_counter;
623 idxd_pmu->counter_width = perfcap.counter_width;
624
625 idxd_pmu_init(idxd_pmu);
626
627 rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);
628 if (rc)
629 goto free;
630
631 rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node);
632 if (rc) {
633 perf_pmu_unregister(&idxd->idxd_pmu->pmu);
634 goto free;
635 }
636out:
637 return rc;
638free:
639 kfree(idxd_pmu);
640 idxd->idxd_pmu = NULL;
641
642 goto out;
643}
644
645void __init perfmon_init(void)
646{
647 int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
648 "driver/dma/idxd/perf:online",
649 perf_event_cpu_online,
650 perf_event_cpu_offline);
651 if (WARN_ON(rc < 0))
652 return;
653
654 cpuhp_slot = rc;
655 cpuhp_set_up = true;
656}
657
658void __exit perfmon_exit(void)
659{
660 if (cpuhp_set_up)
661 cpuhp_remove_multi_state(cpuhp_slot);
662}
663