1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194#include <linux/anon_inodes.h>
195#include <linux/sizes.h>
196#include <linux/uuid.h>
197
198#include "gem/i915_gem_context.h"
199#include "gt/intel_engine_pm.h"
200#include "gt/intel_engine_user.h"
201#include "gt/intel_gt.h"
202#include "gt/intel_lrc_reg.h"
203#include "gt/intel_ring.h"
204
205#include "i915_drv.h"
206#include "i915_perf.h"
207
208
209
210
211
212#define OA_BUFFER_SIZE SZ_16M
213
214#define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248#define OA_TAIL_MARGIN_NSEC 100000ULL
249#define INVALID_TAIL_PTR 0xffffffff
250
251
252
253
254#define DEFAULT_POLL_FREQUENCY_HZ 200
255#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
256
257
258static u32 i915_perf_stream_paranoid = true;
259
260
261
262
263
264
265
266
267
268#define OA_EXPONENT_MAX 31
269
270#define INVALID_CTX_ID 0xffffffff
271
272
273#define OAREPORT_REASON_MASK 0x3f
274#define OAREPORT_REASON_MASK_EXTENDED 0x7f
275#define OAREPORT_REASON_SHIFT 19
276#define OAREPORT_REASON_TIMER (1<<0)
277#define OAREPORT_REASON_CTX_SWITCH (1<<3)
278#define OAREPORT_REASON_CLK_RATIO (1<<5)
279
280
281
282
283
284
285
286
287
288static int oa_sample_rate_hard_limit;
289
290
291
292
293
294
295
296static u32 i915_oa_max_sample_rate = 100000;
297
298
299
300
301
302static const struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = {
303 [I915_OA_FORMAT_A13] = { 0, 64 },
304 [I915_OA_FORMAT_A29] = { 1, 128 },
305 [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 },
306
307 [I915_OA_FORMAT_B4_C8] = { 4, 64 },
308 [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 },
309 [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 },
310 [I915_OA_FORMAT_C4_B8] = { 7, 64 },
311};
312
313static const struct i915_oa_format gen8_plus_oa_formats[I915_OA_FORMAT_MAX] = {
314 [I915_OA_FORMAT_A12] = { 0, 64 },
315 [I915_OA_FORMAT_A12_B8_C8] = { 2, 128 },
316 [I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 },
317 [I915_OA_FORMAT_C4_B8] = { 7, 64 },
318};
319
320static const struct i915_oa_format gen12_oa_formats[I915_OA_FORMAT_MAX] = {
321 [I915_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256 },
322};
323
324#define SAMPLE_OA_REPORT (1<<0)
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349struct perf_open_properties {
350 u32 sample_flags;
351
352 u64 single_context:1;
353 u64 hold_preemption:1;
354 u64 ctx_handle;
355
356
357 int metrics_set;
358 int oa_format;
359 bool oa_periodic;
360 int oa_period_exponent;
361
362 struct intel_engine_cs *engine;
363
364 bool has_sseu;
365 struct intel_sseu sseu;
366
367 u64 poll_oa_period;
368};
369
370struct i915_oa_config_bo {
371 struct llist_node node;
372
373 struct i915_oa_config *oa_config;
374 struct i915_vma *vma;
375};
376
377static struct ctl_table_header *sysctl_header;
378
379static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer);
380
381void i915_oa_config_release(struct kref *ref)
382{
383 struct i915_oa_config *oa_config =
384 container_of(ref, typeof(*oa_config), ref);
385
386 kfree(oa_config->flex_regs);
387 kfree(oa_config->b_counter_regs);
388 kfree(oa_config->mux_regs);
389
390 kfree_rcu(oa_config, rcu);
391}
392
393struct i915_oa_config *
394i915_perf_get_oa_config(struct i915_perf *perf, int metrics_set)
395{
396 struct i915_oa_config *oa_config;
397
398 rcu_read_lock();
399 oa_config = idr_find(&perf->metrics_idr, metrics_set);
400 if (oa_config)
401 oa_config = i915_oa_config_get(oa_config);
402 rcu_read_unlock();
403
404 return oa_config;
405}
406
407static void free_oa_config_bo(struct i915_oa_config_bo *oa_bo)
408{
409 i915_oa_config_put(oa_bo->oa_config);
410 i915_vma_put(oa_bo->vma);
411 kfree(oa_bo);
412}
413
414static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
415{
416 struct intel_uncore *uncore = stream->uncore;
417
418 return intel_uncore_read(uncore, GEN12_OAG_OATAILPTR) &
419 GEN12_OAG_OATAILPTR_MASK;
420}
421
422static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream)
423{
424 struct intel_uncore *uncore = stream->uncore;
425
426 return intel_uncore_read(uncore, GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK;
427}
428
429static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
430{
431 struct intel_uncore *uncore = stream->uncore;
432 u32 oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
433
434 return oastatus1 & GEN7_OASTATUS1_TAIL_MASK;
435}
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
462{
463 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
464 int report_size = stream->oa_buffer.format_size;
465 unsigned long flags;
466 bool pollin;
467 u32 hw_tail;
468 u64 now;
469
470
471
472
473
474 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
475
476 hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
477
478
479
480
481 hw_tail &= ~(report_size - 1);
482
483 now = ktime_get_mono_fast_ns();
484
485 if (hw_tail == stream->oa_buffer.aging_tail &&
486 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) {
487
488
489
490
491 stream->oa_buffer.tail = stream->oa_buffer.aging_tail;
492 } else {
493 u32 head, tail, aged_tail;
494
495
496
497
498
499 head = stream->oa_buffer.head - gtt_offset;
500 aged_tail = stream->oa_buffer.tail - gtt_offset;
501
502 hw_tail -= gtt_offset;
503 tail = hw_tail;
504
505
506
507
508
509
510
511
512
513
514
515
516 while (OA_TAKEN(tail, aged_tail) >= report_size) {
517 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail);
518
519 if (report32[0] != 0 || report32[1] != 0)
520 break;
521
522 tail = (tail - report_size) & (OA_BUFFER_SIZE - 1);
523 }
524
525 if (OA_TAKEN(hw_tail, tail) > report_size &&
526 __ratelimit(&stream->perf->tail_pointer_race))
527 DRM_NOTE("unlanded report(s) head=0x%x "
528 "tail=0x%x hw_tail=0x%x\n",
529 head, tail, hw_tail);
530
531 stream->oa_buffer.tail = gtt_offset + tail;
532 stream->oa_buffer.aging_tail = gtt_offset + hw_tail;
533 stream->oa_buffer.aging_timestamp = now;
534 }
535
536 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset,
537 stream->oa_buffer.head - gtt_offset) >= report_size;
538
539 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
540
541 return pollin;
542}
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559static int append_oa_status(struct i915_perf_stream *stream,
560 char __user *buf,
561 size_t count,
562 size_t *offset,
563 enum drm_i915_perf_record_type type)
564{
565 struct drm_i915_perf_record_header header = { type, 0, sizeof(header) };
566
567 if ((count - *offset) < header.size)
568 return -ENOSPC;
569
570 if (copy_to_user(buf + *offset, &header, sizeof(header)))
571 return -EFAULT;
572
573 (*offset) += header.size;
574
575 return 0;
576}
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595static int append_oa_sample(struct i915_perf_stream *stream,
596 char __user *buf,
597 size_t count,
598 size_t *offset,
599 const u8 *report)
600{
601 int report_size = stream->oa_buffer.format_size;
602 struct drm_i915_perf_record_header header;
603 u32 sample_flags = stream->sample_flags;
604
605 header.type = DRM_I915_PERF_RECORD_SAMPLE;
606 header.pad = 0;
607 header.size = stream->sample_size;
608
609 if ((count - *offset) < header.size)
610 return -ENOSPC;
611
612 buf += *offset;
613 if (copy_to_user(buf, &header, sizeof(header)))
614 return -EFAULT;
615 buf += sizeof(header);
616
617 if (sample_flags & SAMPLE_OA_REPORT) {
618 if (copy_to_user(buf, report, report_size))
619 return -EFAULT;
620 }
621
622 (*offset) += header.size;
623
624 return 0;
625}
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647static int gen8_append_oa_reports(struct i915_perf_stream *stream,
648 char __user *buf,
649 size_t count,
650 size_t *offset)
651{
652 struct intel_uncore *uncore = stream->uncore;
653 int report_size = stream->oa_buffer.format_size;
654 u8 *oa_buf_base = stream->oa_buffer.vaddr;
655 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
656 u32 mask = (OA_BUFFER_SIZE - 1);
657 size_t start_offset = *offset;
658 unsigned long flags;
659 u32 head, tail;
660 u32 taken;
661 int ret = 0;
662
663 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
664 return -EIO;
665
666 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
667
668 head = stream->oa_buffer.head;
669 tail = stream->oa_buffer.tail;
670
671 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
672
673
674
675
676
677 head -= gtt_offset;
678 tail -= gtt_offset;
679
680
681
682
683
684
685
686
687 if (drm_WARN_ONCE(&uncore->i915->drm,
688 head > OA_BUFFER_SIZE || head % report_size ||
689 tail > OA_BUFFER_SIZE || tail % report_size,
690 "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
691 head, tail))
692 return -EIO;
693
694
695 for (;
696 (taken = OA_TAKEN(tail, head));
697 head = (head + report_size) & mask) {
698 u8 *report = oa_buf_base + head;
699 u32 *report32 = (void *)report;
700 u32 ctx_id;
701 u32 reason;
702
703
704
705
706
707
708
709
710
711
712 if (drm_WARN_ON(&uncore->i915->drm,
713 (OA_BUFFER_SIZE - head) < report_size)) {
714 drm_err(&uncore->i915->drm,
715 "Spurious OA head ptr: non-integral report offset\n");
716 break;
717 }
718
719
720
721
722
723
724
725
726
727
728 reason = ((report32[0] >> OAREPORT_REASON_SHIFT) &
729 (IS_GEN(stream->perf->i915, 12) ?
730 OAREPORT_REASON_MASK_EXTENDED :
731 OAREPORT_REASON_MASK));
732 if (reason == 0) {
733 if (__ratelimit(&stream->perf->spurious_report_rs))
734 DRM_NOTE("Skipping spurious, invalid OA report\n");
735 continue;
736 }
737
738 ctx_id = report32[2] & stream->specific_ctx_id_mask;
739
740
741
742
743
744
745
746
747
748 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) &&
749 INTEL_GEN(stream->perf->i915) <= 11)
750 ctx_id = report32[2] = INVALID_CTX_ID;
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783 if (!stream->perf->exclusive_stream->ctx ||
784 stream->specific_ctx_id == ctx_id ||
785 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||
786 reason & OAREPORT_REASON_CTX_SWITCH) {
787
788
789
790
791
792 if (stream->perf->exclusive_stream->ctx &&
793 stream->specific_ctx_id != ctx_id) {
794 report32[2] = INVALID_CTX_ID;
795 }
796
797 ret = append_oa_sample(stream, buf, count, offset,
798 report);
799 if (ret)
800 break;
801
802 stream->oa_buffer.last_ctx_id = ctx_id;
803 }
804
805
806
807
808
809 report32[0] = 0;
810 report32[1] = 0;
811 }
812
813 if (start_offset != *offset) {
814 i915_reg_t oaheadptr;
815
816 oaheadptr = IS_GEN(stream->perf->i915, 12) ?
817 GEN12_OAG_OAHEADPTR : GEN8_OAHEADPTR;
818
819 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
820
821
822
823
824
825 head += gtt_offset;
826 intel_uncore_write(uncore, oaheadptr,
827 head & GEN12_OAG_OAHEADPTR_MASK);
828 stream->oa_buffer.head = head;
829
830 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
831 }
832
833 return ret;
834}
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856static int gen8_oa_read(struct i915_perf_stream *stream,
857 char __user *buf,
858 size_t count,
859 size_t *offset)
860{
861 struct intel_uncore *uncore = stream->uncore;
862 u32 oastatus;
863 i915_reg_t oastatus_reg;
864 int ret;
865
866 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
867 return -EIO;
868
869 oastatus_reg = IS_GEN(stream->perf->i915, 12) ?
870 GEN12_OAG_OASTATUS : GEN8_OASTATUS;
871
872 oastatus = intel_uncore_read(uncore, oastatus_reg);
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888 if (oastatus & GEN8_OASTATUS_OABUFFER_OVERFLOW) {
889 ret = append_oa_status(stream, buf, count, offset,
890 DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
891 if (ret)
892 return ret;
893
894 DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n",
895 stream->period_exponent);
896
897 stream->perf->ops.oa_disable(stream);
898 stream->perf->ops.oa_enable(stream);
899
900
901
902
903
904 oastatus = intel_uncore_read(uncore, oastatus_reg);
905 }
906
907 if (oastatus & GEN8_OASTATUS_REPORT_LOST) {
908 ret = append_oa_status(stream, buf, count, offset,
909 DRM_I915_PERF_RECORD_OA_REPORT_LOST);
910 if (ret)
911 return ret;
912 intel_uncore_write(uncore, oastatus_reg,
913 oastatus & ~GEN8_OASTATUS_REPORT_LOST);
914 }
915
916 return gen8_append_oa_reports(stream, buf, count, offset);
917}
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939static int gen7_append_oa_reports(struct i915_perf_stream *stream,
940 char __user *buf,
941 size_t count,
942 size_t *offset)
943{
944 struct intel_uncore *uncore = stream->uncore;
945 int report_size = stream->oa_buffer.format_size;
946 u8 *oa_buf_base = stream->oa_buffer.vaddr;
947 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
948 u32 mask = (OA_BUFFER_SIZE - 1);
949 size_t start_offset = *offset;
950 unsigned long flags;
951 u32 head, tail;
952 u32 taken;
953 int ret = 0;
954
955 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
956 return -EIO;
957
958 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
959
960 head = stream->oa_buffer.head;
961 tail = stream->oa_buffer.tail;
962
963 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
964
965
966
967
968 head -= gtt_offset;
969 tail -= gtt_offset;
970
971
972
973
974
975
976
977 if (drm_WARN_ONCE(&uncore->i915->drm,
978 head > OA_BUFFER_SIZE || head % report_size ||
979 tail > OA_BUFFER_SIZE || tail % report_size,
980 "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
981 head, tail))
982 return -EIO;
983
984
985 for (;
986 (taken = OA_TAKEN(tail, head));
987 head = (head + report_size) & mask) {
988 u8 *report = oa_buf_base + head;
989 u32 *report32 = (void *)report;
990
991
992
993
994
995
996
997
998
999 if (drm_WARN_ON(&uncore->i915->drm,
1000 (OA_BUFFER_SIZE - head) < report_size)) {
1001 drm_err(&uncore->i915->drm,
1002 "Spurious OA head ptr: non-integral report offset\n");
1003 break;
1004 }
1005
1006
1007
1008
1009
1010
1011
1012 if (report32[0] == 0) {
1013 if (__ratelimit(&stream->perf->spurious_report_rs))
1014 DRM_NOTE("Skipping spurious, invalid OA report\n");
1015 continue;
1016 }
1017
1018 ret = append_oa_sample(stream, buf, count, offset, report);
1019 if (ret)
1020 break;
1021
1022
1023
1024
1025 report32[0] = 0;
1026 report32[1] = 0;
1027 }
1028
1029 if (start_offset != *offset) {
1030 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1031
1032
1033
1034
1035 head += gtt_offset;
1036
1037 intel_uncore_write(uncore, GEN7_OASTATUS2,
1038 (head & GEN7_OASTATUS2_HEAD_MASK) |
1039 GEN7_OASTATUS2_MEM_SELECT_GGTT);
1040 stream->oa_buffer.head = head;
1041
1042 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1043 }
1044
1045 return ret;
1046}
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064static int gen7_oa_read(struct i915_perf_stream *stream,
1065 char __user *buf,
1066 size_t count,
1067 size_t *offset)
1068{
1069 struct intel_uncore *uncore = stream->uncore;
1070 u32 oastatus1;
1071 int ret;
1072
1073 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
1074 return -EIO;
1075
1076 oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
1077
1078
1079
1080
1081
1082
1083 oastatus1 &= ~stream->perf->gen7_latched_oastatus1;
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105 if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) {
1106 ret = append_oa_status(stream, buf, count, offset,
1107 DRM_I915_PERF_RECORD_OA_BUFFER_LOST);
1108 if (ret)
1109 return ret;
1110
1111 DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n",
1112 stream->period_exponent);
1113
1114 stream->perf->ops.oa_disable(stream);
1115 stream->perf->ops.oa_enable(stream);
1116
1117 oastatus1 = intel_uncore_read(uncore, GEN7_OASTATUS1);
1118 }
1119
1120 if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) {
1121 ret = append_oa_status(stream, buf, count, offset,
1122 DRM_I915_PERF_RECORD_OA_REPORT_LOST);
1123 if (ret)
1124 return ret;
1125 stream->perf->gen7_latched_oastatus1 |=
1126 GEN7_OASTATUS1_REPORT_LOST;
1127 }
1128
1129 return gen7_append_oa_reports(stream, buf, count, offset);
1130}
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
1147{
1148
1149 if (!stream->periodic)
1150 return -EIO;
1151
1152 return wait_event_interruptible(stream->poll_wq,
1153 oa_buffer_check_unlocked(stream));
1154}
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166static void i915_oa_poll_wait(struct i915_perf_stream *stream,
1167 struct file *file,
1168 poll_table *wait)
1169{
1170 poll_wait(file, &stream->poll_wq, wait);
1171}
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185static int i915_oa_read(struct i915_perf_stream *stream,
1186 char __user *buf,
1187 size_t count,
1188 size_t *offset)
1189{
1190 return stream->perf->ops.read(stream, buf, count, offset);
1191}
1192
1193static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
1194{
1195 struct i915_gem_engines_iter it;
1196 struct i915_gem_context *ctx = stream->ctx;
1197 struct intel_context *ce;
1198 int err;
1199
1200 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1201 if (ce->engine != stream->engine)
1202 continue;
1203
1204
1205
1206
1207
1208 err = intel_context_pin(ce);
1209 if (err == 0) {
1210 stream->pinned_ctx = ce;
1211 break;
1212 }
1213 }
1214 i915_gem_context_unlock_engines(ctx);
1215
1216 return stream->pinned_ctx;
1217}
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
1230{
1231 struct intel_context *ce;
1232
1233 ce = oa_pin_context(stream);
1234 if (IS_ERR(ce))
1235 return PTR_ERR(ce);
1236
1237 switch (INTEL_GEN(ce->engine->i915)) {
1238 case 7: {
1239
1240
1241
1242
1243 stream->specific_ctx_id = i915_ggtt_offset(ce->state);
1244 stream->specific_ctx_id_mask = 0;
1245 break;
1246 }
1247
1248 case 8:
1249 case 9:
1250 case 10:
1251 if (intel_engine_in_execlists_submission_mode(ce->engine)) {
1252 stream->specific_ctx_id_mask =
1253 (1U << GEN8_CTX_ID_WIDTH) - 1;
1254 stream->specific_ctx_id = stream->specific_ctx_id_mask;
1255 } else {
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266 stream->specific_ctx_id = ce->lrc.lrca >> 12;
1267
1268
1269
1270
1271
1272 stream->specific_ctx_id_mask =
1273 (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1;
1274 }
1275 break;
1276
1277 case 11:
1278 case 12: {
1279 stream->specific_ctx_id_mask =
1280 ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32);
1281
1282
1283
1284
1285
1286 stream->specific_ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32);
1287 break;
1288 }
1289
1290 default:
1291 MISSING_CASE(INTEL_GEN(ce->engine->i915));
1292 }
1293
1294 ce->tag = stream->specific_ctx_id;
1295
1296 drm_dbg(&stream->perf->i915->drm,
1297 "filtering on ctx_id=0x%x ctx_id_mask=0x%x\n",
1298 stream->specific_ctx_id,
1299 stream->specific_ctx_id_mask);
1300
1301 return 0;
1302}
1303
1304
1305
1306
1307
1308
1309
1310
1311static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
1312{
1313 struct intel_context *ce;
1314
1315 ce = fetch_and_zero(&stream->pinned_ctx);
1316 if (ce) {
1317 ce->tag = 0;
1318 intel_context_unpin(ce);
1319 }
1320
1321 stream->specific_ctx_id = INVALID_CTX_ID;
1322 stream->specific_ctx_id_mask = 0;
1323}
1324
1325static void
1326free_oa_buffer(struct i915_perf_stream *stream)
1327{
1328 i915_vma_unpin_and_release(&stream->oa_buffer.vma,
1329 I915_VMA_RELEASE_MAP);
1330
1331 stream->oa_buffer.vaddr = NULL;
1332}
1333
1334static void
1335free_oa_configs(struct i915_perf_stream *stream)
1336{
1337 struct i915_oa_config_bo *oa_bo, *tmp;
1338
1339 i915_oa_config_put(stream->oa_config);
1340 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
1341 free_oa_config_bo(oa_bo);
1342}
1343
1344static void
1345free_noa_wait(struct i915_perf_stream *stream)
1346{
1347 i915_vma_unpin_and_release(&stream->noa_wait, 0);
1348}
1349
1350static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1351{
1352 struct i915_perf *perf = stream->perf;
1353
1354 BUG_ON(stream != perf->exclusive_stream);
1355
1356
1357
1358
1359
1360
1361
1362 WRITE_ONCE(perf->exclusive_stream, NULL);
1363 perf->ops.disable_metric_set(stream);
1364
1365 free_oa_buffer(stream);
1366
1367 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
1368 intel_engine_pm_put(stream->engine);
1369
1370 if (stream->ctx)
1371 oa_put_render_ctx_id(stream);
1372
1373 free_oa_configs(stream);
1374 free_noa_wait(stream);
1375
1376 if (perf->spurious_report_rs.missed) {
1377 DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n",
1378 perf->spurious_report_rs.missed);
1379 }
1380}
1381
1382static void gen7_init_oa_buffer(struct i915_perf_stream *stream)
1383{
1384 struct intel_uncore *uncore = stream->uncore;
1385 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1386 unsigned long flags;
1387
1388 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1389
1390
1391
1392
1393 intel_uncore_write(uncore, GEN7_OASTATUS2,
1394 gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT);
1395 stream->oa_buffer.head = gtt_offset;
1396
1397 intel_uncore_write(uncore, GEN7_OABUFFER, gtt_offset);
1398
1399 intel_uncore_write(uncore, GEN7_OASTATUS1,
1400 gtt_offset | OABUFFER_SIZE_16M);
1401
1402
1403 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;
1404 stream->oa_buffer.tail = gtt_offset;
1405
1406 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1407
1408
1409
1410
1411
1412 stream->perf->gen7_latched_oastatus1 = 0;
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1426}
1427
1428static void gen8_init_oa_buffer(struct i915_perf_stream *stream)
1429{
1430 struct intel_uncore *uncore = stream->uncore;
1431 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1432 unsigned long flags;
1433
1434 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1435
1436 intel_uncore_write(uncore, GEN8_OASTATUS, 0);
1437 intel_uncore_write(uncore, GEN8_OAHEADPTR, gtt_offset);
1438 stream->oa_buffer.head = gtt_offset;
1439
1440 intel_uncore_write(uncore, GEN8_OABUFFER_UDW, 0);
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450 intel_uncore_write(uncore, GEN8_OABUFFER, gtt_offset |
1451 OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
1452 intel_uncore_write(uncore, GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK);
1453
1454
1455 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;
1456 stream->oa_buffer.tail = gtt_offset;
1457
1458
1459
1460
1461
1462
1463 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1464
1465 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1480}
1481
1482static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
1483{
1484 struct intel_uncore *uncore = stream->uncore;
1485 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1486 unsigned long flags;
1487
1488 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1489
1490 intel_uncore_write(uncore, GEN12_OAG_OASTATUS, 0);
1491 intel_uncore_write(uncore, GEN12_OAG_OAHEADPTR,
1492 gtt_offset & GEN12_OAG_OAHEADPTR_MASK);
1493 stream->oa_buffer.head = gtt_offset;
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503 intel_uncore_write(uncore, GEN12_OAG_OABUFFER, gtt_offset |
1504 OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
1505 intel_uncore_write(uncore, GEN12_OAG_OATAILPTR,
1506 gtt_offset & GEN12_OAG_OATAILPTR_MASK);
1507
1508
1509 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;
1510 stream->oa_buffer.tail = gtt_offset;
1511
1512
1513
1514
1515
1516
1517 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1518
1519 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533 memset(stream->oa_buffer.vaddr, 0,
1534 stream->oa_buffer.vma->size);
1535}
1536
1537static int alloc_oa_buffer(struct i915_perf_stream *stream)
1538{
1539 struct drm_i915_private *i915 = stream->perf->i915;
1540 struct drm_i915_gem_object *bo;
1541 struct i915_vma *vma;
1542 int ret;
1543
1544 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))
1545 return -ENODEV;
1546
1547 BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE);
1548 BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
1549
1550 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE);
1551 if (IS_ERR(bo)) {
1552 drm_err(&i915->drm, "Failed to allocate OA buffer\n");
1553 return PTR_ERR(bo);
1554 }
1555
1556 i915_gem_object_set_cache_coherency(bo, I915_CACHE_LLC);
1557
1558
1559 vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0);
1560 if (IS_ERR(vma)) {
1561 ret = PTR_ERR(vma);
1562 goto err_unref;
1563 }
1564 stream->oa_buffer.vma = vma;
1565
1566 stream->oa_buffer.vaddr =
1567 i915_gem_object_pin_map(bo, I915_MAP_WB);
1568 if (IS_ERR(stream->oa_buffer.vaddr)) {
1569 ret = PTR_ERR(stream->oa_buffer.vaddr);
1570 goto err_unpin;
1571 }
1572
1573 return 0;
1574
1575err_unpin:
1576 __i915_vma_unpin(vma);
1577
1578err_unref:
1579 i915_gem_object_put(bo);
1580
1581 stream->oa_buffer.vaddr = NULL;
1582 stream->oa_buffer.vma = NULL;
1583
1584 return ret;
1585}
1586
1587static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs,
1588 bool save, i915_reg_t reg, u32 offset,
1589 u32 dword_count)
1590{
1591 u32 cmd;
1592 u32 d;
1593
1594 cmd = save ? MI_STORE_REGISTER_MEM : MI_LOAD_REGISTER_MEM;
1595 cmd |= MI_SRM_LRM_GLOBAL_GTT;
1596 if (INTEL_GEN(stream->perf->i915) >= 8)
1597 cmd++;
1598
1599 for (d = 0; d < dword_count; d++) {
1600 *cs++ = cmd;
1601 *cs++ = i915_mmio_reg_offset(reg) + 4 * d;
1602 *cs++ = intel_gt_scratch_offset(stream->engine->gt,
1603 offset) + 4 * d;
1604 *cs++ = 0;
1605 }
1606
1607 return cs;
1608}
1609
1610static int alloc_noa_wait(struct i915_perf_stream *stream)
1611{
1612 struct drm_i915_private *i915 = stream->perf->i915;
1613 struct drm_i915_gem_object *bo;
1614 struct i915_vma *vma;
1615 const u64 delay_ticks = 0xffffffffffffffff -
1616 i915_cs_timestamp_ns_to_ticks(i915, atomic64_read(&stream->perf->noa_programming_delay));
1617 const u32 base = stream->engine->mmio_base;
1618#define CS_GPR(x) GEN8_RING_CS_GPR(base, x)
1619 u32 *batch, *ts0, *cs, *jump;
1620 int ret, i;
1621 enum {
1622 START_TS,
1623 NOW_TS,
1624 DELTA_TS,
1625 JUMP_PREDICATE,
1626 DELTA_TARGET,
1627 N_CS_GPR
1628 };
1629
1630 bo = i915_gem_object_create_internal(i915, 4096);
1631 if (IS_ERR(bo)) {
1632 drm_err(&i915->drm,
1633 "Failed to allocate NOA wait batchbuffer\n");
1634 return PTR_ERR(bo);
1635 }
1636
1637
1638
1639
1640
1641
1642 vma = i915_gem_object_ggtt_pin(bo, NULL, 0, 0, PIN_HIGH);
1643 if (IS_ERR(vma)) {
1644 ret = PTR_ERR(vma);
1645 goto err_unref;
1646 }
1647
1648 batch = cs = i915_gem_object_pin_map(bo, I915_MAP_WB);
1649 if (IS_ERR(batch)) {
1650 ret = PTR_ERR(batch);
1651 goto err_unpin;
1652 }
1653
1654
1655 for (i = 0; i < N_CS_GPR; i++)
1656 cs = save_restore_register(
1657 stream, cs, true , CS_GPR(i),
1658 INTEL_GT_SCRATCH_FIELD_PERF_CS_GPR + 8 * i, 2);
1659 cs = save_restore_register(
1660 stream, cs, true , MI_PREDICATE_RESULT_1,
1661 INTEL_GT_SCRATCH_FIELD_PERF_PREDICATE_RESULT_1, 1);
1662
1663
1664 ts0 = cs;
1665
1666
1667
1668
1669
1670
1671 *cs++ = MI_LOAD_REGISTER_IMM(1);
1672 *cs++ = i915_mmio_reg_offset(CS_GPR(START_TS)) + 4;
1673 *cs++ = 0;
1674 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
1675 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(base));
1676 *cs++ = i915_mmio_reg_offset(CS_GPR(START_TS));
1677
1678
1679
1680
1681
1682 jump = cs;
1683
1684
1685
1686
1687
1688
1689 *cs++ = MI_LOAD_REGISTER_IMM(1);
1690 *cs++ = i915_mmio_reg_offset(CS_GPR(NOW_TS)) + 4;
1691 *cs++ = 0;
1692 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
1693 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(base));
1694 *cs++ = i915_mmio_reg_offset(CS_GPR(NOW_TS));
1695
1696
1697
1698
1699
1700 *cs++ = MI_MATH(5);
1701 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(NOW_TS));
1702 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(START_TS));
1703 *cs++ = MI_MATH_SUB;
1704 *cs++ = MI_MATH_STORE(MI_MATH_REG(DELTA_TS), MI_MATH_REG_ACCU);
1705 *cs++ = MI_MATH_STORE(MI_MATH_REG(JUMP_PREDICATE), MI_MATH_REG_CF);
1706
1707
1708
1709
1710
1711
1712 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
1713 *cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE));
1714 *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1);
1715
1716
1717 *cs++ = (INTEL_GEN(i915) < 8 ?
1718 MI_BATCH_BUFFER_START :
1719 MI_BATCH_BUFFER_START_GEN8) |
1720 MI_BATCH_PREDICATE;
1721 *cs++ = i915_ggtt_offset(vma) + (ts0 - batch) * 4;
1722 *cs++ = 0;
1723
1724
1725
1726
1727
1728
1729
1730
1731 *cs++ = MI_LOAD_REGISTER_IMM(2);
1732 *cs++ = i915_mmio_reg_offset(CS_GPR(DELTA_TARGET));
1733 *cs++ = lower_32_bits(delay_ticks);
1734 *cs++ = i915_mmio_reg_offset(CS_GPR(DELTA_TARGET)) + 4;
1735 *cs++ = upper_32_bits(delay_ticks);
1736
1737 *cs++ = MI_MATH(4);
1738 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(DELTA_TS));
1739 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(DELTA_TARGET));
1740 *cs++ = MI_MATH_ADD;
1741 *cs++ = MI_MATH_STOREINV(MI_MATH_REG(JUMP_PREDICATE), MI_MATH_REG_CF);
1742
1743 *cs++ = MI_ARB_CHECK;
1744
1745
1746
1747
1748
1749 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2);
1750 *cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE));
1751 *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1);
1752
1753
1754 *cs++ = (INTEL_GEN(i915) < 8 ?
1755 MI_BATCH_BUFFER_START :
1756 MI_BATCH_BUFFER_START_GEN8) |
1757 MI_BATCH_PREDICATE;
1758 *cs++ = i915_ggtt_offset(vma) + (jump - batch) * 4;
1759 *cs++ = 0;
1760
1761
1762 for (i = 0; i < N_CS_GPR; i++)
1763 cs = save_restore_register(
1764 stream, cs, false , CS_GPR(i),
1765 INTEL_GT_SCRATCH_FIELD_PERF_CS_GPR + 8 * i, 2);
1766 cs = save_restore_register(
1767 stream, cs, false , MI_PREDICATE_RESULT_1,
1768 INTEL_GT_SCRATCH_FIELD_PERF_PREDICATE_RESULT_1, 1);
1769
1770
1771 *cs++ = MI_BATCH_BUFFER_END;
1772
1773 GEM_BUG_ON(cs - batch > PAGE_SIZE / sizeof(*batch));
1774
1775 i915_gem_object_flush_map(bo);
1776 __i915_gem_object_release_map(bo);
1777
1778 stream->noa_wait = vma;
1779 return 0;
1780
1781err_unpin:
1782 i915_vma_unpin_and_release(&vma, 0);
1783err_unref:
1784 i915_gem_object_put(bo);
1785 return ret;
1786}
1787
1788static u32 *write_cs_mi_lri(u32 *cs,
1789 const struct i915_oa_reg *reg_data,
1790 u32 n_regs)
1791{
1792 u32 i;
1793
1794 for (i = 0; i < n_regs; i++) {
1795 if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) {
1796 u32 n_lri = min_t(u32,
1797 n_regs - i,
1798 MI_LOAD_REGISTER_IMM_MAX_REGS);
1799
1800 *cs++ = MI_LOAD_REGISTER_IMM(n_lri);
1801 }
1802 *cs++ = i915_mmio_reg_offset(reg_data[i].addr);
1803 *cs++ = reg_data[i].value;
1804 }
1805
1806 return cs;
1807}
1808
1809static int num_lri_dwords(int num_regs)
1810{
1811 int count = 0;
1812
1813 if (num_regs > 0) {
1814 count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS);
1815 count += num_regs * 2;
1816 }
1817
1818 return count;
1819}
1820
1821static struct i915_oa_config_bo *
1822alloc_oa_config_buffer(struct i915_perf_stream *stream,
1823 struct i915_oa_config *oa_config)
1824{
1825 struct drm_i915_gem_object *obj;
1826 struct i915_oa_config_bo *oa_bo;
1827 size_t config_length = 0;
1828 u32 *cs;
1829 int err;
1830
1831 oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL);
1832 if (!oa_bo)
1833 return ERR_PTR(-ENOMEM);
1834
1835 config_length += num_lri_dwords(oa_config->mux_regs_len);
1836 config_length += num_lri_dwords(oa_config->b_counter_regs_len);
1837 config_length += num_lri_dwords(oa_config->flex_regs_len);
1838 config_length += 3;
1839 config_length = ALIGN(sizeof(u32) * config_length, I915_GTT_PAGE_SIZE);
1840
1841 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length);
1842 if (IS_ERR(obj)) {
1843 err = PTR_ERR(obj);
1844 goto err_free;
1845 }
1846
1847 cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
1848 if (IS_ERR(cs)) {
1849 err = PTR_ERR(cs);
1850 goto err_oa_bo;
1851 }
1852
1853 cs = write_cs_mi_lri(cs,
1854 oa_config->mux_regs,
1855 oa_config->mux_regs_len);
1856 cs = write_cs_mi_lri(cs,
1857 oa_config->b_counter_regs,
1858 oa_config->b_counter_regs_len);
1859 cs = write_cs_mi_lri(cs,
1860 oa_config->flex_regs,
1861 oa_config->flex_regs_len);
1862
1863
1864 *cs++ = (INTEL_GEN(stream->perf->i915) < 8 ?
1865 MI_BATCH_BUFFER_START :
1866 MI_BATCH_BUFFER_START_GEN8);
1867 *cs++ = i915_ggtt_offset(stream->noa_wait);
1868 *cs++ = 0;
1869
1870 i915_gem_object_flush_map(obj);
1871 __i915_gem_object_release_map(obj);
1872
1873 oa_bo->vma = i915_vma_instance(obj,
1874 &stream->engine->gt->ggtt->vm,
1875 NULL);
1876 if (IS_ERR(oa_bo->vma)) {
1877 err = PTR_ERR(oa_bo->vma);
1878 goto err_oa_bo;
1879 }
1880
1881 oa_bo->oa_config = i915_oa_config_get(oa_config);
1882 llist_add(&oa_bo->node, &stream->oa_config_bos);
1883
1884 return oa_bo;
1885
1886err_oa_bo:
1887 i915_gem_object_put(obj);
1888err_free:
1889 kfree(oa_bo);
1890 return ERR_PTR(err);
1891}
1892
1893static struct i915_vma *
1894get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config)
1895{
1896 struct i915_oa_config_bo *oa_bo;
1897
1898
1899
1900
1901
1902 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
1903 if (oa_bo->oa_config == oa_config &&
1904 memcmp(oa_bo->oa_config->uuid,
1905 oa_config->uuid,
1906 sizeof(oa_config->uuid)) == 0)
1907 goto out;
1908 }
1909
1910 oa_bo = alloc_oa_config_buffer(stream, oa_config);
1911 if (IS_ERR(oa_bo))
1912 return ERR_CAST(oa_bo);
1913
1914out:
1915 return i915_vma_get(oa_bo->vma);
1916}
1917
1918static int
1919emit_oa_config(struct i915_perf_stream *stream,
1920 struct i915_oa_config *oa_config,
1921 struct intel_context *ce,
1922 struct i915_active *active)
1923{
1924 struct i915_request *rq;
1925 struct i915_vma *vma;
1926 int err;
1927
1928 vma = get_oa_vma(stream, oa_config);
1929 if (IS_ERR(vma))
1930 return PTR_ERR(vma);
1931
1932 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
1933 if (err)
1934 goto err_vma_put;
1935
1936 intel_engine_pm_get(ce->engine);
1937 rq = i915_request_create(ce);
1938 intel_engine_pm_put(ce->engine);
1939 if (IS_ERR(rq)) {
1940 err = PTR_ERR(rq);
1941 goto err_vma_unpin;
1942 }
1943
1944 if (!IS_ERR_OR_NULL(active)) {
1945
1946 err = i915_request_await_active(rq, active,
1947 I915_ACTIVE_AWAIT_ACTIVE);
1948 if (err)
1949 goto err_add_request;
1950
1951 err = i915_active_add_request(active, rq);
1952 if (err)
1953 goto err_add_request;
1954 }
1955
1956 i915_vma_lock(vma);
1957 err = i915_request_await_object(rq, vma->obj, 0);
1958 if (!err)
1959 err = i915_vma_move_to_active(vma, rq, 0);
1960 i915_vma_unlock(vma);
1961 if (err)
1962 goto err_add_request;
1963
1964 err = rq->engine->emit_bb_start(rq,
1965 vma->node.start, 0,
1966 I915_DISPATCH_SECURE);
1967 if (err)
1968 goto err_add_request;
1969
1970err_add_request:
1971 i915_request_add(rq);
1972err_vma_unpin:
1973 i915_vma_unpin(vma);
1974err_vma_put:
1975 i915_vma_put(vma);
1976 return err;
1977}
1978
1979static struct intel_context *oa_context(struct i915_perf_stream *stream)
1980{
1981 return stream->pinned_ctx ?: stream->engine->kernel_context;
1982}
1983
1984static int
1985hsw_enable_metric_set(struct i915_perf_stream *stream,
1986 struct i915_active *active)
1987{
1988 struct intel_uncore *uncore = stream->uncore;
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000 intel_uncore_rmw(uncore, GEN7_MISCCPCTL,
2001 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
2002 intel_uncore_rmw(uncore, GEN6_UCGCTL1,
2003 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
2004
2005 return emit_oa_config(stream,
2006 stream->oa_config, oa_context(stream),
2007 active);
2008}
2009
2010static void hsw_disable_metric_set(struct i915_perf_stream *stream)
2011{
2012 struct intel_uncore *uncore = stream->uncore;
2013
2014 intel_uncore_rmw(uncore, GEN6_UCGCTL1,
2015 GEN6_CSUNIT_CLOCK_GATE_DISABLE, 0);
2016 intel_uncore_rmw(uncore, GEN7_MISCCPCTL,
2017 0, GEN7_DOP_CLOCK_GATE_ENABLE);
2018
2019 intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, 0);
2020}
2021
2022static u32 oa_config_flex_reg(const struct i915_oa_config *oa_config,
2023 i915_reg_t reg)
2024{
2025 u32 mmio = i915_mmio_reg_offset(reg);
2026 int i;
2027
2028
2029
2030
2031
2032
2033 if (!oa_config)
2034 return 0;
2035
2036 for (i = 0; i < oa_config->flex_regs_len; i++) {
2037 if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio)
2038 return oa_config->flex_regs[i].value;
2039 }
2040
2041 return 0;
2042}
2043
2044
2045
2046
2047
2048
2049
2050static void
2051gen8_update_reg_state_unlocked(const struct intel_context *ce,
2052 const struct i915_perf_stream *stream)
2053{
2054 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2055 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2056
2057 i915_reg_t flex_regs[] = {
2058 EU_PERF_CNTL0,
2059 EU_PERF_CNTL1,
2060 EU_PERF_CNTL2,
2061 EU_PERF_CNTL3,
2062 EU_PERF_CNTL4,
2063 EU_PERF_CNTL5,
2064 EU_PERF_CNTL6,
2065 };
2066 u32 *reg_state = ce->lrc_reg_state;
2067 int i;
2068
2069 reg_state[ctx_oactxctrl + 1] =
2070 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2071 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2072 GEN8_OA_COUNTER_RESUME;
2073
2074 for (i = 0; i < ARRAY_SIZE(flex_regs); i++)
2075 reg_state[ctx_flexeu0 + i * 2 + 1] =
2076 oa_config_flex_reg(stream->oa_config, flex_regs[i]);
2077}
2078
2079struct flex {
2080 i915_reg_t reg;
2081 u32 offset;
2082 u32 value;
2083};
2084
2085static int
2086gen8_store_flex(struct i915_request *rq,
2087 struct intel_context *ce,
2088 const struct flex *flex, unsigned int count)
2089{
2090 u32 offset;
2091 u32 *cs;
2092
2093 cs = intel_ring_begin(rq, 4 * count);
2094 if (IS_ERR(cs))
2095 return PTR_ERR(cs);
2096
2097 offset = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET;
2098 do {
2099 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
2100 *cs++ = offset + flex->offset * sizeof(u32);
2101 *cs++ = 0;
2102 *cs++ = flex->value;
2103 } while (flex++, --count);
2104
2105 intel_ring_advance(rq, cs);
2106
2107 return 0;
2108}
2109
2110static int
2111gen8_load_flex(struct i915_request *rq,
2112 struct intel_context *ce,
2113 const struct flex *flex, unsigned int count)
2114{
2115 u32 *cs;
2116
2117 GEM_BUG_ON(!count || count > 63);
2118
2119 cs = intel_ring_begin(rq, 2 * count + 2);
2120 if (IS_ERR(cs))
2121 return PTR_ERR(cs);
2122
2123 *cs++ = MI_LOAD_REGISTER_IMM(count);
2124 do {
2125 *cs++ = i915_mmio_reg_offset(flex->reg);
2126 *cs++ = flex->value;
2127 } while (flex++, --count);
2128 *cs++ = MI_NOOP;
2129
2130 intel_ring_advance(rq, cs);
2131
2132 return 0;
2133}
2134
2135static int gen8_modify_context(struct intel_context *ce,
2136 const struct flex *flex, unsigned int count)
2137{
2138 struct i915_request *rq;
2139 int err;
2140
2141 rq = intel_engine_create_kernel_request(ce->engine);
2142 if (IS_ERR(rq))
2143 return PTR_ERR(rq);
2144
2145
2146 err = intel_context_prepare_remote_request(ce, rq);
2147 if (err == 0)
2148 err = gen8_store_flex(rq, ce, flex, count);
2149
2150 i915_request_add(rq);
2151 return err;
2152}
2153
2154static int
2155gen8_modify_self(struct intel_context *ce,
2156 const struct flex *flex, unsigned int count,
2157 struct i915_active *active)
2158{
2159 struct i915_request *rq;
2160 int err;
2161
2162 intel_engine_pm_get(ce->engine);
2163 rq = i915_request_create(ce);
2164 intel_engine_pm_put(ce->engine);
2165 if (IS_ERR(rq))
2166 return PTR_ERR(rq);
2167
2168 if (!IS_ERR_OR_NULL(active)) {
2169 err = i915_active_add_request(active, rq);
2170 if (err)
2171 goto err_add_request;
2172 }
2173
2174 err = gen8_load_flex(rq, ce, flex, count);
2175 if (err)
2176 goto err_add_request;
2177
2178err_add_request:
2179 i915_request_add(rq);
2180 return err;
2181}
2182
2183static int gen8_configure_context(struct i915_gem_context *ctx,
2184 struct flex *flex, unsigned int count)
2185{
2186 struct i915_gem_engines_iter it;
2187 struct intel_context *ce;
2188 int err = 0;
2189
2190 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
2191 GEM_BUG_ON(ce == ce->engine->kernel_context);
2192
2193 if (ce->engine->class != RENDER_CLASS)
2194 continue;
2195
2196
2197 if (!intel_context_pin_if_active(ce))
2198 continue;
2199
2200 flex->value = intel_sseu_make_rpcs(ce->engine->gt, &ce->sseu);
2201 err = gen8_modify_context(ce, flex, count);
2202
2203 intel_context_unpin(ce);
2204 if (err)
2205 break;
2206 }
2207 i915_gem_context_unlock_engines(ctx);
2208
2209 return err;
2210}
2211
2212static int gen12_configure_oar_context(struct i915_perf_stream *stream,
2213 struct i915_active *active)
2214{
2215 int err;
2216 struct intel_context *ce = stream->pinned_ctx;
2217 u32 format = stream->oa_buffer.format;
2218 struct flex regs_context[] = {
2219 {
2220 GEN8_OACTXCONTROL,
2221 stream->perf->ctx_oactxctrl_offset + 1,
2222 active ? GEN8_OA_COUNTER_RESUME : 0,
2223 },
2224 };
2225
2226
2227
2228#define GEN12_OAR_OACONTROL_OFFSET 0x5B0
2229 struct flex regs_lri[] = {
2230 {
2231 GEN12_OAR_OACONTROL,
2232 GEN12_OAR_OACONTROL_OFFSET + 1,
2233 (format << GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT) |
2234 (active ? GEN12_OAR_OACONTROL_COUNTER_ENABLE : 0)
2235 },
2236 {
2237 RING_CONTEXT_CONTROL(ce->engine->mmio_base),
2238 CTX_CONTEXT_CONTROL,
2239 _MASKED_FIELD(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE,
2240 active ?
2241 GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE :
2242 0)
2243 },
2244 };
2245
2246
2247 err = intel_context_lock_pinned(ce);
2248 if (err)
2249 return err;
2250
2251 err = gen8_modify_context(ce, regs_context, ARRAY_SIZE(regs_context));
2252 intel_context_unlock_pinned(ce);
2253 if (err)
2254 return err;
2255
2256
2257 return gen8_modify_self(ce, regs_lri, ARRAY_SIZE(regs_lri), active);
2258}
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285static int
2286oa_configure_all_contexts(struct i915_perf_stream *stream,
2287 struct flex *regs,
2288 size_t num_regs,
2289 struct i915_active *active)
2290{
2291 struct drm_i915_private *i915 = stream->perf->i915;
2292 struct intel_engine_cs *engine;
2293 struct i915_gem_context *ctx, *cn;
2294 int err;
2295
2296 lockdep_assert_held(&stream->perf->lock);
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314 spin_lock(&i915->gem.contexts.lock);
2315 list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) {
2316 if (!kref_get_unless_zero(&ctx->ref))
2317 continue;
2318
2319 spin_unlock(&i915->gem.contexts.lock);
2320
2321 err = gen8_configure_context(ctx, regs, num_regs);
2322 if (err) {
2323 i915_gem_context_put(ctx);
2324 return err;
2325 }
2326
2327 spin_lock(&i915->gem.contexts.lock);
2328 list_safe_reset_next(ctx, cn, link);
2329 i915_gem_context_put(ctx);
2330 }
2331 spin_unlock(&i915->gem.contexts.lock);
2332
2333
2334
2335
2336
2337
2338 for_each_uabi_engine(engine, i915) {
2339 struct intel_context *ce = engine->kernel_context;
2340
2341 if (engine->class != RENDER_CLASS)
2342 continue;
2343
2344 regs[0].value = intel_sseu_make_rpcs(engine->gt, &ce->sseu);
2345
2346 err = gen8_modify_self(ce, regs, num_regs, active);
2347 if (err)
2348 return err;
2349 }
2350
2351 return 0;
2352}
2353
2354static int
2355gen12_configure_all_contexts(struct i915_perf_stream *stream,
2356 const struct i915_oa_config *oa_config,
2357 struct i915_active *active)
2358{
2359 struct flex regs[] = {
2360 {
2361 GEN8_R_PWR_CLK_STATE,
2362 CTX_R_PWR_CLK_STATE,
2363 },
2364 };
2365
2366 return oa_configure_all_contexts(stream,
2367 regs, ARRAY_SIZE(regs),
2368 active);
2369}
2370
2371static int
2372lrc_configure_all_contexts(struct i915_perf_stream *stream,
2373 const struct i915_oa_config *oa_config,
2374 struct i915_active *active)
2375{
2376
2377 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2378#define ctx_flexeuN(N) (ctx_flexeu0 + 2 * (N) + 1)
2379 struct flex regs[] = {
2380 {
2381 GEN8_R_PWR_CLK_STATE,
2382 CTX_R_PWR_CLK_STATE,
2383 },
2384 {
2385 GEN8_OACTXCONTROL,
2386 stream->perf->ctx_oactxctrl_offset + 1,
2387 },
2388 { EU_PERF_CNTL0, ctx_flexeuN(0) },
2389 { EU_PERF_CNTL1, ctx_flexeuN(1) },
2390 { EU_PERF_CNTL2, ctx_flexeuN(2) },
2391 { EU_PERF_CNTL3, ctx_flexeuN(3) },
2392 { EU_PERF_CNTL4, ctx_flexeuN(4) },
2393 { EU_PERF_CNTL5, ctx_flexeuN(5) },
2394 { EU_PERF_CNTL6, ctx_flexeuN(6) },
2395 };
2396#undef ctx_flexeuN
2397 int i;
2398
2399 regs[1].value =
2400 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2401 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2402 GEN8_OA_COUNTER_RESUME;
2403
2404 for (i = 2; i < ARRAY_SIZE(regs); i++)
2405 regs[i].value = oa_config_flex_reg(oa_config, regs[i].reg);
2406
2407 return oa_configure_all_contexts(stream,
2408 regs, ARRAY_SIZE(regs),
2409 active);
2410}
2411
2412static int
2413gen8_enable_metric_set(struct i915_perf_stream *stream,
2414 struct i915_active *active)
2415{
2416 struct intel_uncore *uncore = stream->uncore;
2417 struct i915_oa_config *oa_config = stream->oa_config;
2418 int ret;
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443 if (IS_GEN_RANGE(stream->perf->i915, 9, 11)) {
2444 intel_uncore_write(uncore, GEN8_OA_DEBUG,
2445 _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
2446 GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
2447 }
2448
2449
2450
2451
2452
2453
2454 ret = lrc_configure_all_contexts(stream, oa_config, active);
2455 if (ret)
2456 return ret;
2457
2458 return emit_oa_config(stream,
2459 stream->oa_config, oa_context(stream),
2460 active);
2461}
2462
2463static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
2464{
2465 return _MASKED_FIELD(GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
2466 (stream->sample_flags & SAMPLE_OA_REPORT) ?
2467 0 : GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
2468}
2469
2470static int
2471gen12_enable_metric_set(struct i915_perf_stream *stream,
2472 struct i915_active *active)
2473{
2474 struct intel_uncore *uncore = stream->uncore;
2475 struct i915_oa_config *oa_config = stream->oa_config;
2476 bool periodic = stream->periodic;
2477 u32 period_exponent = stream->period_exponent;
2478 int ret;
2479
2480 intel_uncore_write(uncore, GEN12_OAG_OA_DEBUG,
2481
2482 _MASKED_BIT_ENABLE(GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
2483 GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO) |
2484
2485
2486
2487
2488 oag_report_ctx_switches(stream));
2489
2490 intel_uncore_write(uncore, GEN12_OAG_OAGLBCTXCTRL, periodic ?
2491 (GEN12_OAG_OAGLBCTXCTRL_COUNTER_RESUME |
2492 GEN12_OAG_OAGLBCTXCTRL_TIMER_ENABLE |
2493 (period_exponent << GEN12_OAG_OAGLBCTXCTRL_TIMER_PERIOD_SHIFT))
2494 : 0);
2495
2496
2497
2498
2499
2500
2501 ret = gen12_configure_all_contexts(stream, oa_config, active);
2502 if (ret)
2503 return ret;
2504
2505
2506
2507
2508
2509
2510 if (stream->ctx) {
2511 ret = gen12_configure_oar_context(stream, active);
2512 if (ret)
2513 return ret;
2514 }
2515
2516 return emit_oa_config(stream,
2517 stream->oa_config, oa_context(stream),
2518 active);
2519}
2520
2521static void gen8_disable_metric_set(struct i915_perf_stream *stream)
2522{
2523 struct intel_uncore *uncore = stream->uncore;
2524
2525
2526 lrc_configure_all_contexts(stream, NULL, NULL);
2527
2528 intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, 0);
2529}
2530
2531static void gen10_disable_metric_set(struct i915_perf_stream *stream)
2532{
2533 struct intel_uncore *uncore = stream->uncore;
2534
2535
2536 lrc_configure_all_contexts(stream, NULL, NULL);
2537
2538
2539 intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, 0);
2540}
2541
2542static void gen12_disable_metric_set(struct i915_perf_stream *stream)
2543{
2544 struct intel_uncore *uncore = stream->uncore;
2545
2546
2547 gen12_configure_all_contexts(stream, NULL, NULL);
2548
2549
2550 if (stream->ctx)
2551 gen12_configure_oar_context(stream, NULL);
2552
2553
2554 intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, 0);
2555}
2556
2557static void gen7_oa_enable(struct i915_perf_stream *stream)
2558{
2559 struct intel_uncore *uncore = stream->uncore;
2560 struct i915_gem_context *ctx = stream->ctx;
2561 u32 ctx_id = stream->specific_ctx_id;
2562 bool periodic = stream->periodic;
2563 u32 period_exponent = stream->period_exponent;
2564 u32 report_format = stream->oa_buffer.format;
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575 gen7_init_oa_buffer(stream);
2576
2577 intel_uncore_write(uncore, GEN7_OACONTROL,
2578 (ctx_id & GEN7_OACONTROL_CTX_MASK) |
2579 (period_exponent <<
2580 GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
2581 (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
2582 (report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
2583 (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
2584 GEN7_OACONTROL_ENABLE);
2585}
2586
2587static void gen8_oa_enable(struct i915_perf_stream *stream)
2588{
2589 struct intel_uncore *uncore = stream->uncore;
2590 u32 report_format = stream->oa_buffer.format;
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601 gen8_init_oa_buffer(stream);
2602
2603
2604
2605
2606
2607
2608 intel_uncore_write(uncore, GEN8_OACONTROL,
2609 (report_format << GEN8_OA_REPORT_FORMAT_SHIFT) |
2610 GEN8_OA_COUNTER_ENABLE);
2611}
2612
2613static void gen12_oa_enable(struct i915_perf_stream *stream)
2614{
2615 struct intel_uncore *uncore = stream->uncore;
2616 u32 report_format = stream->oa_buffer.format;
2617
2618
2619
2620
2621
2622 if (!(stream->sample_flags & SAMPLE_OA_REPORT))
2623 return;
2624
2625 gen12_init_oa_buffer(stream);
2626
2627 intel_uncore_write(uncore, GEN12_OAG_OACONTROL,
2628 (report_format << GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT) |
2629 GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE);
2630}
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641static void i915_oa_stream_enable(struct i915_perf_stream *stream)
2642{
2643 stream->pollin = false;
2644
2645 stream->perf->ops.oa_enable(stream);
2646
2647 if (stream->periodic)
2648 hrtimer_start(&stream->poll_check_timer,
2649 ns_to_ktime(stream->poll_oa_period),
2650 HRTIMER_MODE_REL_PINNED);
2651}
2652
2653static void gen7_oa_disable(struct i915_perf_stream *stream)
2654{
2655 struct intel_uncore *uncore = stream->uncore;
2656
2657 intel_uncore_write(uncore, GEN7_OACONTROL, 0);
2658 if (intel_wait_for_register(uncore,
2659 GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0,
2660 50))
2661 drm_err(&stream->perf->i915->drm,
2662 "wait for OA to be disabled timed out\n");
2663}
2664
2665static void gen8_oa_disable(struct i915_perf_stream *stream)
2666{
2667 struct intel_uncore *uncore = stream->uncore;
2668
2669 intel_uncore_write(uncore, GEN8_OACONTROL, 0);
2670 if (intel_wait_for_register(uncore,
2671 GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0,
2672 50))
2673 drm_err(&stream->perf->i915->drm,
2674 "wait for OA to be disabled timed out\n");
2675}
2676
2677static void gen12_oa_disable(struct i915_perf_stream *stream)
2678{
2679 struct intel_uncore *uncore = stream->uncore;
2680
2681 intel_uncore_write(uncore, GEN12_OAG_OACONTROL, 0);
2682 if (intel_wait_for_register(uncore,
2683 GEN12_OAG_OACONTROL,
2684 GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE, 0,
2685 50))
2686 drm_err(&stream->perf->i915->drm,
2687 "wait for OA to be disabled timed out\n");
2688
2689 intel_uncore_write(uncore, GEN12_OA_TLB_INV_CR, 1);
2690 if (intel_wait_for_register(uncore,
2691 GEN12_OA_TLB_INV_CR,
2692 1, 0,
2693 50))
2694 drm_err(&stream->perf->i915->drm,
2695 "wait for OA tlb invalidate timed out\n");
2696}
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706static void i915_oa_stream_disable(struct i915_perf_stream *stream)
2707{
2708 stream->perf->ops.oa_disable(stream);
2709
2710 if (stream->periodic)
2711 hrtimer_cancel(&stream->poll_check_timer);
2712}
2713
2714static const struct i915_perf_stream_ops i915_oa_stream_ops = {
2715 .destroy = i915_oa_stream_destroy,
2716 .enable = i915_oa_stream_enable,
2717 .disable = i915_oa_stream_disable,
2718 .wait_unlocked = i915_oa_wait_unlocked,
2719 .poll_wait = i915_oa_poll_wait,
2720 .read = i915_oa_read,
2721};
2722
2723static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream)
2724{
2725 struct i915_active *active;
2726 int err;
2727
2728 active = i915_active_create();
2729 if (!active)
2730 return -ENOMEM;
2731
2732 err = stream->perf->ops.enable_metric_set(stream, active);
2733 if (err == 0)
2734 __i915_active_wait(active, TASK_UNINTERRUPTIBLE);
2735
2736 i915_active_put(active);
2737 return err;
2738}
2739
2740static void
2741get_default_sseu_config(struct intel_sseu *out_sseu,
2742 struct intel_engine_cs *engine)
2743{
2744 const struct sseu_dev_info *devinfo_sseu = &engine->gt->info.sseu;
2745
2746 *out_sseu = intel_sseu_from_device_info(devinfo_sseu);
2747
2748 if (IS_GEN(engine->i915, 11)) {
2749
2750
2751
2752
2753
2754 out_sseu->subslice_mask =
2755 ~(~0 << (hweight8(out_sseu->subslice_mask) / 2));
2756 out_sseu->slice_mask = 0x1;
2757 }
2758}
2759
2760static int
2761get_sseu_config(struct intel_sseu *out_sseu,
2762 struct intel_engine_cs *engine,
2763 const struct drm_i915_gem_context_param_sseu *drm_sseu)
2764{
2765 if (drm_sseu->engine.engine_class != engine->uabi_class ||
2766 drm_sseu->engine.engine_instance != engine->uabi_instance)
2767 return -EINVAL;
2768
2769 return i915_gem_user_to_context_sseu(engine->gt, drm_sseu, out_sseu);
2770}
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790static int i915_oa_stream_init(struct i915_perf_stream *stream,
2791 struct drm_i915_perf_open_param *param,
2792 struct perf_open_properties *props)
2793{
2794 struct drm_i915_private *i915 = stream->perf->i915;
2795 struct i915_perf *perf = stream->perf;
2796 int format_size;
2797 int ret;
2798
2799 if (!props->engine) {
2800 DRM_DEBUG("OA engine not specified\n");
2801 return -EINVAL;
2802 }
2803
2804
2805
2806
2807
2808
2809 if (!perf->metrics_kobj) {
2810 DRM_DEBUG("OA metrics weren't advertised via sysfs\n");
2811 return -EINVAL;
2812 }
2813
2814 if (!(props->sample_flags & SAMPLE_OA_REPORT) &&
2815 (INTEL_GEN(perf->i915) < 12 || !stream->ctx)) {
2816 DRM_DEBUG("Only OA report sampling supported\n");
2817 return -EINVAL;
2818 }
2819
2820 if (!perf->ops.enable_metric_set) {
2821 DRM_DEBUG("OA unit not supported\n");
2822 return -ENODEV;
2823 }
2824
2825
2826
2827
2828
2829
2830 if (perf->exclusive_stream) {
2831 DRM_DEBUG("OA unit already in use\n");
2832 return -EBUSY;
2833 }
2834
2835 if (!props->oa_format) {
2836 DRM_DEBUG("OA report format not specified\n");
2837 return -EINVAL;
2838 }
2839
2840 stream->engine = props->engine;
2841 stream->uncore = stream->engine->gt->uncore;
2842
2843 stream->sample_size = sizeof(struct drm_i915_perf_record_header);
2844
2845 format_size = perf->oa_formats[props->oa_format].size;
2846
2847 stream->sample_flags = props->sample_flags;
2848 stream->sample_size += format_size;
2849
2850 stream->oa_buffer.format_size = format_size;
2851 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0))
2852 return -EINVAL;
2853
2854 stream->hold_preemption = props->hold_preemption;
2855
2856 stream->oa_buffer.format =
2857 perf->oa_formats[props->oa_format].format;
2858
2859 stream->periodic = props->oa_periodic;
2860 if (stream->periodic)
2861 stream->period_exponent = props->oa_period_exponent;
2862
2863 if (stream->ctx) {
2864 ret = oa_get_render_ctx_id(stream);
2865 if (ret) {
2866 DRM_DEBUG("Invalid context id to filter with\n");
2867 return ret;
2868 }
2869 }
2870
2871 ret = alloc_noa_wait(stream);
2872 if (ret) {
2873 DRM_DEBUG("Unable to allocate NOA wait batch buffer\n");
2874 goto err_noa_wait_alloc;
2875 }
2876
2877 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set);
2878 if (!stream->oa_config) {
2879 DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set);
2880 ret = -EINVAL;
2881 goto err_config;
2882 }
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896 intel_engine_pm_get(stream->engine);
2897 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL);
2898
2899 ret = alloc_oa_buffer(stream);
2900 if (ret)
2901 goto err_oa_buf_alloc;
2902
2903 stream->ops = &i915_oa_stream_ops;
2904
2905 perf->sseu = props->sseu;
2906 WRITE_ONCE(perf->exclusive_stream, stream);
2907
2908 ret = i915_perf_stream_enable_sync(stream);
2909 if (ret) {
2910 DRM_DEBUG("Unable to enable metric set\n");
2911 goto err_enable;
2912 }
2913
2914 DRM_DEBUG("opening stream oa config uuid=%s\n",
2915 stream->oa_config->uuid);
2916
2917 hrtimer_init(&stream->poll_check_timer,
2918 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2919 stream->poll_check_timer.function = oa_poll_check_timer_cb;
2920 init_waitqueue_head(&stream->poll_wq);
2921 spin_lock_init(&stream->oa_buffer.ptr_lock);
2922
2923 return 0;
2924
2925err_enable:
2926 WRITE_ONCE(perf->exclusive_stream, NULL);
2927 perf->ops.disable_metric_set(stream);
2928
2929 free_oa_buffer(stream);
2930
2931err_oa_buf_alloc:
2932 free_oa_configs(stream);
2933
2934 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
2935 intel_engine_pm_put(stream->engine);
2936
2937err_config:
2938 free_noa_wait(stream);
2939
2940err_noa_wait_alloc:
2941 if (stream->ctx)
2942 oa_put_render_ctx_id(stream);
2943
2944 return ret;
2945}
2946
2947void i915_oa_init_reg_state(const struct intel_context *ce,
2948 const struct intel_engine_cs *engine)
2949{
2950 struct i915_perf_stream *stream;
2951
2952 if (engine->class != RENDER_CLASS)
2953 return;
2954
2955
2956 stream = READ_ONCE(engine->i915->perf.exclusive_stream);
2957 if (stream && INTEL_GEN(stream->perf->i915) < 12)
2958 gen8_update_reg_state_unlocked(ce, stream);
2959}
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979static ssize_t i915_perf_read(struct file *file,
2980 char __user *buf,
2981 size_t count,
2982 loff_t *ppos)
2983{
2984 struct i915_perf_stream *stream = file->private_data;
2985 struct i915_perf *perf = stream->perf;
2986 size_t offset = 0;
2987 int ret;
2988
2989
2990
2991
2992
2993 if (!stream->enabled)
2994 return -EIO;
2995
2996 if (!(file->f_flags & O_NONBLOCK)) {
2997
2998
2999
3000
3001
3002
3003
3004 do {
3005 ret = stream->ops->wait_unlocked(stream);
3006 if (ret)
3007 return ret;
3008
3009 mutex_lock(&perf->lock);
3010 ret = stream->ops->read(stream, buf, count, &offset);
3011 mutex_unlock(&perf->lock);
3012 } while (!offset && !ret);
3013 } else {
3014 mutex_lock(&perf->lock);
3015 ret = stream->ops->read(stream, buf, count, &offset);
3016 mutex_unlock(&perf->lock);
3017 }
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030 if (ret != -ENOSPC)
3031 stream->pollin = false;
3032
3033
3034 return offset ?: (ret ?: -EAGAIN);
3035}
3036
3037static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer)
3038{
3039 struct i915_perf_stream *stream =
3040 container_of(hrtimer, typeof(*stream), poll_check_timer);
3041
3042 if (oa_buffer_check_unlocked(stream)) {
3043 stream->pollin = true;
3044 wake_up(&stream->poll_wq);
3045 }
3046
3047 hrtimer_forward_now(hrtimer,
3048 ns_to_ktime(stream->poll_oa_period));
3049
3050 return HRTIMER_RESTART;
3051}
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream,
3069 struct file *file,
3070 poll_table *wait)
3071{
3072 __poll_t events = 0;
3073
3074 stream->ops->poll_wait(stream, file, wait);
3075
3076
3077
3078
3079
3080
3081
3082 if (stream->pollin)
3083 events |= EPOLLIN;
3084
3085 return events;
3086}
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101static __poll_t i915_perf_poll(struct file *file, poll_table *wait)
3102{
3103 struct i915_perf_stream *stream = file->private_data;
3104 struct i915_perf *perf = stream->perf;
3105 __poll_t ret;
3106
3107 mutex_lock(&perf->lock);
3108 ret = i915_perf_poll_locked(stream, file, wait);
3109 mutex_unlock(&perf->lock);
3110
3111 return ret;
3112}
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124static void i915_perf_enable_locked(struct i915_perf_stream *stream)
3125{
3126 if (stream->enabled)
3127 return;
3128
3129
3130 stream->enabled = true;
3131
3132 if (stream->ops->enable)
3133 stream->ops->enable(stream);
3134
3135 if (stream->hold_preemption)
3136 intel_context_set_nopreempt(stream->pinned_ctx);
3137}
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153static void i915_perf_disable_locked(struct i915_perf_stream *stream)
3154{
3155 if (!stream->enabled)
3156 return;
3157
3158
3159 stream->enabled = false;
3160
3161 if (stream->hold_preemption)
3162 intel_context_clear_nopreempt(stream->pinned_ctx);
3163
3164 if (stream->ops->disable)
3165 stream->ops->disable(stream);
3166}
3167
3168static long i915_perf_config_locked(struct i915_perf_stream *stream,
3169 unsigned long metrics_set)
3170{
3171 struct i915_oa_config *config;
3172 long ret = stream->oa_config->id;
3173
3174 config = i915_perf_get_oa_config(stream->perf, metrics_set);
3175 if (!config)
3176 return -EINVAL;
3177
3178 if (config != stream->oa_config) {
3179 int err;
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190 err = emit_oa_config(stream, config, oa_context(stream), NULL);
3191 if (!err)
3192 config = xchg(&stream->oa_config, config);
3193 else
3194 ret = err;
3195 }
3196
3197 i915_oa_config_put(config);
3198
3199 return ret;
3200}
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
3215 unsigned int cmd,
3216 unsigned long arg)
3217{
3218 switch (cmd) {
3219 case I915_PERF_IOCTL_ENABLE:
3220 i915_perf_enable_locked(stream);
3221 return 0;
3222 case I915_PERF_IOCTL_DISABLE:
3223 i915_perf_disable_locked(stream);
3224 return 0;
3225 case I915_PERF_IOCTL_CONFIG:
3226 return i915_perf_config_locked(stream, arg);
3227 }
3228
3229 return -EINVAL;
3230}
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243static long i915_perf_ioctl(struct file *file,
3244 unsigned int cmd,
3245 unsigned long arg)
3246{
3247 struct i915_perf_stream *stream = file->private_data;
3248 struct i915_perf *perf = stream->perf;
3249 long ret;
3250
3251 mutex_lock(&perf->lock);
3252 ret = i915_perf_ioctl_locked(stream, cmd, arg);
3253 mutex_unlock(&perf->lock);
3254
3255 return ret;
3256}
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
3269{
3270 if (stream->enabled)
3271 i915_perf_disable_locked(stream);
3272
3273 if (stream->ops->destroy)
3274 stream->ops->destroy(stream);
3275
3276 if (stream->ctx)
3277 i915_gem_context_put(stream->ctx);
3278
3279 kfree(stream);
3280}
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293static int i915_perf_release(struct inode *inode, struct file *file)
3294{
3295 struct i915_perf_stream *stream = file->private_data;
3296 struct i915_perf *perf = stream->perf;
3297
3298 mutex_lock(&perf->lock);
3299 i915_perf_destroy_locked(stream);
3300 mutex_unlock(&perf->lock);
3301
3302
3303 drm_dev_put(&perf->i915->drm);
3304
3305 return 0;
3306}
3307
3308
3309static const struct file_operations fops = {
3310 .owner = THIS_MODULE,
3311 .llseek = no_llseek,
3312 .release = i915_perf_release,
3313 .poll = i915_perf_poll,
3314 .read = i915_perf_read,
3315 .unlocked_ioctl = i915_perf_ioctl,
3316
3317
3318
3319 .compat_ioctl = i915_perf_ioctl,
3320};
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347static int
3348i915_perf_open_ioctl_locked(struct i915_perf *perf,
3349 struct drm_i915_perf_open_param *param,
3350 struct perf_open_properties *props,
3351 struct drm_file *file)
3352{
3353 struct i915_gem_context *specific_ctx = NULL;
3354 struct i915_perf_stream *stream = NULL;
3355 unsigned long f_flags = 0;
3356 bool privileged_op = true;
3357 int stream_fd;
3358 int ret;
3359
3360 if (props->single_context) {
3361 u32 ctx_handle = props->ctx_handle;
3362 struct drm_i915_file_private *file_priv = file->driver_priv;
3363
3364 specific_ctx = i915_gem_context_lookup(file_priv, ctx_handle);
3365 if (!specific_ctx) {
3366 DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n",
3367 ctx_handle);
3368 ret = -ENOENT;
3369 goto err;
3370 }
3371 }
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392 if (IS_HASWELL(perf->i915) && specific_ctx)
3393 privileged_op = false;
3394 else if (IS_GEN(perf->i915, 12) && specific_ctx &&
3395 (props->sample_flags & SAMPLE_OA_REPORT) == 0)
3396 privileged_op = false;
3397
3398 if (props->hold_preemption) {
3399 if (!props->single_context) {
3400 DRM_DEBUG("preemption disable with no context\n");
3401 ret = -EINVAL;
3402 goto err;
3403 }
3404 privileged_op = true;
3405 }
3406
3407
3408
3409
3410 if (props->has_sseu)
3411 privileged_op = true;
3412 else
3413 get_default_sseu_config(&props->sseu, props->engine);
3414
3415
3416
3417
3418
3419
3420 if (privileged_op &&
3421 i915_perf_stream_paranoid && !perfmon_capable()) {
3422 DRM_DEBUG("Insufficient privileges to open i915 perf stream\n");
3423 ret = -EACCES;
3424 goto err_ctx;
3425 }
3426
3427 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
3428 if (!stream) {
3429 ret = -ENOMEM;
3430 goto err_ctx;
3431 }
3432
3433 stream->perf = perf;
3434 stream->ctx = specific_ctx;
3435 stream->poll_oa_period = props->poll_oa_period;
3436
3437 ret = i915_oa_stream_init(stream, param, props);
3438 if (ret)
3439 goto err_alloc;
3440
3441
3442
3443
3444
3445 if (WARN_ON(stream->sample_flags != props->sample_flags)) {
3446 ret = -ENODEV;
3447 goto err_flags;
3448 }
3449
3450 if (param->flags & I915_PERF_FLAG_FD_CLOEXEC)
3451 f_flags |= O_CLOEXEC;
3452 if (param->flags & I915_PERF_FLAG_FD_NONBLOCK)
3453 f_flags |= O_NONBLOCK;
3454
3455 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags);
3456 if (stream_fd < 0) {
3457 ret = stream_fd;
3458 goto err_flags;
3459 }
3460
3461 if (!(param->flags & I915_PERF_FLAG_DISABLED))
3462 i915_perf_enable_locked(stream);
3463
3464
3465
3466
3467 drm_dev_get(&perf->i915->drm);
3468
3469 return stream_fd;
3470
3471err_flags:
3472 if (stream->ops->destroy)
3473 stream->ops->destroy(stream);
3474err_alloc:
3475 kfree(stream);
3476err_ctx:
3477 if (specific_ctx)
3478 i915_gem_context_put(specific_ctx);
3479err:
3480 return ret;
3481}
3482
3483static u64 oa_exponent_to_ns(struct i915_perf *perf, int exponent)
3484{
3485 return i915_cs_timestamp_ticks_to_ns(perf->i915, 2ULL << exponent);
3486}
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503static int read_properties_unlocked(struct i915_perf *perf,
3504 u64 __user *uprops,
3505 u32 n_props,
3506 struct perf_open_properties *props)
3507{
3508 u64 __user *uprop = uprops;
3509 u32 i;
3510 int ret;
3511
3512 memset(props, 0, sizeof(struct perf_open_properties));
3513 props->poll_oa_period = DEFAULT_POLL_PERIOD_NS;
3514
3515 if (!n_props) {
3516 DRM_DEBUG("No i915 perf properties given\n");
3517 return -EINVAL;
3518 }
3519
3520
3521 props->engine = intel_engine_lookup_user(perf->i915,
3522 I915_ENGINE_CLASS_RENDER,
3523 0);
3524 if (!props->engine) {
3525 DRM_DEBUG("No RENDER-capable engines\n");
3526 return -EINVAL;
3527 }
3528
3529
3530
3531
3532
3533
3534
3535 if (n_props >= DRM_I915_PERF_PROP_MAX) {
3536 DRM_DEBUG("More i915 perf properties specified than exist\n");
3537 return -EINVAL;
3538 }
3539
3540 for (i = 0; i < n_props; i++) {
3541 u64 oa_period, oa_freq_hz;
3542 u64 id, value;
3543
3544 ret = get_user(id, uprop);
3545 if (ret)
3546 return ret;
3547
3548 ret = get_user(value, uprop + 1);
3549 if (ret)
3550 return ret;
3551
3552 if (id == 0 || id >= DRM_I915_PERF_PROP_MAX) {
3553 DRM_DEBUG("Unknown i915 perf property ID\n");
3554 return -EINVAL;
3555 }
3556
3557 switch ((enum drm_i915_perf_property_id)id) {
3558 case DRM_I915_PERF_PROP_CTX_HANDLE:
3559 props->single_context = 1;
3560 props->ctx_handle = value;
3561 break;
3562 case DRM_I915_PERF_PROP_SAMPLE_OA:
3563 if (value)
3564 props->sample_flags |= SAMPLE_OA_REPORT;
3565 break;
3566 case DRM_I915_PERF_PROP_OA_METRICS_SET:
3567 if (value == 0) {
3568 DRM_DEBUG("Unknown OA metric set ID\n");
3569 return -EINVAL;
3570 }
3571 props->metrics_set = value;
3572 break;
3573 case DRM_I915_PERF_PROP_OA_FORMAT:
3574 if (value == 0 || value >= I915_OA_FORMAT_MAX) {
3575 DRM_DEBUG("Out-of-range OA report format %llu\n",
3576 value);
3577 return -EINVAL;
3578 }
3579 if (!perf->oa_formats[value].size) {
3580 DRM_DEBUG("Unsupported OA report format %llu\n",
3581 value);
3582 return -EINVAL;
3583 }
3584 props->oa_format = value;
3585 break;
3586 case DRM_I915_PERF_PROP_OA_EXPONENT:
3587 if (value > OA_EXPONENT_MAX) {
3588 DRM_DEBUG("OA timer exponent too high (> %u)\n",
3589 OA_EXPONENT_MAX);
3590 return -EINVAL;
3591 }
3592
3593
3594
3595
3596
3597
3598
3599 BUILD_BUG_ON(sizeof(oa_period) != 8);
3600 oa_period = oa_exponent_to_ns(perf, value);
3601
3602
3603
3604
3605
3606
3607
3608 if (oa_period <= NSEC_PER_SEC) {
3609 u64 tmp = NSEC_PER_SEC;
3610 do_div(tmp, oa_period);
3611 oa_freq_hz = tmp;
3612 } else
3613 oa_freq_hz = 0;
3614
3615 if (oa_freq_hz > i915_oa_max_sample_rate && !perfmon_capable()) {
3616 DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without CAP_PERFMON or CAP_SYS_ADMIN privileges\n",
3617 i915_oa_max_sample_rate);
3618 return -EACCES;
3619 }
3620
3621 props->oa_periodic = true;
3622 props->oa_period_exponent = value;
3623 break;
3624 case DRM_I915_PERF_PROP_HOLD_PREEMPTION:
3625 props->hold_preemption = !!value;
3626 break;
3627 case DRM_I915_PERF_PROP_GLOBAL_SSEU: {
3628 struct drm_i915_gem_context_param_sseu user_sseu;
3629
3630 if (copy_from_user(&user_sseu,
3631 u64_to_user_ptr(value),
3632 sizeof(user_sseu))) {
3633 DRM_DEBUG("Unable to copy global sseu parameter\n");
3634 return -EFAULT;
3635 }
3636
3637 ret = get_sseu_config(&props->sseu, props->engine, &user_sseu);
3638 if (ret) {
3639 DRM_DEBUG("Invalid SSEU configuration\n");
3640 return ret;
3641 }
3642 props->has_sseu = true;
3643 break;
3644 }
3645 case DRM_I915_PERF_PROP_POLL_OA_PERIOD:
3646 if (value < 100000 ) {
3647 DRM_DEBUG("OA availability timer too small (%lluns < 100us)\n",
3648 value);
3649 return -EINVAL;
3650 }
3651 props->poll_oa_period = value;
3652 break;
3653 case DRM_I915_PERF_PROP_MAX:
3654 MISSING_CASE(id);
3655 return -EINVAL;
3656 }
3657
3658 uprop += 2;
3659 }
3660
3661 return 0;
3662}
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688int i915_perf_open_ioctl(struct drm_device *dev, void *data,
3689 struct drm_file *file)
3690{
3691 struct i915_perf *perf = &to_i915(dev)->perf;
3692 struct drm_i915_perf_open_param *param = data;
3693 struct perf_open_properties props;
3694 u32 known_open_flags;
3695 int ret;
3696
3697 if (!perf->i915) {
3698 DRM_DEBUG("i915 perf interface not available for this system\n");
3699 return -ENOTSUPP;
3700 }
3701
3702 known_open_flags = I915_PERF_FLAG_FD_CLOEXEC |
3703 I915_PERF_FLAG_FD_NONBLOCK |
3704 I915_PERF_FLAG_DISABLED;
3705 if (param->flags & ~known_open_flags) {
3706 DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n");
3707 return -EINVAL;
3708 }
3709
3710 ret = read_properties_unlocked(perf,
3711 u64_to_user_ptr(param->properties_ptr),
3712 param->num_properties,
3713 &props);
3714 if (ret)
3715 return ret;
3716
3717 mutex_lock(&perf->lock);
3718 ret = i915_perf_open_ioctl_locked(perf, param, &props, file);
3719 mutex_unlock(&perf->lock);
3720
3721 return ret;
3722}
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732void i915_perf_register(struct drm_i915_private *i915)
3733{
3734 struct i915_perf *perf = &i915->perf;
3735
3736 if (!perf->i915)
3737 return;
3738
3739
3740
3741
3742
3743 mutex_lock(&perf->lock);
3744
3745 perf->metrics_kobj =
3746 kobject_create_and_add("metrics",
3747 &i915->drm.primary->kdev->kobj);
3748
3749 mutex_unlock(&perf->lock);
3750}
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761void i915_perf_unregister(struct drm_i915_private *i915)
3762{
3763 struct i915_perf *perf = &i915->perf;
3764
3765 if (!perf->metrics_kobj)
3766 return;
3767
3768 kobject_put(perf->metrics_kobj);
3769 perf->metrics_kobj = NULL;
3770}
3771
3772static bool gen8_is_valid_flex_addr(struct i915_perf *perf, u32 addr)
3773{
3774 static const i915_reg_t flex_eu_regs[] = {
3775 EU_PERF_CNTL0,
3776 EU_PERF_CNTL1,
3777 EU_PERF_CNTL2,
3778 EU_PERF_CNTL3,
3779 EU_PERF_CNTL4,
3780 EU_PERF_CNTL5,
3781 EU_PERF_CNTL6,
3782 };
3783 int i;
3784
3785 for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
3786 if (i915_mmio_reg_offset(flex_eu_regs[i]) == addr)
3787 return true;
3788 }
3789 return false;
3790}
3791
3792#define ADDR_IN_RANGE(addr, start, end) \
3793 ((addr) >= (start) && \
3794 (addr) <= (end))
3795
3796#define REG_IN_RANGE(addr, start, end) \
3797 ((addr) >= i915_mmio_reg_offset(start) && \
3798 (addr) <= i915_mmio_reg_offset(end))
3799
3800#define REG_EQUAL(addr, mmio) \
3801 ((addr) == i915_mmio_reg_offset(mmio))
3802
3803static bool gen7_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
3804{
3805 return REG_IN_RANGE(addr, OASTARTTRIG1, OASTARTTRIG8) ||
3806 REG_IN_RANGE(addr, OAREPORTTRIG1, OAREPORTTRIG8) ||
3807 REG_IN_RANGE(addr, OACEC0_0, OACEC7_1);
3808}
3809
3810static bool gen7_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
3811{
3812 return REG_EQUAL(addr, HALF_SLICE_CHICKEN2) ||
3813 REG_IN_RANGE(addr, MICRO_BP0_0, NOA_WRITE) ||
3814 REG_IN_RANGE(addr, OA_PERFCNT1_LO, OA_PERFCNT2_HI) ||
3815 REG_IN_RANGE(addr, OA_PERFMATRIX_LO, OA_PERFMATRIX_HI);
3816}
3817
3818static bool gen8_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
3819{
3820 return gen7_is_valid_mux_addr(perf, addr) ||
3821 REG_EQUAL(addr, WAIT_FOR_RC6_EXIT) ||
3822 REG_IN_RANGE(addr, RPM_CONFIG0, NOA_CONFIG(8));
3823}
3824
3825static bool gen10_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
3826{
3827 return gen8_is_valid_mux_addr(perf, addr) ||
3828 REG_EQUAL(addr, GEN10_NOA_WRITE_HIGH) ||
3829 REG_IN_RANGE(addr, OA_PERFCNT3_LO, OA_PERFCNT4_HI);
3830}
3831
3832static bool hsw_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
3833{
3834 return gen7_is_valid_mux_addr(perf, addr) ||
3835 ADDR_IN_RANGE(addr, 0x25100, 0x2FF90) ||
3836 REG_IN_RANGE(addr, HSW_MBVID2_NOA0, HSW_MBVID2_NOA9) ||
3837 REG_EQUAL(addr, HSW_MBVID2_MISR0);
3838}
3839
3840static bool chv_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
3841{
3842 return gen7_is_valid_mux_addr(perf, addr) ||
3843 ADDR_IN_RANGE(addr, 0x182300, 0x1823A4);
3844}
3845
3846static bool gen12_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr)
3847{
3848 return REG_IN_RANGE(addr, GEN12_OAG_OASTARTTRIG1, GEN12_OAG_OASTARTTRIG8) ||
3849 REG_IN_RANGE(addr, GEN12_OAG_OAREPORTTRIG1, GEN12_OAG_OAREPORTTRIG8) ||
3850 REG_IN_RANGE(addr, GEN12_OAG_CEC0_0, GEN12_OAG_CEC7_1) ||
3851 REG_IN_RANGE(addr, GEN12_OAG_SCEC0_0, GEN12_OAG_SCEC7_1) ||
3852 REG_EQUAL(addr, GEN12_OAA_DBG_REG) ||
3853 REG_EQUAL(addr, GEN12_OAG_OA_PESS) ||
3854 REG_EQUAL(addr, GEN12_OAG_SPCTR_CNF);
3855}
3856
3857static bool gen12_is_valid_mux_addr(struct i915_perf *perf, u32 addr)
3858{
3859 return REG_EQUAL(addr, NOA_WRITE) ||
3860 REG_EQUAL(addr, GEN10_NOA_WRITE_HIGH) ||
3861 REG_EQUAL(addr, GDT_CHICKEN_BITS) ||
3862 REG_EQUAL(addr, WAIT_FOR_RC6_EXIT) ||
3863 REG_EQUAL(addr, RPM_CONFIG0) ||
3864 REG_EQUAL(addr, RPM_CONFIG1) ||
3865 REG_IN_RANGE(addr, NOA_CONFIG(0), NOA_CONFIG(8));
3866}
3867
3868static u32 mask_reg_value(u32 reg, u32 val)
3869{
3870
3871
3872
3873
3874 if (REG_EQUAL(reg, HALF_SLICE_CHICKEN2))
3875 val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE);
3876
3877
3878
3879
3880
3881 if (REG_EQUAL(reg, WAIT_FOR_RC6_EXIT))
3882 val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
3883
3884 return val;
3885}
3886
3887static struct i915_oa_reg *alloc_oa_regs(struct i915_perf *perf,
3888 bool (*is_valid)(struct i915_perf *perf, u32 addr),
3889 u32 __user *regs,
3890 u32 n_regs)
3891{
3892 struct i915_oa_reg *oa_regs;
3893 int err;
3894 u32 i;
3895
3896 if (!n_regs)
3897 return NULL;
3898
3899
3900 GEM_BUG_ON(!is_valid);
3901 if (!is_valid)
3902 return ERR_PTR(-EINVAL);
3903
3904 oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL);
3905 if (!oa_regs)
3906 return ERR_PTR(-ENOMEM);
3907
3908 for (i = 0; i < n_regs; i++) {
3909 u32 addr, value;
3910
3911 err = get_user(addr, regs);
3912 if (err)
3913 goto addr_err;
3914
3915 if (!is_valid(perf, addr)) {
3916 DRM_DEBUG("Invalid oa_reg address: %X\n", addr);
3917 err = -EINVAL;
3918 goto addr_err;
3919 }
3920
3921 err = get_user(value, regs + 1);
3922 if (err)
3923 goto addr_err;
3924
3925 oa_regs[i].addr = _MMIO(addr);
3926 oa_regs[i].value = mask_reg_value(addr, value);
3927
3928 regs += 2;
3929 }
3930
3931 return oa_regs;
3932
3933addr_err:
3934 kfree(oa_regs);
3935 return ERR_PTR(err);
3936}
3937
3938static ssize_t show_dynamic_id(struct device *dev,
3939 struct device_attribute *attr,
3940 char *buf)
3941{
3942 struct i915_oa_config *oa_config =
3943 container_of(attr, typeof(*oa_config), sysfs_metric_id);
3944
3945 return sprintf(buf, "%d\n", oa_config->id);
3946}
3947
3948static int create_dynamic_oa_sysfs_entry(struct i915_perf *perf,
3949 struct i915_oa_config *oa_config)
3950{
3951 sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
3952 oa_config->sysfs_metric_id.attr.name = "id";
3953 oa_config->sysfs_metric_id.attr.mode = S_IRUGO;
3954 oa_config->sysfs_metric_id.show = show_dynamic_id;
3955 oa_config->sysfs_metric_id.store = NULL;
3956
3957 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
3958 oa_config->attrs[1] = NULL;
3959
3960 oa_config->sysfs_metric.name = oa_config->uuid;
3961 oa_config->sysfs_metric.attrs = oa_config->attrs;
3962
3963 return sysfs_create_group(perf->metrics_kobj,
3964 &oa_config->sysfs_metric);
3965}
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
3981 struct drm_file *file)
3982{
3983 struct i915_perf *perf = &to_i915(dev)->perf;
3984 struct drm_i915_perf_oa_config *args = data;
3985 struct i915_oa_config *oa_config, *tmp;
3986 struct i915_oa_reg *regs;
3987 int err, id;
3988
3989 if (!perf->i915) {
3990 DRM_DEBUG("i915 perf interface not available for this system\n");
3991 return -ENOTSUPP;
3992 }
3993
3994 if (!perf->metrics_kobj) {
3995 DRM_DEBUG("OA metrics weren't advertised via sysfs\n");
3996 return -EINVAL;
3997 }
3998
3999 if (i915_perf_stream_paranoid && !perfmon_capable()) {
4000 DRM_DEBUG("Insufficient privileges to add i915 OA config\n");
4001 return -EACCES;
4002 }
4003
4004 if ((!args->mux_regs_ptr || !args->n_mux_regs) &&
4005 (!args->boolean_regs_ptr || !args->n_boolean_regs) &&
4006 (!args->flex_regs_ptr || !args->n_flex_regs)) {
4007 DRM_DEBUG("No OA registers given\n");
4008 return -EINVAL;
4009 }
4010
4011 oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL);
4012 if (!oa_config) {
4013 DRM_DEBUG("Failed to allocate memory for the OA config\n");
4014 return -ENOMEM;
4015 }
4016
4017 oa_config->perf = perf;
4018 kref_init(&oa_config->ref);
4019
4020 if (!uuid_is_valid(args->uuid)) {
4021 DRM_DEBUG("Invalid uuid format for OA config\n");
4022 err = -EINVAL;
4023 goto reg_err;
4024 }
4025
4026
4027
4028
4029 memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid));
4030
4031 oa_config->mux_regs_len = args->n_mux_regs;
4032 regs = alloc_oa_regs(perf,
4033 perf->ops.is_valid_mux_reg,
4034 u64_to_user_ptr(args->mux_regs_ptr),
4035 args->n_mux_regs);
4036
4037 if (IS_ERR(regs)) {
4038 DRM_DEBUG("Failed to create OA config for mux_regs\n");
4039 err = PTR_ERR(regs);
4040 goto reg_err;
4041 }
4042 oa_config->mux_regs = regs;
4043
4044 oa_config->b_counter_regs_len = args->n_boolean_regs;
4045 regs = alloc_oa_regs(perf,
4046 perf->ops.is_valid_b_counter_reg,
4047 u64_to_user_ptr(args->boolean_regs_ptr),
4048 args->n_boolean_regs);
4049
4050 if (IS_ERR(regs)) {
4051 DRM_DEBUG("Failed to create OA config for b_counter_regs\n");
4052 err = PTR_ERR(regs);
4053 goto reg_err;
4054 }
4055 oa_config->b_counter_regs = regs;
4056
4057 if (INTEL_GEN(perf->i915) < 8) {
4058 if (args->n_flex_regs != 0) {
4059 err = -EINVAL;
4060 goto reg_err;
4061 }
4062 } else {
4063 oa_config->flex_regs_len = args->n_flex_regs;
4064 regs = alloc_oa_regs(perf,
4065 perf->ops.is_valid_flex_reg,
4066 u64_to_user_ptr(args->flex_regs_ptr),
4067 args->n_flex_regs);
4068
4069 if (IS_ERR(regs)) {
4070 DRM_DEBUG("Failed to create OA config for flex_regs\n");
4071 err = PTR_ERR(regs);
4072 goto reg_err;
4073 }
4074 oa_config->flex_regs = regs;
4075 }
4076
4077 err = mutex_lock_interruptible(&perf->metrics_lock);
4078 if (err)
4079 goto reg_err;
4080
4081
4082
4083
4084 idr_for_each_entry(&perf->metrics_idr, tmp, id) {
4085 if (!strcmp(tmp->uuid, oa_config->uuid)) {
4086 DRM_DEBUG("OA config already exists with this uuid\n");
4087 err = -EADDRINUSE;
4088 goto sysfs_err;
4089 }
4090 }
4091
4092 err = create_dynamic_oa_sysfs_entry(perf, oa_config);
4093 if (err) {
4094 DRM_DEBUG("Failed to create sysfs entry for OA config\n");
4095 goto sysfs_err;
4096 }
4097
4098
4099 oa_config->id = idr_alloc(&perf->metrics_idr,
4100 oa_config, 2,
4101 0, GFP_KERNEL);
4102 if (oa_config->id < 0) {
4103 DRM_DEBUG("Failed to create sysfs entry for OA config\n");
4104 err = oa_config->id;
4105 goto sysfs_err;
4106 }
4107
4108 mutex_unlock(&perf->metrics_lock);
4109
4110 DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id);
4111
4112 return oa_config->id;
4113
4114sysfs_err:
4115 mutex_unlock(&perf->metrics_lock);
4116reg_err:
4117 i915_oa_config_put(oa_config);
4118 DRM_DEBUG("Failed to add new OA config\n");
4119 return err;
4120}
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
4134 struct drm_file *file)
4135{
4136 struct i915_perf *perf = &to_i915(dev)->perf;
4137 u64 *arg = data;
4138 struct i915_oa_config *oa_config;
4139 int ret;
4140
4141 if (!perf->i915) {
4142 DRM_DEBUG("i915 perf interface not available for this system\n");
4143 return -ENOTSUPP;
4144 }
4145
4146 if (i915_perf_stream_paranoid && !perfmon_capable()) {
4147 DRM_DEBUG("Insufficient privileges to remove i915 OA config\n");
4148 return -EACCES;
4149 }
4150
4151 ret = mutex_lock_interruptible(&perf->metrics_lock);
4152 if (ret)
4153 return ret;
4154
4155 oa_config = idr_find(&perf->metrics_idr, *arg);
4156 if (!oa_config) {
4157 DRM_DEBUG("Failed to remove unknown OA config\n");
4158 ret = -ENOENT;
4159 goto err_unlock;
4160 }
4161
4162 GEM_BUG_ON(*arg != oa_config->id);
4163
4164 sysfs_remove_group(perf->metrics_kobj, &oa_config->sysfs_metric);
4165
4166 idr_remove(&perf->metrics_idr, *arg);
4167
4168 mutex_unlock(&perf->metrics_lock);
4169
4170 DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
4171
4172 i915_oa_config_put(oa_config);
4173
4174 return 0;
4175
4176err_unlock:
4177 mutex_unlock(&perf->metrics_lock);
4178 return ret;
4179}
4180
4181static struct ctl_table oa_table[] = {
4182 {
4183 .procname = "perf_stream_paranoid",
4184 .data = &i915_perf_stream_paranoid,
4185 .maxlen = sizeof(i915_perf_stream_paranoid),
4186 .mode = 0644,
4187 .proc_handler = proc_dointvec_minmax,
4188 .extra1 = SYSCTL_ZERO,
4189 .extra2 = SYSCTL_ONE,
4190 },
4191 {
4192 .procname = "oa_max_sample_rate",
4193 .data = &i915_oa_max_sample_rate,
4194 .maxlen = sizeof(i915_oa_max_sample_rate),
4195 .mode = 0644,
4196 .proc_handler = proc_dointvec_minmax,
4197 .extra1 = SYSCTL_ZERO,
4198 .extra2 = &oa_sample_rate_hard_limit,
4199 },
4200 {}
4201};
4202
4203static struct ctl_table i915_root[] = {
4204 {
4205 .procname = "i915",
4206 .maxlen = 0,
4207 .mode = 0555,
4208 .child = oa_table,
4209 },
4210 {}
4211};
4212
4213static struct ctl_table dev_root[] = {
4214 {
4215 .procname = "dev",
4216 .maxlen = 0,
4217 .mode = 0555,
4218 .child = i915_root,
4219 },
4220 {}
4221};
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232void i915_perf_init(struct drm_i915_private *i915)
4233{
4234 struct i915_perf *perf = &i915->perf;
4235
4236
4237
4238 if (IS_HASWELL(i915)) {
4239 perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr;
4240 perf->ops.is_valid_mux_reg = hsw_is_valid_mux_addr;
4241 perf->ops.is_valid_flex_reg = NULL;
4242 perf->ops.enable_metric_set = hsw_enable_metric_set;
4243 perf->ops.disable_metric_set = hsw_disable_metric_set;
4244 perf->ops.oa_enable = gen7_oa_enable;
4245 perf->ops.oa_disable = gen7_oa_disable;
4246 perf->ops.read = gen7_oa_read;
4247 perf->ops.oa_hw_tail_read = gen7_oa_hw_tail_read;
4248
4249 perf->oa_formats = hsw_oa_formats;
4250 } else if (HAS_LOGICAL_RING_CONTEXTS(i915)) {
4251
4252
4253
4254
4255
4256
4257 perf->ops.read = gen8_oa_read;
4258
4259 if (IS_GEN_RANGE(i915, 8, 9)) {
4260 perf->oa_formats = gen8_plus_oa_formats;
4261
4262 perf->ops.is_valid_b_counter_reg =
4263 gen7_is_valid_b_counter_addr;
4264 perf->ops.is_valid_mux_reg =
4265 gen8_is_valid_mux_addr;
4266 perf->ops.is_valid_flex_reg =
4267 gen8_is_valid_flex_addr;
4268
4269 if (IS_CHERRYVIEW(i915)) {
4270 perf->ops.is_valid_mux_reg =
4271 chv_is_valid_mux_addr;
4272 }
4273
4274 perf->ops.oa_enable = gen8_oa_enable;
4275 perf->ops.oa_disable = gen8_oa_disable;
4276 perf->ops.enable_metric_set = gen8_enable_metric_set;
4277 perf->ops.disable_metric_set = gen8_disable_metric_set;
4278 perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
4279
4280 if (IS_GEN(i915, 8)) {
4281 perf->ctx_oactxctrl_offset = 0x120;
4282 perf->ctx_flexeu0_offset = 0x2ce;
4283
4284 perf->gen8_valid_ctx_bit = BIT(25);
4285 } else {
4286 perf->ctx_oactxctrl_offset = 0x128;
4287 perf->ctx_flexeu0_offset = 0x3de;
4288
4289 perf->gen8_valid_ctx_bit = BIT(16);
4290 }
4291 } else if (IS_GEN_RANGE(i915, 10, 11)) {
4292 perf->oa_formats = gen8_plus_oa_formats;
4293
4294 perf->ops.is_valid_b_counter_reg =
4295 gen7_is_valid_b_counter_addr;
4296 perf->ops.is_valid_mux_reg =
4297 gen10_is_valid_mux_addr;
4298 perf->ops.is_valid_flex_reg =
4299 gen8_is_valid_flex_addr;
4300
4301 perf->ops.oa_enable = gen8_oa_enable;
4302 perf->ops.oa_disable = gen8_oa_disable;
4303 perf->ops.enable_metric_set = gen8_enable_metric_set;
4304 perf->ops.disable_metric_set = gen10_disable_metric_set;
4305 perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
4306
4307 if (IS_GEN(i915, 10)) {
4308 perf->ctx_oactxctrl_offset = 0x128;
4309 perf->ctx_flexeu0_offset = 0x3de;
4310 } else {
4311 perf->ctx_oactxctrl_offset = 0x124;
4312 perf->ctx_flexeu0_offset = 0x78e;
4313 }
4314 perf->gen8_valid_ctx_bit = BIT(16);
4315 } else if (IS_GEN(i915, 12)) {
4316 perf->oa_formats = gen12_oa_formats;
4317
4318 perf->ops.is_valid_b_counter_reg =
4319 gen12_is_valid_b_counter_addr;
4320 perf->ops.is_valid_mux_reg =
4321 gen12_is_valid_mux_addr;
4322 perf->ops.is_valid_flex_reg =
4323 gen8_is_valid_flex_addr;
4324
4325 perf->ops.oa_enable = gen12_oa_enable;
4326 perf->ops.oa_disable = gen12_oa_disable;
4327 perf->ops.enable_metric_set = gen12_enable_metric_set;
4328 perf->ops.disable_metric_set = gen12_disable_metric_set;
4329 perf->ops.oa_hw_tail_read = gen12_oa_hw_tail_read;
4330
4331 perf->ctx_flexeu0_offset = 0;
4332 perf->ctx_oactxctrl_offset = 0x144;
4333 }
4334 }
4335
4336 if (perf->ops.enable_metric_set) {
4337 mutex_init(&perf->lock);
4338
4339 oa_sample_rate_hard_limit =
4340 RUNTIME_INFO(i915)->cs_timestamp_frequency_hz / 2;
4341
4342 mutex_init(&perf->metrics_lock);
4343 idr_init(&perf->metrics_idr);
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355 ratelimit_state_init(&perf->spurious_report_rs, 5 * HZ, 10);
4356
4357
4358
4359
4360 ratelimit_set_flags(&perf->spurious_report_rs,
4361 RATELIMIT_MSG_ON_RELEASE);
4362
4363 ratelimit_state_init(&perf->tail_pointer_race,
4364 5 * HZ, 10);
4365 ratelimit_set_flags(&perf->tail_pointer_race,
4366 RATELIMIT_MSG_ON_RELEASE);
4367
4368 atomic64_set(&perf->noa_programming_delay,
4369 500 * 1000 );
4370
4371 perf->i915 = i915;
4372 }
4373}
4374
4375static int destroy_config(int id, void *p, void *data)
4376{
4377 i915_oa_config_put(p);
4378 return 0;
4379}
4380
4381void i915_perf_sysctl_register(void)
4382{
4383 sysctl_header = register_sysctl_table(dev_root);
4384}
4385
4386void i915_perf_sysctl_unregister(void)
4387{
4388 unregister_sysctl_table(sysctl_header);
4389}
4390
4391
4392
4393
4394
4395void i915_perf_fini(struct drm_i915_private *i915)
4396{
4397 struct i915_perf *perf = &i915->perf;
4398
4399 if (!perf->i915)
4400 return;
4401
4402 idr_for_each(&perf->metrics_idr, destroy_config, perf);
4403 idr_destroy(&perf->metrics_idr);
4404
4405 memset(&perf->ops, 0, sizeof(perf->ops));
4406 perf->i915 = NULL;
4407}
4408
4409
4410
4411
4412
4413
4414int i915_perf_ioctl_version(void)
4415{
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436 return 5;
4437}
4438
4439#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4440#include "selftests/i915_perf.c"
4441#endif
4442