1
2
3
4
5
6#include <linux/bitops.h>
7#include <linux/kernel.h>
8#include <linux/moduleparam.h>
9#include <linux/init.h>
10#include <linux/types.h>
11#include <linux/device.h>
12#include <linux/io.h>
13#include <linux/err.h>
14#include <linux/fs.h>
15#include <linux/slab.h>
16#include <linux/delay.h>
17#include <linux/smp.h>
18#include <linux/sysfs.h>
19#include <linux/stat.h>
20#include <linux/clk.h>
21#include <linux/cpu.h>
22#include <linux/cpu_pm.h>
23#include <linux/coresight.h>
24#include <linux/coresight-pmu.h>
25#include <linux/pm_wakeup.h>
26#include <linux/amba/bus.h>
27#include <linux/seq_file.h>
28#include <linux/uaccess.h>
29#include <linux/perf_event.h>
30#include <linux/platform_device.h>
31#include <linux/pm_runtime.h>
32#include <linux/property.h>
33
34#include <asm/barrier.h>
35#include <asm/sections.h>
36#include <asm/sysreg.h>
37#include <asm/local.h>
38#include <asm/virt.h>
39
40#include "coresight-etm4x.h"
41#include "coresight-etm-perf.h"
42#include "coresight-etm4x-cfg.h"
43#include "coresight-syscfg.h"
44
45static int boot_enable;
46module_param(boot_enable, int, 0444);
47MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
48
49#define PARAM_PM_SAVE_FIRMWARE 0
50#define PARAM_PM_SAVE_NEVER 1
51#define PARAM_PM_SAVE_SELF_HOSTED 2
52
53static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
54module_param(pm_save_enable, int, 0444);
55MODULE_PARM_DESC(pm_save_enable,
56 "Save/restore state on power down: 1 = never, 2 = self-hosted");
57
58static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
59static void etm4_set_default_config(struct etmv4_config *config);
60static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
61 struct perf_event *event);
62static u64 etm4_get_access_type(struct etmv4_config *config);
63
64static enum cpuhp_state hp_online;
65
66struct etm4_init_arg {
67 unsigned int pid;
68 struct etmv4_drvdata *drvdata;
69 struct csdev_access *csa;
70};
71
72
73
74
75
76
77
78
79
80
81static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n)
82{
83 return (n < drvdata->nr_ss_cmp) &&
84 drvdata->nr_pe &&
85 (drvdata->config.ss_status[n] & TRCSSCSRn_PC);
86}
87
88u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit)
89{
90 u64 res = 0;
91
92 switch (offset) {
93 ETM4x_READ_SYSREG_CASES(res)
94 default :
95 pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n",
96 offset);
97 }
98
99 if (!_relaxed)
100 __iormb(res);
101
102 return res;
103}
104
105void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)
106{
107 if (!_relaxed)
108 __iowmb();
109 if (!_64bit)
110 val &= GENMASK(31, 0);
111
112 switch (offset) {
113 ETM4x_WRITE_SYSREG_CASES(val)
114 default :
115 pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n",
116 offset);
117 }
118}
119
120static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit)
121{
122 u64 res = 0;
123
124 switch (offset) {
125 ETE_READ_CASES(res)
126 default :
127 pr_warn_ratelimited("ete: trying to read unsupported register @%x\n",
128 offset);
129 }
130
131 if (!_relaxed)
132 __iormb(res);
133
134 return res;
135}
136
137static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)
138{
139 if (!_relaxed)
140 __iowmb();
141 if (!_64bit)
142 val &= GENMASK(31, 0);
143
144 switch (offset) {
145 ETE_WRITE_CASES(val)
146 default :
147 pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n",
148 offset);
149 }
150}
151
152static void etm_detect_os_lock(struct etmv4_drvdata *drvdata,
153 struct csdev_access *csa)
154{
155 u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR);
156
157 drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr);
158}
159
160static void etm_write_os_lock(struct etmv4_drvdata *drvdata,
161 struct csdev_access *csa, u32 val)
162{
163 val = !!val;
164
165 switch (drvdata->os_lock_model) {
166 case ETM_OSLOCK_PRESENT:
167 etm4x_relaxed_write32(csa, val, TRCOSLAR);
168 break;
169 case ETM_OSLOCK_PE:
170 write_sysreg_s(val, SYS_OSLAR_EL1);
171 break;
172 default:
173 pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n",
174 smp_processor_id(), drvdata->os_lock_model);
175 fallthrough;
176 case ETM_OSLOCK_NI:
177 return;
178 }
179 isb();
180}
181
182static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata,
183 struct csdev_access *csa)
184{
185 WARN_ON(drvdata->cpu != smp_processor_id());
186
187
188 etm_write_os_lock(drvdata, csa, 0x0);
189 drvdata->os_unlock = true;
190}
191
192static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
193{
194 if (!WARN_ON(!drvdata->csdev))
195 etm4_os_unlock_csa(drvdata, &drvdata->csdev->access);
196}
197
198static void etm4_os_lock(struct etmv4_drvdata *drvdata)
199{
200 if (WARN_ON(!drvdata->csdev))
201 return;
202
203 etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1);
204 drvdata->os_unlock = false;
205}
206
207static void etm4_cs_lock(struct etmv4_drvdata *drvdata,
208 struct csdev_access *csa)
209{
210
211 if (csa->io_mem)
212 CS_LOCK(csa->base);
213}
214
215static void etm4_cs_unlock(struct etmv4_drvdata *drvdata,
216 struct csdev_access *csa)
217{
218 if (csa->io_mem)
219 CS_UNLOCK(csa->base);
220}
221
222static int etm4_cpu_id(struct coresight_device *csdev)
223{
224 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
225
226 return drvdata->cpu;
227}
228
229static int etm4_trace_id(struct coresight_device *csdev)
230{
231 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
232
233 return drvdata->trcid;
234}
235
236struct etm4_enable_arg {
237 struct etmv4_drvdata *drvdata;
238 int rc;
239};
240
241#ifdef CONFIG_ETM4X_IMPDEF_FEATURE
242
243#define HISI_HIP08_AMBA_ID 0x000b6d01
244#define ETM4_AMBA_MASK 0xfffff
245#define HISI_HIP08_CORE_COMMIT_MASK 0x3000
246#define HISI_HIP08_CORE_COMMIT_SHIFT 12
247#define HISI_HIP08_CORE_COMMIT_FULL 0b00
248#define HISI_HIP08_CORE_COMMIT_LVL_1 0b01
249#define HISI_HIP08_CORE_COMMIT_REG sys_reg(3, 1, 15, 2, 5)
250
251struct etm4_arch_features {
252 void (*arch_callback)(bool enable);
253};
254
255static bool etm4_hisi_match_pid(unsigned int id)
256{
257 return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID;
258}
259
260static void etm4_hisi_config_core_commit(bool enable)
261{
262 u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 :
263 HISI_HIP08_CORE_COMMIT_FULL;
264 u64 val;
265
266
267
268
269
270
271
272 val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG);
273 val &= ~HISI_HIP08_CORE_COMMIT_MASK;
274 val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT;
275 write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG);
276}
277
278static struct etm4_arch_features etm4_features[] = {
279 [ETM4_IMPDEF_HISI_CORE_COMMIT] = {
280 .arch_callback = etm4_hisi_config_core_commit,
281 },
282 {},
283};
284
285static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
286{
287 struct etm4_arch_features *ftr;
288 int bit;
289
290 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {
291 ftr = &etm4_features[bit];
292
293 if (ftr->arch_callback)
294 ftr->arch_callback(true);
295 }
296}
297
298static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)
299{
300 struct etm4_arch_features *ftr;
301 int bit;
302
303 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {
304 ftr = &etm4_features[bit];
305
306 if (ftr->arch_callback)
307 ftr->arch_callback(false);
308 }
309}
310
311static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,
312 unsigned int id)
313{
314 if (etm4_hisi_match_pid(id))
315 set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features);
316}
317#else
318static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
319{
320}
321
322static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)
323{
324}
325
326static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,
327 unsigned int id)
328{
329}
330#endif
331
332static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
333{
334 int i, rc;
335 struct etmv4_config *config = &drvdata->config;
336 struct coresight_device *csdev = drvdata->csdev;
337 struct device *etm_dev = &csdev->dev;
338 struct csdev_access *csa = &csdev->access;
339
340
341 etm4_cs_unlock(drvdata, csa);
342 etm4_enable_arch_specific(drvdata);
343
344 etm4_os_unlock(drvdata);
345
346 rc = coresight_claim_device_unlocked(csdev);
347 if (rc)
348 goto done;
349
350
351 etm4x_relaxed_write32(csa, 0, TRCPRGCTLR);
352
353
354
355
356
357
358
359 if (!csa->io_mem)
360 isb();
361
362
363 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
364 dev_err(etm_dev,
365 "timeout while waiting for Idle Trace Status\n");
366 if (drvdata->nr_pe)
367 etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR);
368 etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR);
369
370 etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR);
371 etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R);
372 etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R);
373 if (drvdata->stallctl)
374 etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR);
375 etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR);
376 etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR);
377 etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR);
378 etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR);
379 etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR);
380 etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR);
381 etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR);
382 etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR);
383 if (drvdata->nr_pe_cmp)
384 etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR);
385 for (i = 0; i < drvdata->nrseqstate - 1; i++)
386 etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i));
387 etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR);
388 etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR);
389 etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR);
390 for (i = 0; i < drvdata->nr_cntr; i++) {
391 etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i));
392 etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i));
393 etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i));
394 }
395
396
397
398
399
400 for (i = 2; i < drvdata->nr_resource * 2; i++)
401 etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i));
402
403 for (i = 0; i < drvdata->nr_ss_cmp; i++) {
404
405 if (config->ss_ctrl[i] || config->ss_pe_cmp[i])
406 config->ss_status[i] &= ~BIT(31);
407 etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i));
408 etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i));
409 if (etm4x_sspcicrn_present(drvdata, i))
410 etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i));
411 }
412 for (i = 0; i < drvdata->nr_addr_cmp; i++) {
413 etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i));
414 etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i));
415 }
416 for (i = 0; i < drvdata->numcidc; i++)
417 etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i));
418 etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0);
419 if (drvdata->numcidc > 4)
420 etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1);
421
422 for (i = 0; i < drvdata->numvmidc; i++)
423 etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i));
424 etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0);
425 if (drvdata->numvmidc > 4)
426 etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1);
427
428 if (!drvdata->skip_power_up) {
429 u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR);
430
431
432
433
434
435 etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR);
436 }
437
438
439
440
441
442 if (etm4x_is_ete(drvdata))
443 etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR);
444
445
446 etm4x_relaxed_write32(csa, 1, TRCPRGCTLR);
447
448
449 if (!csa->io_mem)
450 isb();
451
452
453 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
454 dev_err(etm_dev,
455 "timeout while waiting for Idle Trace Status\n");
456
457
458
459
460
461 dsb(sy);
462 isb();
463
464done:
465 etm4_cs_lock(drvdata, csa);
466
467 dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n",
468 drvdata->cpu, rc);
469 return rc;
470}
471
472static void etm4_enable_hw_smp_call(void *info)
473{
474 struct etm4_enable_arg *arg = info;
475
476 if (WARN_ON(!arg))
477 return;
478 arg->rc = etm4_enable_hw(arg->drvdata);
479}
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata)
495{
496 int ctridx, ret = -EINVAL;
497 int counter, rselector;
498 u32 val = 0;
499 struct etmv4_config *config = &drvdata->config;
500
501
502 if (!drvdata->nr_cntr)
503 goto out;
504
505
506 for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++)
507 if (config->cntr_val[ctridx] == 0)
508 break;
509
510
511 if (ctridx == drvdata->nr_cntr) {
512 pr_debug("%s: no available counter found\n", __func__);
513 ret = -ENOSPC;
514 goto out;
515 }
516
517
518
519
520
521
522
523 for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++)
524 if (!config->res_ctrl[rselector])
525 break;
526
527 if (rselector == drvdata->nr_resource * 2) {
528 pr_debug("%s: no available resource selector found\n",
529 __func__);
530 ret = -ENOSPC;
531 goto out;
532 }
533
534
535 counter = 1 << ctridx;
536
537
538
539
540
541 config->cntr_val[ctridx] = 1;
542 config->cntrldvr[ctridx] = 1;
543
544
545 val = 0x1 << 16 |
546 0x0 << 7 |
547 0x1;
548
549 config->cntr_ctrl[ctridx] = val;
550
551 val = 0x2 << 16 |
552 counter << 0;
553
554 config->res_ctrl[rselector] = val;
555
556 val = 0x0 << 7 |
557 rselector;
558
559 config->ts_ctrl = val;
560
561 ret = 0;
562out:
563 return ret;
564}
565
566static int etm4_parse_event_config(struct coresight_device *csdev,
567 struct perf_event *event)
568{
569 int ret = 0;
570 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
571 struct etmv4_config *config = &drvdata->config;
572 struct perf_event_attr *attr = &event->attr;
573 unsigned long cfg_hash;
574 int preset;
575
576
577 memset(config, 0, sizeof(struct etmv4_config));
578
579 if (attr->exclude_kernel)
580 config->mode = ETM_MODE_EXCL_KERN;
581
582 if (attr->exclude_user)
583 config->mode = ETM_MODE_EXCL_USER;
584
585
586 etm4_set_default_config(config);
587
588
589 ret = etm4_set_event_filters(drvdata, event);
590 if (ret)
591 goto out;
592
593
594 if (attr->config & BIT(ETM_OPT_CYCACC)) {
595 config->cfg |= BIT(4);
596
597 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
598 }
599 if (attr->config & BIT(ETM_OPT_TS)) {
600
601
602
603
604
605 ret = etm4_config_timestamp_event(drvdata);
606
607
608
609
610
611 if (ret)
612 goto out;
613
614
615 config->cfg |= BIT(11);
616 }
617
618 if (attr->config & BIT(ETM_OPT_CTXTID))
619
620 config->cfg |= BIT(ETM4_CFG_BIT_CTXTID);
621
622
623
624
625
626
627 if (attr->config & BIT(ETM_OPT_CTXTID2)) {
628 if (!is_kernel_in_hyp_mode()) {
629 ret = -EINVAL;
630 goto out;
631 }
632 config->cfg |= BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT);
633 }
634
635
636 if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
637
638 config->cfg |= BIT(12);
639
640
641
642
643
644
645
646
647
648 if (attr->config2 & GENMASK_ULL(63, 32)) {
649 cfg_hash = (u32)(attr->config2 >> 32);
650 preset = attr->config & 0xF;
651 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset);
652 }
653
654out:
655 return ret;
656}
657
658static int etm4_enable_perf(struct coresight_device *csdev,
659 struct perf_event *event)
660{
661 int ret = 0;
662 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
663
664 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
665 ret = -EINVAL;
666 goto out;
667 }
668
669
670 ret = etm4_parse_event_config(csdev, event);
671 if (ret)
672 goto out;
673
674 ret = etm4_enable_hw(drvdata);
675
676out:
677 return ret;
678}
679
680static int etm4_enable_sysfs(struct coresight_device *csdev)
681{
682 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
683 struct etm4_enable_arg arg = { };
684 int ret;
685
686 spin_lock(&drvdata->spinlock);
687
688
689
690
691
692 arg.drvdata = drvdata;
693 ret = smp_call_function_single(drvdata->cpu,
694 etm4_enable_hw_smp_call, &arg, 1);
695 if (!ret)
696 ret = arg.rc;
697 if (!ret)
698 drvdata->sticky_enable = true;
699 spin_unlock(&drvdata->spinlock);
700
701 if (!ret)
702 dev_dbg(&csdev->dev, "ETM tracing enabled\n");
703 return ret;
704}
705
706static int etm4_enable(struct coresight_device *csdev,
707 struct perf_event *event, u32 mode)
708{
709 int ret;
710 u32 val;
711 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
712
713 val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);
714
715
716 if (val)
717 return -EBUSY;
718
719 switch (mode) {
720 case CS_MODE_SYSFS:
721 ret = etm4_enable_sysfs(csdev);
722 break;
723 case CS_MODE_PERF:
724 ret = etm4_enable_perf(csdev, event);
725 break;
726 default:
727 ret = -EINVAL;
728 }
729
730
731 if (ret)
732 local_set(&drvdata->mode, CS_MODE_DISABLED);
733
734 return ret;
735}
736
737static void etm4_disable_hw(void *info)
738{
739 u32 control;
740 u64 trfcr;
741 struct etmv4_drvdata *drvdata = info;
742 struct etmv4_config *config = &drvdata->config;
743 struct coresight_device *csdev = drvdata->csdev;
744 struct device *etm_dev = &csdev->dev;
745 struct csdev_access *csa = &csdev->access;
746 int i;
747
748 etm4_cs_unlock(drvdata, csa);
749 etm4_disable_arch_specific(drvdata);
750
751 if (!drvdata->skip_power_up) {
752
753 control = etm4x_relaxed_read32(csa, TRCPDCR);
754 control &= ~TRCPDCR_PU;
755 etm4x_relaxed_write32(csa, control, TRCPDCR);
756 }
757
758 control = etm4x_relaxed_read32(csa, TRCPRGCTLR);
759
760
761 control &= ~0x1;
762
763
764
765
766
767 if (drvdata->trfc) {
768 trfcr = read_sysreg_s(SYS_TRFCR_EL1);
769 write_sysreg_s(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE),
770 SYS_TRFCR_EL1);
771 isb();
772 }
773
774
775
776
777
778 dsb(sy);
779 isb();
780
781 tsb_csync();
782 etm4x_relaxed_write32(csa, control, TRCPRGCTLR);
783
784
785 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1))
786 dev_err(etm_dev,
787 "timeout while waiting for PM stable Trace Status\n");
788 if (drvdata->trfc)
789 write_sysreg_s(trfcr, SYS_TRFCR_EL1);
790
791
792 for (i = 0; i < drvdata->nr_ss_cmp; i++) {
793 config->ss_status[i] =
794 etm4x_relaxed_read32(csa, TRCSSCSRn(i));
795 }
796
797
798 for (i = 0; i < drvdata->nr_cntr; i++) {
799 config->cntr_val[i] =
800 etm4x_relaxed_read32(csa, TRCCNTVRn(i));
801 }
802
803 coresight_disclaim_device_unlocked(csdev);
804 etm4_cs_lock(drvdata, csa);
805
806 dev_dbg(&drvdata->csdev->dev,
807 "cpu: %d disable smp call done\n", drvdata->cpu);
808}
809
810static int etm4_disable_perf(struct coresight_device *csdev,
811 struct perf_event *event)
812{
813 u32 control;
814 struct etm_filters *filters = event->hw.addr_filters;
815 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
816 struct perf_event_attr *attr = &event->attr;
817
818 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
819 return -EINVAL;
820
821 etm4_disable_hw(drvdata);
822
823
824
825
826 if (attr->config2 & GENMASK_ULL(63, 32))
827 cscfg_csdev_disable_active_config(csdev);
828
829
830
831
832
833
834
835 control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR);
836
837 filters->ssstatus = (control & BIT(9));
838
839 return 0;
840}
841
842static void etm4_disable_sysfs(struct coresight_device *csdev)
843{
844 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
845
846
847
848
849
850
851
852 cpus_read_lock();
853 spin_lock(&drvdata->spinlock);
854
855
856
857
858
859 smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);
860
861 spin_unlock(&drvdata->spinlock);
862 cpus_read_unlock();
863
864 dev_dbg(&csdev->dev, "ETM tracing disabled\n");
865}
866
867static void etm4_disable(struct coresight_device *csdev,
868 struct perf_event *event)
869{
870 u32 mode;
871 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
872
873
874
875
876
877
878 mode = local_read(&drvdata->mode);
879
880 switch (mode) {
881 case CS_MODE_DISABLED:
882 break;
883 case CS_MODE_SYSFS:
884 etm4_disable_sysfs(csdev);
885 break;
886 case CS_MODE_PERF:
887 etm4_disable_perf(csdev, event);
888 break;
889 }
890
891 if (mode)
892 local_set(&drvdata->mode, CS_MODE_DISABLED);
893}
894
895static const struct coresight_ops_source etm4_source_ops = {
896 .cpu_id = etm4_cpu_id,
897 .trace_id = etm4_trace_id,
898 .enable = etm4_enable,
899 .disable = etm4_disable,
900};
901
902static const struct coresight_ops etm4_cs_ops = {
903 .source_ops = &etm4_source_ops,
904};
905
906static inline bool cpu_supports_sysreg_trace(void)
907{
908 u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
909
910 return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0;
911}
912
913static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata,
914 struct csdev_access *csa)
915{
916 u32 devarch;
917
918 if (!cpu_supports_sysreg_trace())
919 return false;
920
921
922
923
924 devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH);
925 switch (devarch & ETM_DEVARCH_ID_MASK) {
926 case ETM_DEVARCH_ETMv4x_ARCH:
927 *csa = (struct csdev_access) {
928 .io_mem = false,
929 .read = etm4x_sysreg_read,
930 .write = etm4x_sysreg_write,
931 };
932 break;
933 case ETM_DEVARCH_ETE_ARCH:
934 *csa = (struct csdev_access) {
935 .io_mem = false,
936 .read = ete_sysreg_read,
937 .write = ete_sysreg_write,
938 };
939 break;
940 default:
941 return false;
942 }
943
944 drvdata->arch = etm_devarch_to_arch(devarch);
945 return true;
946}
947
948static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata,
949 struct csdev_access *csa)
950{
951 u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH);
952 u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1);
953
954
955
956
957
958
959
960 if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) {
961 drvdata->arch = etm_devarch_to_arch(devarch);
962 } else {
963 pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n",
964 smp_processor_id(), devarch);
965
966 if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4)
967 return false;
968 drvdata->arch = etm_trcidr_to_arch(idr1);
969 }
970
971 *csa = CSDEV_ACCESS_IOMEM(drvdata->base);
972 return true;
973}
974
975static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata,
976 struct csdev_access *csa)
977{
978
979
980
981
982
983 if (drvdata->base)
984 return etm4_init_iomem_access(drvdata, csa);
985
986 if (etm4_init_sysreg_access(drvdata, csa))
987 return true;
988
989 return false;
990}
991
992static void cpu_enable_tracing(struct etmv4_drvdata *drvdata)
993{
994 u64 dfr0 = read_sysreg(id_aa64dfr0_el1);
995 u64 trfcr;
996
997 if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT))
998 return;
999
1000 drvdata->trfc = true;
1001
1002
1003
1004
1005
1006 trfcr = (TRFCR_ELx_TS_VIRTUAL |
1007 TRFCR_ELx_ExTRE |
1008 TRFCR_ELx_E0TRE);
1009
1010
1011 if (is_kernel_in_hyp_mode())
1012 trfcr |= TRFCR_EL2_CX;
1013
1014 write_sysreg_s(trfcr, SYS_TRFCR_EL1);
1015}
1016
1017static void etm4_init_arch_data(void *info)
1018{
1019 u32 etmidr0;
1020 u32 etmidr2;
1021 u32 etmidr3;
1022 u32 etmidr4;
1023 u32 etmidr5;
1024 struct etm4_init_arg *init_arg = info;
1025 struct etmv4_drvdata *drvdata;
1026 struct csdev_access *csa;
1027 int i;
1028
1029 drvdata = init_arg->drvdata;
1030 csa = init_arg->csa;
1031
1032
1033
1034
1035
1036
1037 if (!etm4_init_csdev_access(drvdata, csa))
1038 return;
1039
1040
1041 etm_detect_os_lock(drvdata, csa);
1042
1043
1044 etm4_os_unlock_csa(drvdata, csa);
1045 etm4_cs_unlock(drvdata, csa);
1046
1047 etm4_check_arch_features(drvdata, init_arg->pid);
1048
1049
1050 etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0);
1051
1052
1053 if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2))
1054 drvdata->instrp0 = true;
1055 else
1056 drvdata->instrp0 = false;
1057
1058
1059 if (BMVAL(etmidr0, 5, 5))
1060 drvdata->trcbb = true;
1061 else
1062 drvdata->trcbb = false;
1063
1064
1065 if (BMVAL(etmidr0, 6, 6))
1066 drvdata->trccond = true;
1067 else
1068 drvdata->trccond = false;
1069
1070
1071 if (BMVAL(etmidr0, 7, 7))
1072 drvdata->trccci = true;
1073 else
1074 drvdata->trccci = false;
1075
1076
1077 if (BMVAL(etmidr0, 9, 9))
1078 drvdata->retstack = true;
1079 else
1080 drvdata->retstack = false;
1081
1082
1083 drvdata->nr_event = BMVAL(etmidr0, 10, 11);
1084
1085 drvdata->q_support = BMVAL(etmidr0, 15, 16);
1086
1087 drvdata->ts_size = BMVAL(etmidr0, 24, 28);
1088
1089
1090 etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2);
1091
1092 drvdata->ctxid_size = BMVAL(etmidr2, 5, 9);
1093
1094 drvdata->vmid_size = BMVAL(etmidr2, 10, 14);
1095
1096 drvdata->ccsize = BMVAL(etmidr2, 25, 28);
1097
1098 etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3);
1099
1100 drvdata->ccitmin = BMVAL(etmidr3, 0, 11);
1101
1102 drvdata->s_ex_level = BMVAL(etmidr3, 16, 19);
1103 drvdata->config.s_ex_level = drvdata->s_ex_level;
1104
1105 drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23);
1106
1107
1108
1109
1110
1111 if (BMVAL(etmidr3, 24, 24))
1112 drvdata->trc_error = true;
1113 else
1114 drvdata->trc_error = false;
1115
1116
1117 if (BMVAL(etmidr3, 25, 25))
1118 drvdata->syncpr = true;
1119 else
1120 drvdata->syncpr = false;
1121
1122
1123 if (BMVAL(etmidr3, 26, 26))
1124 drvdata->stallctl = true;
1125 else
1126 drvdata->stallctl = false;
1127
1128
1129 if (BMVAL(etmidr3, 27, 27))
1130 drvdata->sysstall = true;
1131 else
1132 drvdata->sysstall = false;
1133
1134
1135
1136
1137
1138
1139
1140 drvdata->nr_pe = (BMVAL(etmidr3, 12, 13) << 3) | BMVAL(etmidr3, 28, 30);
1141
1142
1143 if (BMVAL(etmidr3, 31, 31))
1144 drvdata->nooverflow = true;
1145 else
1146 drvdata->nooverflow = false;
1147
1148
1149 etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4);
1150
1151 drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3);
1152
1153 drvdata->nr_pe_cmp = BMVAL(etmidr4, 12, 15);
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164 drvdata->nr_resource = BMVAL(etmidr4, 16, 19);
1165 if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0))
1166 drvdata->nr_resource += 1;
1167
1168
1169
1170
1171
1172 drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23);
1173 for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1174 drvdata->config.ss_status[i] =
1175 etm4x_relaxed_read32(csa, TRCSSCSRn(i));
1176 }
1177
1178 drvdata->numcidc = BMVAL(etmidr4, 24, 27);
1179
1180 drvdata->numvmidc = BMVAL(etmidr4, 28, 31);
1181
1182 etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5);
1183
1184 drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8);
1185
1186 drvdata->trcid_size = BMVAL(etmidr5, 16, 21);
1187
1188 if (BMVAL(etmidr5, 22, 22))
1189 drvdata->atbtrig = true;
1190 else
1191 drvdata->atbtrig = false;
1192
1193
1194
1195
1196 if (BMVAL(etmidr5, 23, 23) && (!drvdata->skip_power_up))
1197 drvdata->lpoverride = true;
1198 else
1199 drvdata->lpoverride = false;
1200
1201 drvdata->nrseqstate = BMVAL(etmidr5, 25, 27);
1202
1203 drvdata->nr_cntr = BMVAL(etmidr5, 28, 30);
1204 etm4_cs_lock(drvdata, csa);
1205 cpu_enable_tracing(drvdata);
1206}
1207
1208static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config)
1209{
1210 return etm4_get_access_type(config) << TRCVICTLR_EXLEVEL_SHIFT;
1211}
1212
1213
1214static void etm4_set_victlr_access(struct etmv4_config *config)
1215{
1216 config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK;
1217 config->vinst_ctrl |= etm4_get_victlr_access_type(config);
1218}
1219
1220static void etm4_set_default_config(struct etmv4_config *config)
1221{
1222
1223 config->eventctrl0 = 0x0;
1224 config->eventctrl1 = 0x0;
1225
1226
1227 config->stall_ctrl = 0x0;
1228
1229
1230 config->syncfreq = 0xC;
1231
1232
1233 config->ts_ctrl = 0x0;
1234
1235
1236 config->vinst_ctrl = BIT(0);
1237
1238
1239 etm4_set_victlr_access(config);
1240}
1241
1242static u64 etm4_get_ns_access_type(struct etmv4_config *config)
1243{
1244 u64 access_type = 0;
1245
1246
1247
1248
1249
1250
1251 if (!is_kernel_in_hyp_mode()) {
1252
1253 access_type = ETM_EXLEVEL_NS_HYP;
1254 if (config->mode & ETM_MODE_EXCL_KERN)
1255 access_type |= ETM_EXLEVEL_NS_OS;
1256 } else if (config->mode & ETM_MODE_EXCL_KERN) {
1257 access_type = ETM_EXLEVEL_NS_HYP;
1258 }
1259
1260 if (config->mode & ETM_MODE_EXCL_USER)
1261 access_type |= ETM_EXLEVEL_NS_APP;
1262
1263 return access_type;
1264}
1265
1266
1267
1268
1269
1270
1271static u64 etm4_get_access_type(struct etmv4_config *config)
1272{
1273
1274 return etm4_get_ns_access_type(config) | (u64)config->s_ex_level;
1275}
1276
1277static u64 etm4_get_comparator_access_type(struct etmv4_config *config)
1278{
1279 return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT;
1280}
1281
1282static void etm4_set_comparator_filter(struct etmv4_config *config,
1283 u64 start, u64 stop, int comparator)
1284{
1285 u64 access_type = etm4_get_comparator_access_type(config);
1286
1287
1288 config->addr_val[comparator] = start;
1289 config->addr_acc[comparator] = access_type;
1290 config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
1291
1292
1293 config->addr_val[comparator + 1] = stop;
1294 config->addr_acc[comparator + 1] = access_type;
1295 config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312 config->viiectlr |= BIT(comparator / 2);
1313}
1314
1315static void etm4_set_start_stop_filter(struct etmv4_config *config,
1316 u64 address, int comparator,
1317 enum etm_addr_type type)
1318{
1319 int shift;
1320 u64 access_type = etm4_get_comparator_access_type(config);
1321
1322
1323 config->addr_val[comparator] = address;
1324 config->addr_acc[comparator] = access_type;
1325 config->addr_type[comparator] = type;
1326
1327
1328
1329
1330
1331
1332 shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
1333 config->vissctlr |= BIT(shift + comparator);
1334}
1335
1336static void etm4_set_default_filter(struct etmv4_config *config)
1337{
1338
1339 config->viiectlr = 0x0;
1340
1341
1342
1343
1344
1345 config->vinst_ctrl |= BIT(9);
1346 config->mode |= ETM_MODE_VIEWINST_STARTSTOP;
1347
1348
1349 config->vissctlr = 0x0;
1350}
1351
1352static void etm4_set_default(struct etmv4_config *config)
1353{
1354 if (WARN_ON_ONCE(!config))
1355 return;
1356
1357
1358
1359
1360
1361
1362
1363
1364 etm4_set_default_config(config);
1365 etm4_set_default_filter(config);
1366}
1367
1368static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
1369{
1370 int nr_comparator, index = 0;
1371 struct etmv4_config *config = &drvdata->config;
1372
1373
1374
1375
1376
1377 nr_comparator = drvdata->nr_addr_cmp * 2;
1378
1379
1380 while (index < nr_comparator) {
1381 switch (type) {
1382 case ETM_ADDR_TYPE_RANGE:
1383 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
1384 config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
1385 return index;
1386
1387
1388 index += 2;
1389 break;
1390 case ETM_ADDR_TYPE_START:
1391 case ETM_ADDR_TYPE_STOP:
1392 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
1393 return index;
1394
1395
1396 index += 1;
1397 break;
1398 default:
1399 return -EINVAL;
1400 }
1401 }
1402
1403
1404 return -ENOSPC;
1405}
1406
1407static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
1408 struct perf_event *event)
1409{
1410 int i, comparator, ret = 0;
1411 u64 address;
1412 struct etmv4_config *config = &drvdata->config;
1413 struct etm_filters *filters = event->hw.addr_filters;
1414
1415 if (!filters)
1416 goto default_filter;
1417
1418
1419 perf_event_addr_filters_sync(event);
1420
1421
1422
1423
1424
1425 if (!filters->nr_filters)
1426 goto default_filter;
1427
1428 for (i = 0; i < filters->nr_filters; i++) {
1429 struct etm_filter *filter = &filters->etm_filter[i];
1430 enum etm_addr_type type = filter->type;
1431
1432
1433 comparator = etm4_get_next_comparator(drvdata, type);
1434 if (comparator < 0) {
1435 ret = comparator;
1436 goto out;
1437 }
1438
1439 switch (type) {
1440 case ETM_ADDR_TYPE_RANGE:
1441 etm4_set_comparator_filter(config,
1442 filter->start_addr,
1443 filter->stop_addr,
1444 comparator);
1445
1446
1447
1448
1449 config->vinst_ctrl |= BIT(9);
1450
1451
1452 config->vissctlr = 0x0;
1453 break;
1454 case ETM_ADDR_TYPE_START:
1455 case ETM_ADDR_TYPE_STOP:
1456
1457 address = (type == ETM_ADDR_TYPE_START ?
1458 filter->start_addr :
1459 filter->stop_addr);
1460
1461
1462 etm4_set_start_stop_filter(config, address,
1463 comparator, type);
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476 if (filters->ssstatus)
1477 config->vinst_ctrl |= BIT(9);
1478
1479
1480 config->viiectlr = 0x0;
1481 break;
1482 default:
1483 ret = -EINVAL;
1484 goto out;
1485 }
1486 }
1487
1488 goto out;
1489
1490
1491default_filter:
1492 etm4_set_default_filter(config);
1493
1494out:
1495 return ret;
1496}
1497
1498void etm4_config_trace_mode(struct etmv4_config *config)
1499{
1500 u32 mode;
1501
1502 mode = config->mode;
1503 mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);
1504
1505
1506 WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));
1507
1508
1509 if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
1510 return;
1511
1512 etm4_set_victlr_access(config);
1513}
1514
1515static int etm4_online_cpu(unsigned int cpu)
1516{
1517 if (!etmdrvdata[cpu])
1518 return 0;
1519
1520 if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
1521 coresight_enable(etmdrvdata[cpu]->csdev);
1522 return 0;
1523}
1524
1525static int etm4_starting_cpu(unsigned int cpu)
1526{
1527 if (!etmdrvdata[cpu])
1528 return 0;
1529
1530 spin_lock(&etmdrvdata[cpu]->spinlock);
1531 if (!etmdrvdata[cpu]->os_unlock)
1532 etm4_os_unlock(etmdrvdata[cpu]);
1533
1534 if (local_read(&etmdrvdata[cpu]->mode))
1535 etm4_enable_hw(etmdrvdata[cpu]);
1536 spin_unlock(&etmdrvdata[cpu]->spinlock);
1537 return 0;
1538}
1539
1540static int etm4_dying_cpu(unsigned int cpu)
1541{
1542 if (!etmdrvdata[cpu])
1543 return 0;
1544
1545 spin_lock(&etmdrvdata[cpu]->spinlock);
1546 if (local_read(&etmdrvdata[cpu]->mode))
1547 etm4_disable_hw(etmdrvdata[cpu]);
1548 spin_unlock(&etmdrvdata[cpu]->spinlock);
1549 return 0;
1550}
1551
1552static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
1553{
1554 drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
1555}
1556
1557static int etm4_cpu_save(struct etmv4_drvdata *drvdata)
1558{
1559 int i, ret = 0;
1560 struct etmv4_save_state *state;
1561 struct coresight_device *csdev = drvdata->csdev;
1562 struct csdev_access *csa;
1563 struct device *etm_dev;
1564
1565 if (WARN_ON(!csdev))
1566 return -ENODEV;
1567
1568 etm_dev = &csdev->dev;
1569 csa = &csdev->access;
1570
1571
1572
1573
1574
1575 dsb(sy);
1576 isb();
1577
1578 etm4_cs_unlock(drvdata, csa);
1579
1580 etm4_os_lock(drvdata);
1581
1582
1583 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) {
1584 dev_err(etm_dev,
1585 "timeout while waiting for PM Stable Status\n");
1586 etm4_os_unlock(drvdata);
1587 ret = -EBUSY;
1588 goto out;
1589 }
1590
1591 state = drvdata->save_state;
1592
1593 state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR);
1594 if (drvdata->nr_pe)
1595 state->trcprocselr = etm4x_read32(csa, TRCPROCSELR);
1596 state->trcconfigr = etm4x_read32(csa, TRCCONFIGR);
1597 state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR);
1598 state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R);
1599 state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R);
1600 if (drvdata->stallctl)
1601 state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR);
1602 state->trctsctlr = etm4x_read32(csa, TRCTSCTLR);
1603 state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR);
1604 state->trcccctlr = etm4x_read32(csa, TRCCCCTLR);
1605 state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR);
1606 state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR);
1607 state->trcqctlr = etm4x_read32(csa, TRCQCTLR);
1608
1609 state->trcvictlr = etm4x_read32(csa, TRCVICTLR);
1610 state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR);
1611 state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR);
1612 if (drvdata->nr_pe_cmp)
1613 state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR);
1614 state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR);
1615 state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR);
1616 state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR);
1617
1618 for (i = 0; i < drvdata->nrseqstate - 1; i++)
1619 state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i));
1620
1621 state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR);
1622 state->trcseqstr = etm4x_read32(csa, TRCSEQSTR);
1623 state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR);
1624
1625 for (i = 0; i < drvdata->nr_cntr; i++) {
1626 state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i));
1627 state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i));
1628 state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i));
1629 }
1630
1631 for (i = 0; i < drvdata->nr_resource * 2; i++)
1632 state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i));
1633
1634 for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1635 state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i));
1636 state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i));
1637 if (etm4x_sspcicrn_present(drvdata, i))
1638 state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i));
1639 }
1640
1641 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1642 state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i));
1643 state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i));
1644 }
1645
1646
1647
1648
1649
1650
1651
1652
1653 for (i = 0; i < drvdata->numcidc; i++)
1654 state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i));
1655
1656 for (i = 0; i < drvdata->numvmidc; i++)
1657 state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i));
1658
1659 state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0);
1660 if (drvdata->numcidc > 4)
1661 state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1);
1662
1663 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0);
1664 if (drvdata->numvmidc > 4)
1665 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1);
1666
1667 state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR);
1668
1669 if (!drvdata->skip_power_up)
1670 state->trcpdcr = etm4x_read32(csa, TRCPDCR);
1671
1672
1673 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) {
1674 dev_err(etm_dev,
1675 "timeout while waiting for Idle Trace Status\n");
1676 etm4_os_unlock(drvdata);
1677 ret = -EBUSY;
1678 goto out;
1679 }
1680
1681 drvdata->state_needs_restore = true;
1682
1683
1684
1685
1686
1687
1688 if (!drvdata->skip_power_up)
1689 etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU),
1690 TRCPDCR);
1691out:
1692 etm4_cs_lock(drvdata, csa);
1693 return ret;
1694}
1695
1696static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)
1697{
1698 int i;
1699 struct etmv4_save_state *state = drvdata->save_state;
1700 struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base);
1701 struct csdev_access *csa = &tmp_csa;
1702
1703 etm4_cs_unlock(drvdata, csa);
1704 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
1705
1706 etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR);
1707 if (drvdata->nr_pe)
1708 etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR);
1709 etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR);
1710 etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR);
1711 etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R);
1712 etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R);
1713 if (drvdata->stallctl)
1714 etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR);
1715 etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR);
1716 etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR);
1717 etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR);
1718 etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR);
1719 etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR);
1720 etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR);
1721
1722 etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR);
1723 etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR);
1724 etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR);
1725 if (drvdata->nr_pe_cmp)
1726 etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR);
1727 etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR);
1728 etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR);
1729 etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR);
1730
1731 for (i = 0; i < drvdata->nrseqstate - 1; i++)
1732 etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i));
1733
1734 etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR);
1735 etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR);
1736 etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR);
1737
1738 for (i = 0; i < drvdata->nr_cntr; i++) {
1739 etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i));
1740 etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i));
1741 etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i));
1742 }
1743
1744 for (i = 0; i < drvdata->nr_resource * 2; i++)
1745 etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i));
1746
1747 for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1748 etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i));
1749 etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i));
1750 if (etm4x_sspcicrn_present(drvdata, i))
1751 etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i));
1752 }
1753
1754 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1755 etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i));
1756 etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i));
1757 }
1758
1759 for (i = 0; i < drvdata->numcidc; i++)
1760 etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i));
1761
1762 for (i = 0; i < drvdata->numvmidc; i++)
1763 etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i));
1764
1765 etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0);
1766 if (drvdata->numcidc > 4)
1767 etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1);
1768
1769 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0);
1770 if (drvdata->numvmidc > 4)
1771 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1);
1772
1773 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
1774
1775 if (!drvdata->skip_power_up)
1776 etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR);
1777
1778 drvdata->state_needs_restore = false;
1779
1780
1781
1782
1783
1784 dsb(sy);
1785 isb();
1786
1787
1788 etm4_os_unlock(drvdata);
1789 etm4_cs_lock(drvdata, csa);
1790}
1791
1792static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
1793 void *v)
1794{
1795 struct etmv4_drvdata *drvdata;
1796 unsigned int cpu = smp_processor_id();
1797
1798 if (!etmdrvdata[cpu])
1799 return NOTIFY_OK;
1800
1801 drvdata = etmdrvdata[cpu];
1802
1803 if (!drvdata->save_state)
1804 return NOTIFY_OK;
1805
1806 if (WARN_ON_ONCE(drvdata->cpu != cpu))
1807 return NOTIFY_BAD;
1808
1809 switch (cmd) {
1810 case CPU_PM_ENTER:
1811
1812 if (local_read(&drvdata->mode))
1813 if (etm4_cpu_save(drvdata))
1814 return NOTIFY_BAD;
1815 break;
1816 case CPU_PM_EXIT:
1817 case CPU_PM_ENTER_FAILED:
1818 if (drvdata->state_needs_restore)
1819 etm4_cpu_restore(drvdata);
1820 break;
1821 default:
1822 return NOTIFY_DONE;
1823 }
1824
1825 return NOTIFY_OK;
1826}
1827
1828static struct notifier_block etm4_cpu_pm_nb = {
1829 .notifier_call = etm4_cpu_pm_notify,
1830};
1831
1832
1833static int __init etm4_pm_setup(void)
1834{
1835 int ret;
1836
1837 ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);
1838 if (ret)
1839 return ret;
1840
1841 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
1842 "arm/coresight4:starting",
1843 etm4_starting_cpu, etm4_dying_cpu);
1844
1845 if (ret)
1846 goto unregister_notifier;
1847
1848 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
1849 "arm/coresight4:online",
1850 etm4_online_cpu, NULL);
1851
1852
1853 if (ret > 0) {
1854 hp_online = ret;
1855 return 0;
1856 }
1857
1858
1859 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1860
1861unregister_notifier:
1862 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
1863 return ret;
1864}
1865
1866static void etm4_pm_clear(void)
1867{
1868 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
1869 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1870 if (hp_online) {
1871 cpuhp_remove_state_nocalls(hp_online);
1872 hp_online = 0;
1873 }
1874}
1875
1876static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid)
1877{
1878 int ret;
1879 struct coresight_platform_data *pdata = NULL;
1880 struct etmv4_drvdata *drvdata;
1881 struct coresight_desc desc = { 0 };
1882 struct etm4_init_arg init_arg = { 0 };
1883 u8 major, minor;
1884 char *type_name;
1885
1886 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1887 if (!drvdata)
1888 return -ENOMEM;
1889
1890 dev_set_drvdata(dev, drvdata);
1891
1892 if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
1893 pm_save_enable = coresight_loses_context_with_cpu(dev) ?
1894 PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;
1895
1896 if (pm_save_enable != PARAM_PM_SAVE_NEVER) {
1897 drvdata->save_state = devm_kmalloc(dev,
1898 sizeof(struct etmv4_save_state), GFP_KERNEL);
1899 if (!drvdata->save_state)
1900 return -ENOMEM;
1901 }
1902
1903 drvdata->base = base;
1904
1905 spin_lock_init(&drvdata->spinlock);
1906
1907 drvdata->cpu = coresight_get_cpu(dev);
1908 if (drvdata->cpu < 0)
1909 return drvdata->cpu;
1910
1911 init_arg.drvdata = drvdata;
1912 init_arg.csa = &desc.access;
1913 init_arg.pid = etm_pid;
1914
1915 if (smp_call_function_single(drvdata->cpu,
1916 etm4_init_arch_data, &init_arg, 1))
1917 dev_err(dev, "ETM arch init failed\n");
1918
1919 if (!drvdata->arch)
1920 return -EINVAL;
1921
1922
1923 if (!desc.access.io_mem ||
1924 fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up"))
1925 drvdata->skip_power_up = true;
1926
1927 major = ETM_ARCH_MAJOR_VERSION(drvdata->arch);
1928 minor = ETM_ARCH_MINOR_VERSION(drvdata->arch);
1929
1930 if (etm4x_is_ete(drvdata)) {
1931 type_name = "ete";
1932
1933 major -= 4;
1934 } else {
1935 type_name = "etm";
1936 }
1937
1938 desc.name = devm_kasprintf(dev, GFP_KERNEL,
1939 "%s%d", type_name, drvdata->cpu);
1940 if (!desc.name)
1941 return -ENOMEM;
1942
1943 etm4_init_trace_id(drvdata);
1944 etm4_set_default(&drvdata->config);
1945
1946 pdata = coresight_get_platform_data(dev);
1947 if (IS_ERR(pdata))
1948 return PTR_ERR(pdata);
1949
1950 dev->platform_data = pdata;
1951
1952 desc.type = CORESIGHT_DEV_TYPE_SOURCE;
1953 desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
1954 desc.ops = &etm4_cs_ops;
1955 desc.pdata = pdata;
1956 desc.dev = dev;
1957 desc.groups = coresight_etmv4_groups;
1958 drvdata->csdev = coresight_register(&desc);
1959 if (IS_ERR(drvdata->csdev))
1960 return PTR_ERR(drvdata->csdev);
1961
1962 ret = etm_perf_symlink(drvdata->csdev, true);
1963 if (ret) {
1964 coresight_unregister(drvdata->csdev);
1965 return ret;
1966 }
1967
1968
1969 ret = etm4_cscfg_register(drvdata->csdev);
1970 if (ret) {
1971 coresight_unregister(drvdata->csdev);
1972 return ret;
1973 }
1974
1975 etmdrvdata[drvdata->cpu] = drvdata;
1976
1977 dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n",
1978 drvdata->cpu, type_name, major, minor);
1979
1980 if (boot_enable) {
1981 coresight_enable(drvdata->csdev);
1982 drvdata->boot_enable = true;
1983 }
1984
1985 return 0;
1986}
1987
1988static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id)
1989{
1990 void __iomem *base;
1991 struct device *dev = &adev->dev;
1992 struct resource *res = &adev->res;
1993 int ret;
1994
1995
1996 base = devm_ioremap_resource(dev, res);
1997 if (IS_ERR(base))
1998 return PTR_ERR(base);
1999
2000 ret = etm4_probe(dev, base, id->id);
2001 if (!ret)
2002 pm_runtime_put(&adev->dev);
2003
2004 return ret;
2005}
2006
2007static int etm4_probe_platform_dev(struct platform_device *pdev)
2008{
2009 int ret;
2010
2011 pm_runtime_get_noresume(&pdev->dev);
2012 pm_runtime_set_active(&pdev->dev);
2013 pm_runtime_enable(&pdev->dev);
2014
2015
2016
2017
2018
2019
2020 ret = etm4_probe(&pdev->dev, NULL, 0);
2021
2022 pm_runtime_put(&pdev->dev);
2023 return ret;
2024}
2025
2026static struct amba_cs_uci_id uci_id_etm4[] = {
2027 {
2028
2029 .devarch = ETM_DEVARCH_ETMv4x_ARCH,
2030 .devarch_mask = ETM_DEVARCH_ID_MASK,
2031 .devtype = 0x00000013,
2032 }
2033};
2034
2035static void clear_etmdrvdata(void *info)
2036{
2037 int cpu = *(int *)info;
2038
2039 etmdrvdata[cpu] = NULL;
2040}
2041
2042static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata)
2043{
2044 etm_perf_symlink(drvdata->csdev, false);
2045
2046
2047
2048
2049 cpus_read_lock();
2050
2051
2052
2053
2054
2055
2056 if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1))
2057 etmdrvdata[drvdata->cpu] = NULL;
2058
2059 cpus_read_unlock();
2060
2061 cscfg_unregister_csdev(drvdata->csdev);
2062 coresight_unregister(drvdata->csdev);
2063
2064 return 0;
2065}
2066
2067static void __exit etm4_remove_amba(struct amba_device *adev)
2068{
2069 struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev);
2070
2071 if (drvdata)
2072 etm4_remove_dev(drvdata);
2073}
2074
2075static int __exit etm4_remove_platform_dev(struct platform_device *pdev)
2076{
2077 int ret = 0;
2078 struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev);
2079
2080 if (drvdata)
2081 ret = etm4_remove_dev(drvdata);
2082 pm_runtime_disable(&pdev->dev);
2083 return ret;
2084}
2085
2086static const struct amba_id etm4_ids[] = {
2087 CS_AMBA_ID(0x000bb95d),
2088 CS_AMBA_ID(0x000bb95e),
2089 CS_AMBA_ID(0x000bb95a),
2090 CS_AMBA_ID(0x000bb959),
2091 CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),
2092 CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),
2093 CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),
2094 CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),
2095 CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),
2096 CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),
2097 CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),
2098 CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),
2099 CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),
2100 CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),
2101 CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),
2102 CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),
2103 CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),
2104 CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),
2105 {},
2106};
2107
2108MODULE_DEVICE_TABLE(amba, etm4_ids);
2109
2110static struct amba_driver etm4x_amba_driver = {
2111 .drv = {
2112 .name = "coresight-etm4x",
2113 .owner = THIS_MODULE,
2114 .suppress_bind_attrs = true,
2115 },
2116 .probe = etm4_probe_amba,
2117 .remove = etm4_remove_amba,
2118 .id_table = etm4_ids,
2119};
2120
2121static const struct of_device_id etm4_sysreg_match[] = {
2122 { .compatible = "arm,coresight-etm4x-sysreg" },
2123 { .compatible = "arm,embedded-trace-extension" },
2124 {}
2125};
2126
2127static struct platform_driver etm4_platform_driver = {
2128 .probe = etm4_probe_platform_dev,
2129 .remove = etm4_remove_platform_dev,
2130 .driver = {
2131 .name = "coresight-etm4x",
2132 .of_match_table = etm4_sysreg_match,
2133 .suppress_bind_attrs = true,
2134 },
2135};
2136
2137static int __init etm4x_init(void)
2138{
2139 int ret;
2140
2141 ret = etm4_pm_setup();
2142
2143
2144 if (ret)
2145 return ret;
2146
2147 ret = amba_driver_register(&etm4x_amba_driver);
2148 if (ret) {
2149 pr_err("Error registering etm4x AMBA driver\n");
2150 goto clear_pm;
2151 }
2152
2153 ret = platform_driver_register(&etm4_platform_driver);
2154 if (!ret)
2155 return 0;
2156
2157 pr_err("Error registering etm4x platform driver\n");
2158 amba_driver_unregister(&etm4x_amba_driver);
2159
2160clear_pm:
2161 etm4_pm_clear();
2162 return ret;
2163}
2164
2165static void __exit etm4x_exit(void)
2166{
2167 amba_driver_unregister(&etm4x_amba_driver);
2168 platform_driver_unregister(&etm4_platform_driver);
2169 etm4_pm_clear();
2170}
2171
2172module_init(etm4x_init);
2173module_exit(etm4x_exit);
2174
2175MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>");
2176MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>");
2177MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver");
2178MODULE_LICENSE("GPL v2");
2179