1
2
3
4
5
6
7#include <linux/atomic.h>
8#include <linux/coresight.h>
9#include <linux/dma-mapping.h>
10#include <linux/iommu.h>
11#include <linux/idr.h>
12#include <linux/mutex.h>
13#include <linux/refcount.h>
14#include <linux/slab.h>
15#include <linux/types.h>
16#include <linux/vmalloc.h>
17#include "coresight-catu.h"
18#include "coresight-etm-perf.h"
19#include "coresight-priv.h"
20#include "coresight-tmc.h"
21
22struct etr_flat_buf {
23 struct device *dev;
24 dma_addr_t daddr;
25 void *vaddr;
26 size_t size;
27};
28
29
30
31
32
33
34
35
36
37
38
39struct etr_perf_buffer {
40 struct tmc_drvdata *drvdata;
41 struct etr_buf *etr_buf;
42 pid_t pid;
43 bool snapshot;
44 unsigned long head;
45 int nr_pages;
46 void **pages;
47};
48
49
50#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
51
52
53#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77typedef u32 sgte_t;
78
79#define ETR_SG_PAGE_SHIFT 12
80#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
81#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
82#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
83#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
84
85#define ETR_SG_ET_MASK 0x3
86#define ETR_SG_ET_LAST 0x1
87#define ETR_SG_ET_NORMAL 0x2
88#define ETR_SG_ET_LINK 0x3
89
90#define ETR_SG_ADDR_SHIFT 4
91
92#define ETR_SG_ENTRY(addr, type) \
93 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
94 (type & ETR_SG_ET_MASK))
95
96#define ETR_SG_ADDR(entry) \
97 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
98#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
99
100
101
102
103
104
105
106struct etr_sg_table {
107 struct tmc_sg_table *sg_table;
108 dma_addr_t hwaddr;
109};
110
111
112
113
114
115
116
117
118
119
120
121static inline unsigned long __attribute_const__
122tmc_etr_sg_table_entries(int nr_pages)
123{
124 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
125 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
126
127
128
129
130
131 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
132 nr_sglinks--;
133 return nr_sgpages + nr_sglinks;
134}
135
136
137
138
139
140
141static long
142tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
143{
144 int i;
145 dma_addr_t page_start;
146
147 for (i = 0; i < tmc_pages->nr_pages; i++) {
148 page_start = tmc_pages->daddrs[i];
149 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
150 return i * PAGE_SIZE + (addr - page_start);
151 }
152
153 return -EINVAL;
154}
155
156
157
158
159
160
161static void tmc_pages_free(struct tmc_pages *tmc_pages,
162 struct device *dev, enum dma_data_direction dir)
163{
164 int i;
165
166 for (i = 0; i < tmc_pages->nr_pages; i++) {
167 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
168 dma_unmap_page(dev, tmc_pages->daddrs[i],
169 PAGE_SIZE, dir);
170 if (tmc_pages->pages && tmc_pages->pages[i])
171 __free_page(tmc_pages->pages[i]);
172 }
173
174 kfree(tmc_pages->pages);
175 kfree(tmc_pages->daddrs);
176 tmc_pages->pages = NULL;
177 tmc_pages->daddrs = NULL;
178 tmc_pages->nr_pages = 0;
179}
180
181
182
183
184
185
186
187
188
189static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
190 struct device *dev, int node,
191 enum dma_data_direction dir, void **pages)
192{
193 int i, nr_pages;
194 dma_addr_t paddr;
195 struct page *page;
196
197 nr_pages = tmc_pages->nr_pages;
198 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
199 GFP_KERNEL);
200 if (!tmc_pages->daddrs)
201 return -ENOMEM;
202 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
203 GFP_KERNEL);
204 if (!tmc_pages->pages) {
205 kfree(tmc_pages->daddrs);
206 tmc_pages->daddrs = NULL;
207 return -ENOMEM;
208 }
209
210 for (i = 0; i < nr_pages; i++) {
211 if (pages && pages[i]) {
212 page = virt_to_page(pages[i]);
213
214 get_page(page);
215 } else {
216 page = alloc_pages_node(node,
217 GFP_KERNEL | __GFP_ZERO, 0);
218 }
219 paddr = dma_map_page(dev, page, 0, PAGE_SIZE, dir);
220 if (dma_mapping_error(dev, paddr))
221 goto err;
222 tmc_pages->daddrs[i] = paddr;
223 tmc_pages->pages[i] = page;
224 }
225 return 0;
226err:
227 tmc_pages_free(tmc_pages, dev, dir);
228 return -ENOMEM;
229}
230
231static inline long
232tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
233{
234 return tmc_pages_get_offset(&sg_table->data_pages, addr);
235}
236
237static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
238{
239 if (sg_table->table_vaddr)
240 vunmap(sg_table->table_vaddr);
241 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
242}
243
244static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
245{
246 if (sg_table->data_vaddr)
247 vunmap(sg_table->data_vaddr);
248 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
249}
250
251void tmc_free_sg_table(struct tmc_sg_table *sg_table)
252{
253 tmc_free_table_pages(sg_table);
254 tmc_free_data_pages(sg_table);
255}
256
257
258
259
260
261
262static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
263{
264 int rc;
265 struct tmc_pages *table_pages = &sg_table->table_pages;
266
267 rc = tmc_pages_alloc(table_pages, sg_table->dev,
268 dev_to_node(sg_table->dev),
269 DMA_TO_DEVICE, NULL);
270 if (rc)
271 return rc;
272 sg_table->table_vaddr = vmap(table_pages->pages,
273 table_pages->nr_pages,
274 VM_MAP,
275 PAGE_KERNEL);
276 if (!sg_table->table_vaddr)
277 rc = -ENOMEM;
278 else
279 sg_table->table_daddr = table_pages->daddrs[0];
280 return rc;
281}
282
283static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
284{
285 int rc;
286
287
288 rc = tmc_pages_alloc(&sg_table->data_pages,
289 sg_table->dev, sg_table->node,
290 DMA_FROM_DEVICE, pages);
291 if (!rc) {
292 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
293 sg_table->data_pages.nr_pages,
294 VM_MAP,
295 PAGE_KERNEL);
296 if (!sg_table->data_vaddr)
297 rc = -ENOMEM;
298 }
299 return rc;
300}
301
302
303
304
305
306
307
308
309
310
311
312
313struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
314 int node,
315 int nr_tpages,
316 int nr_dpages,
317 void **pages)
318{
319 long rc;
320 struct tmc_sg_table *sg_table;
321
322 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
323 if (!sg_table)
324 return ERR_PTR(-ENOMEM);
325 sg_table->data_pages.nr_pages = nr_dpages;
326 sg_table->table_pages.nr_pages = nr_tpages;
327 sg_table->node = node;
328 sg_table->dev = dev;
329
330 rc = tmc_alloc_data_pages(sg_table, pages);
331 if (!rc)
332 rc = tmc_alloc_table_pages(sg_table);
333 if (rc) {
334 tmc_free_sg_table(sg_table);
335 kfree(sg_table);
336 return ERR_PTR(rc);
337 }
338
339 return sg_table;
340}
341
342
343
344
345
346void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
347 u64 offset, u64 size)
348{
349 int i, index, start;
350 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
351 struct device *dev = table->dev;
352 struct tmc_pages *data = &table->data_pages;
353
354 start = offset >> PAGE_SHIFT;
355 for (i = start; i < (start + npages); i++) {
356 index = i % data->nr_pages;
357 dma_sync_single_for_cpu(dev, data->daddrs[index],
358 PAGE_SIZE, DMA_FROM_DEVICE);
359 }
360}
361
362
363void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
364{
365 int i;
366 struct device *dev = sg_table->dev;
367 struct tmc_pages *table_pages = &sg_table->table_pages;
368
369 for (i = 0; i < table_pages->nr_pages; i++)
370 dma_sync_single_for_device(dev, table_pages->daddrs[i],
371 PAGE_SIZE, DMA_TO_DEVICE);
372}
373
374
375
376
377
378
379
380
381
382ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
383 u64 offset, size_t len, char **bufpp)
384{
385 size_t size;
386 int pg_idx = offset >> PAGE_SHIFT;
387 int pg_offset = offset & (PAGE_SIZE - 1);
388 struct tmc_pages *data_pages = &sg_table->data_pages;
389
390 size = tmc_sg_table_buf_size(sg_table);
391 if (offset >= size)
392 return -EINVAL;
393
394
395 len = (len < (size - offset)) ? len : size - offset;
396
397 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
398 if (len > 0)
399 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
400 return len;
401}
402
403#ifdef ETR_SG_DEBUG
404
405static unsigned long
406tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
407 dma_addr_t addr, bool table)
408{
409 long offset;
410 unsigned long base;
411 struct tmc_pages *tmc_pages;
412
413 if (table) {
414 tmc_pages = &sg_table->table_pages;
415 base = (unsigned long)sg_table->table_vaddr;
416 } else {
417 tmc_pages = &sg_table->data_pages;
418 base = (unsigned long)sg_table->data_vaddr;
419 }
420
421 offset = tmc_pages_get_offset(tmc_pages, addr);
422 if (offset < 0)
423 return 0;
424 return base + offset;
425}
426
427
428static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
429{
430 sgte_t *ptr;
431 int i = 0;
432 dma_addr_t addr;
433 struct tmc_sg_table *sg_table = etr_table->sg_table;
434
435 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
436 etr_table->hwaddr, true);
437 while (ptr) {
438 addr = ETR_SG_ADDR(*ptr);
439 switch (ETR_SG_ET(*ptr)) {
440 case ETR_SG_ET_NORMAL:
441 dev_dbg(sg_table->dev,
442 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
443 ptr++;
444 break;
445 case ETR_SG_ET_LINK:
446 dev_dbg(sg_table->dev,
447 "%05d: *** %p\t:{L} 0x%llx ***\n",
448 i, ptr, addr);
449 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
450 addr, true);
451 break;
452 case ETR_SG_ET_LAST:
453 dev_dbg(sg_table->dev,
454 "%05d: ### %p\t:[L] 0x%llx ###\n",
455 i, ptr, addr);
456 return;
457 default:
458 dev_dbg(sg_table->dev,
459 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
460 i, ptr, addr);
461 return;
462 }
463 i++;
464 }
465 dev_dbg(sg_table->dev, "******* End of Table *****\n");
466}
467#else
468static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
469#endif
470
471
472
473
474
475
476
477#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
478static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
479{
480 dma_addr_t paddr;
481 int i, type, nr_entries;
482 int tpidx = 0;
483 int sgtidx = 0;
484 int sgtentry = 0;
485 int dpidx = 0;
486 int spidx = 0;
487 sgte_t *ptr;
488 struct tmc_sg_table *sg_table = etr_table->sg_table;
489 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
490 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
491
492 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
493
494
495
496 ptr = sg_table->table_vaddr;
497
498
499
500
501 for (i = 0; i < nr_entries - 1; i++) {
502 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
503
504
505
506
507
508
509
510
511 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
512 paddr = table_daddrs[tpidx + 1];
513 } else {
514 paddr = table_daddrs[tpidx] +
515 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
516 }
517 type = ETR_SG_ET_LINK;
518 } else {
519
520
521
522
523 type = ETR_SG_ET_NORMAL;
524 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
525 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
526 dpidx++;
527 }
528 *ptr++ = ETR_SG_ENTRY(paddr, type);
529
530
531
532
533 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
534 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
535 tpidx++;
536 }
537 }
538
539
540 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
541 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
542}
543
544
545
546
547
548
549
550
551
552
553static struct etr_sg_table *
554tmc_init_etr_sg_table(struct device *dev, int node,
555 unsigned long size, void **pages)
556{
557 int nr_entries, nr_tpages;
558 int nr_dpages = size >> PAGE_SHIFT;
559 struct tmc_sg_table *sg_table;
560 struct etr_sg_table *etr_table;
561
562 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
563 if (!etr_table)
564 return ERR_PTR(-ENOMEM);
565 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
566 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
567
568 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
569 if (IS_ERR(sg_table)) {
570 kfree(etr_table);
571 return ERR_CAST(sg_table);
572 }
573
574 etr_table->sg_table = sg_table;
575
576 etr_table->hwaddr = sg_table->table_daddr;
577 tmc_etr_sg_table_populate(etr_table);
578
579 tmc_sg_table_sync_table(sg_table);
580 tmc_etr_sg_table_dump(etr_table);
581
582 return etr_table;
583}
584
585
586
587
588static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
589 struct etr_buf *etr_buf, int node,
590 void **pages)
591{
592 struct etr_flat_buf *flat_buf;
593
594
595 if (pages)
596 return -EINVAL;
597
598 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
599 if (!flat_buf)
600 return -ENOMEM;
601
602 flat_buf->vaddr = dma_alloc_coherent(drvdata->dev, etr_buf->size,
603 &flat_buf->daddr, GFP_KERNEL);
604 if (!flat_buf->vaddr) {
605 kfree(flat_buf);
606 return -ENOMEM;
607 }
608
609 flat_buf->size = etr_buf->size;
610 flat_buf->dev = drvdata->dev;
611 etr_buf->hwaddr = flat_buf->daddr;
612 etr_buf->mode = ETR_MODE_FLAT;
613 etr_buf->private = flat_buf;
614 return 0;
615}
616
617static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
618{
619 struct etr_flat_buf *flat_buf = etr_buf->private;
620
621 if (flat_buf && flat_buf->daddr)
622 dma_free_coherent(flat_buf->dev, flat_buf->size,
623 flat_buf->vaddr, flat_buf->daddr);
624 kfree(flat_buf);
625}
626
627static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
628{
629
630
631
632
633 etr_buf->offset = rrp - etr_buf->hwaddr;
634 if (etr_buf->full)
635 etr_buf->len = etr_buf->size;
636 else
637 etr_buf->len = rwp - rrp;
638}
639
640static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
641 u64 offset, size_t len, char **bufpp)
642{
643 struct etr_flat_buf *flat_buf = etr_buf->private;
644
645 *bufpp = (char *)flat_buf->vaddr + offset;
646
647
648
649
650 return len;
651}
652
653static const struct etr_buf_operations etr_flat_buf_ops = {
654 .alloc = tmc_etr_alloc_flat_buf,
655 .free = tmc_etr_free_flat_buf,
656 .sync = tmc_etr_sync_flat_buf,
657 .get_data = tmc_etr_get_data_flat_buf,
658};
659
660
661
662
663
664static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
665 struct etr_buf *etr_buf, int node,
666 void **pages)
667{
668 struct etr_sg_table *etr_table;
669
670 etr_table = tmc_init_etr_sg_table(drvdata->dev, node,
671 etr_buf->size, pages);
672 if (IS_ERR(etr_table))
673 return -ENOMEM;
674 etr_buf->hwaddr = etr_table->hwaddr;
675 etr_buf->mode = ETR_MODE_ETR_SG;
676 etr_buf->private = etr_table;
677 return 0;
678}
679
680static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
681{
682 struct etr_sg_table *etr_table = etr_buf->private;
683
684 if (etr_table) {
685 tmc_free_sg_table(etr_table->sg_table);
686 kfree(etr_table);
687 }
688}
689
690static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
691 size_t len, char **bufpp)
692{
693 struct etr_sg_table *etr_table = etr_buf->private;
694
695 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
696}
697
698static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
699{
700 long r_offset, w_offset;
701 struct etr_sg_table *etr_table = etr_buf->private;
702 struct tmc_sg_table *table = etr_table->sg_table;
703
704
705 r_offset = tmc_sg_get_data_page_offset(table, rrp);
706 if (r_offset < 0) {
707 dev_warn(table->dev,
708 "Unable to map RRP %llx to offset\n", rrp);
709 etr_buf->len = 0;
710 return;
711 }
712
713 w_offset = tmc_sg_get_data_page_offset(table, rwp);
714 if (w_offset < 0) {
715 dev_warn(table->dev,
716 "Unable to map RWP %llx to offset\n", rwp);
717 etr_buf->len = 0;
718 return;
719 }
720
721 etr_buf->offset = r_offset;
722 if (etr_buf->full)
723 etr_buf->len = etr_buf->size;
724 else
725 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
726 w_offset - r_offset;
727 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
728}
729
730static const struct etr_buf_operations etr_sg_buf_ops = {
731 .alloc = tmc_etr_alloc_sg_buf,
732 .free = tmc_etr_free_sg_buf,
733 .sync = tmc_etr_sync_sg_buf,
734 .get_data = tmc_etr_get_data_sg_buf,
735};
736
737
738
739
740
741
742
743
744
745struct coresight_device *
746tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
747{
748 int i;
749 struct coresight_device *tmp, *etr = drvdata->csdev;
750
751 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
752 return NULL;
753
754 for (i = 0; i < etr->nr_outport; i++) {
755 tmp = etr->conns[i].child_dev;
756 if (tmp && coresight_is_catu_device(tmp))
757 return tmp;
758 }
759
760 return NULL;
761}
762
763static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
764 struct etr_buf *etr_buf)
765{
766 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
767
768 if (catu && helper_ops(catu)->enable)
769 return helper_ops(catu)->enable(catu, etr_buf);
770 return 0;
771}
772
773static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
774{
775 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
776
777 if (catu && helper_ops(catu)->disable)
778 helper_ops(catu)->disable(catu, drvdata->etr_buf);
779}
780
781static const struct etr_buf_operations *etr_buf_ops[] = {
782 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
783 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
784 [ETR_MODE_CATU] = IS_ENABLED(CONFIG_CORESIGHT_CATU)
785 ? &etr_catu_buf_ops : NULL,
786};
787
788static inline int tmc_etr_mode_alloc_buf(int mode,
789 struct tmc_drvdata *drvdata,
790 struct etr_buf *etr_buf, int node,
791 void **pages)
792{
793 int rc = -EINVAL;
794
795 switch (mode) {
796 case ETR_MODE_FLAT:
797 case ETR_MODE_ETR_SG:
798 case ETR_MODE_CATU:
799 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
800 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
801 node, pages);
802 if (!rc)
803 etr_buf->ops = etr_buf_ops[mode];
804 return rc;
805 default:
806 return -EINVAL;
807 }
808}
809
810
811
812
813
814
815
816
817
818static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
819 ssize_t size, int flags,
820 int node, void **pages)
821{
822 int rc = -ENOMEM;
823 bool has_etr_sg, has_iommu;
824 bool has_sg, has_catu;
825 struct etr_buf *etr_buf;
826
827 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
828 has_iommu = iommu_get_domain_for_dev(drvdata->dev);
829 has_catu = !!tmc_etr_get_catu_device(drvdata);
830
831 has_sg = has_catu || has_etr_sg;
832
833 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
834 if (!etr_buf)
835 return ERR_PTR(-ENOMEM);
836
837 etr_buf->size = size;
838
839
840
841
842
843
844
845
846
847
848
849
850
851 if (!pages &&
852 (!has_sg || has_iommu || size < SZ_1M))
853 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
854 etr_buf, node, pages);
855 if (rc && has_etr_sg)
856 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
857 etr_buf, node, pages);
858 if (rc && has_catu)
859 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
860 etr_buf, node, pages);
861 if (rc) {
862 kfree(etr_buf);
863 return ERR_PTR(rc);
864 }
865
866 dev_dbg(drvdata->dev, "allocated buffer of size %ldKB in mode %d\n",
867 (unsigned long)size >> 10, etr_buf->mode);
868 return etr_buf;
869}
870
871static void tmc_free_etr_buf(struct etr_buf *etr_buf)
872{
873 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
874 etr_buf->ops->free(etr_buf);
875 kfree(etr_buf);
876}
877
878
879
880
881
882
883
884static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
885 u64 offset, size_t len, char **bufpp)
886{
887
888 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
889
890 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
891}
892
893static inline s64
894tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
895{
896 ssize_t len;
897 char *bufp;
898
899 len = tmc_etr_buf_get_data(etr_buf, offset,
900 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
901 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
902 return -EINVAL;
903 coresight_insert_barrier_packet(bufp);
904 return offset + CORESIGHT_BARRIER_PKT_SIZE;
905}
906
907
908
909
910
911
912
913static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
914{
915 struct etr_buf *etr_buf = drvdata->etr_buf;
916 u64 rrp, rwp;
917 u32 status;
918
919 rrp = tmc_read_rrp(drvdata);
920 rwp = tmc_read_rwp(drvdata);
921 status = readl_relaxed(drvdata->base + TMC_STS);
922 etr_buf->full = status & TMC_STS_FULL;
923
924 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
925
926 etr_buf->ops->sync(etr_buf, rrp, rwp);
927
928
929 if (etr_buf->full)
930 tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
931}
932
933static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
934{
935 u32 axictl, sts;
936 struct etr_buf *etr_buf = drvdata->etr_buf;
937
938 CS_UNLOCK(drvdata->base);
939
940
941 tmc_wait_for_tmcready(drvdata);
942
943 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
944 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
945
946 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
947 axictl &= ~TMC_AXICTL_CLEAR_MASK;
948 axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
949 axictl |= TMC_AXICTL_AXCACHE_OS;
950
951 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
952 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
953 axictl |= TMC_AXICTL_ARCACHE_OS;
954 }
955
956 if (etr_buf->mode == ETR_MODE_ETR_SG)
957 axictl |= TMC_AXICTL_SCT_GAT_MODE;
958
959 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
960 tmc_write_dba(drvdata, etr_buf->hwaddr);
961
962
963
964
965
966 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
967 tmc_write_rrp(drvdata, etr_buf->hwaddr);
968 tmc_write_rwp(drvdata, etr_buf->hwaddr);
969 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
970 writel_relaxed(sts, drvdata->base + TMC_STS);
971 }
972
973 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
974 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
975 TMC_FFCR_TRIGON_TRIGIN,
976 drvdata->base + TMC_FFCR);
977 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
978 tmc_enable_hw(drvdata);
979
980 CS_LOCK(drvdata->base);
981}
982
983static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
984 struct etr_buf *etr_buf)
985{
986 int rc;
987
988
989 if (WARN_ON(!etr_buf))
990 return -EINVAL;
991
992 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
993 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
994 return -EINVAL;
995
996 if (WARN_ON(drvdata->etr_buf))
997 return -EBUSY;
998
999
1000
1001
1002
1003 rc = tmc_etr_enable_catu(drvdata, etr_buf);
1004 if (rc)
1005 return rc;
1006 rc = coresight_claim_device(drvdata->base);
1007 if (!rc) {
1008 drvdata->etr_buf = etr_buf;
1009 __tmc_etr_enable_hw(drvdata);
1010 }
1011
1012 return rc;
1013}
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1026 loff_t pos, size_t len, char **bufpp)
1027{
1028 s64 offset;
1029 ssize_t actual = len;
1030 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1031
1032 if (pos + actual > etr_buf->len)
1033 actual = etr_buf->len - pos;
1034 if (actual <= 0)
1035 return actual;
1036
1037
1038 offset = etr_buf->offset + pos;
1039 if (offset >= etr_buf->size)
1040 offset -= etr_buf->size;
1041 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1042}
1043
1044static struct etr_buf *
1045tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1046{
1047 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1048 0, cpu_to_node(0), NULL);
1049}
1050
1051static void
1052tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1053{
1054 if (buf)
1055 tmc_free_etr_buf(buf);
1056}
1057
1058static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1059{
1060 struct etr_buf *etr_buf = drvdata->etr_buf;
1061
1062 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1063 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1064 drvdata->sysfs_buf = NULL;
1065 } else {
1066 tmc_sync_etr_buf(drvdata);
1067 }
1068}
1069
1070static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1071{
1072 CS_UNLOCK(drvdata->base);
1073
1074 tmc_flush_and_stop(drvdata);
1075
1076
1077
1078
1079 if (drvdata->mode == CS_MODE_SYSFS)
1080 tmc_etr_sync_sysfs_buf(drvdata);
1081
1082 tmc_disable_hw(drvdata);
1083
1084 CS_LOCK(drvdata->base);
1085
1086}
1087
1088static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1089{
1090 __tmc_etr_disable_hw(drvdata);
1091
1092 tmc_etr_disable_catu(drvdata);
1093 coresight_disclaim_device(drvdata->base);
1094
1095 drvdata->etr_buf = NULL;
1096}
1097
1098static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1099{
1100 int ret = 0;
1101 unsigned long flags;
1102 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1103 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113 spin_lock_irqsave(&drvdata->spinlock, flags);
1114 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1115 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1116 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1117
1118
1119 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1120 if (IS_ERR(new_buf))
1121 return PTR_ERR(new_buf);
1122
1123
1124 spin_lock_irqsave(&drvdata->spinlock, flags);
1125 }
1126
1127 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1128 ret = -EBUSY;
1129 goto out;
1130 }
1131
1132
1133
1134
1135
1136
1137 if (drvdata->mode == CS_MODE_SYSFS) {
1138 atomic_inc(csdev->refcnt);
1139 goto out;
1140 }
1141
1142
1143
1144
1145
1146 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1147 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1148 free_buf = sysfs_buf;
1149 drvdata->sysfs_buf = new_buf;
1150 }
1151
1152 ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1153 if (!ret) {
1154 drvdata->mode = CS_MODE_SYSFS;
1155 atomic_inc(csdev->refcnt);
1156 }
1157out:
1158 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1159
1160
1161 if (free_buf)
1162 tmc_etr_free_sysfs_buf(free_buf);
1163
1164 if (!ret)
1165 dev_dbg(drvdata->dev, "TMC-ETR enabled\n");
1166
1167 return ret;
1168}
1169
1170
1171
1172
1173
1174
1175
1176
1177static struct etr_buf *
1178alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1179 int nr_pages, void **pages, bool snapshot)
1180{
1181 int node, cpu = event->cpu;
1182 struct etr_buf *etr_buf;
1183 unsigned long size;
1184
1185 if (cpu == -1)
1186 cpu = smp_processor_id();
1187 node = cpu_to_node(cpu);
1188
1189
1190
1191
1192
1193 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1194 etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1195 0, node, NULL);
1196 if (!IS_ERR(etr_buf))
1197 goto done;
1198 }
1199
1200
1201
1202
1203
1204 size = drvdata->size;
1205 do {
1206 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1207 if (!IS_ERR(etr_buf))
1208 goto done;
1209 size /= 2;
1210 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1211
1212 return ERR_PTR(-ENOMEM);
1213
1214done:
1215 return etr_buf;
1216}
1217
1218static struct etr_buf *
1219get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1220 struct perf_event *event, int nr_pages,
1221 void **pages, bool snapshot)
1222{
1223 int ret;
1224 pid_t pid = task_pid_nr(event->owner);
1225 struct etr_buf *etr_buf;
1226
1227retry:
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246 mutex_lock(&drvdata->idr_mutex);
1247 etr_buf = idr_find(&drvdata->idr, pid);
1248 if (etr_buf) {
1249 refcount_inc(&etr_buf->refcount);
1250 mutex_unlock(&drvdata->idr_mutex);
1251 return etr_buf;
1252 }
1253
1254
1255 mutex_unlock(&drvdata->idr_mutex);
1256
1257 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1258 if (IS_ERR(etr_buf))
1259 return etr_buf;
1260
1261 refcount_set(&etr_buf->refcount, 1);
1262
1263
1264 mutex_lock(&drvdata->idr_mutex);
1265 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1266 mutex_unlock(&drvdata->idr_mutex);
1267
1268
1269 if (ret == -ENOSPC) {
1270 tmc_free_etr_buf(etr_buf);
1271 goto retry;
1272 }
1273
1274
1275 if (ret == -ENOMEM) {
1276 tmc_free_etr_buf(etr_buf);
1277 return ERR_PTR(ret);
1278 }
1279
1280
1281 return etr_buf;
1282}
1283
1284static struct etr_buf *
1285get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1286 struct perf_event *event, int nr_pages,
1287 void **pages, bool snapshot)
1288{
1289 struct etr_buf *etr_buf;
1290
1291
1292
1293
1294
1295 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1296 if (IS_ERR(etr_buf))
1297 goto out;
1298
1299 refcount_set(&etr_buf->refcount, 1);
1300out:
1301 return etr_buf;
1302}
1303
1304static struct etr_buf *
1305get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1306 int nr_pages, void **pages, bool snapshot)
1307{
1308 if (event->cpu == -1)
1309 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1310 pages, snapshot);
1311
1312 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1313 pages, snapshot);
1314}
1315
1316static struct etr_perf_buffer *
1317tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1318 int nr_pages, void **pages, bool snapshot)
1319{
1320 int node, cpu = event->cpu;
1321 struct etr_buf *etr_buf;
1322 struct etr_perf_buffer *etr_perf;
1323
1324 if (cpu == -1)
1325 cpu = smp_processor_id();
1326 node = cpu_to_node(cpu);
1327
1328 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1329 if (!etr_perf)
1330 return ERR_PTR(-ENOMEM);
1331
1332 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1333 if (!IS_ERR(etr_buf))
1334 goto done;
1335
1336 kfree(etr_perf);
1337 return ERR_PTR(-ENOMEM);
1338
1339done:
1340
1341
1342
1343
1344 etr_perf->drvdata = drvdata;
1345 etr_perf->etr_buf = etr_buf;
1346
1347 return etr_perf;
1348}
1349
1350
1351static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1352 struct perf_event *event, void **pages,
1353 int nr_pages, bool snapshot)
1354{
1355 struct etr_perf_buffer *etr_perf;
1356 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1357
1358 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1359 nr_pages, pages, snapshot);
1360 if (IS_ERR(etr_perf)) {
1361 dev_dbg(drvdata->dev, "Unable to allocate ETR buffer\n");
1362 return NULL;
1363 }
1364
1365 etr_perf->pid = task_pid_nr(event->owner);
1366 etr_perf->snapshot = snapshot;
1367 etr_perf->nr_pages = nr_pages;
1368 etr_perf->pages = pages;
1369
1370 return etr_perf;
1371}
1372
1373static void tmc_free_etr_buffer(void *config)
1374{
1375 struct etr_perf_buffer *etr_perf = config;
1376 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1377 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1378
1379 if (!etr_buf)
1380 goto free_etr_perf_buffer;
1381
1382 mutex_lock(&drvdata->idr_mutex);
1383
1384 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1385 mutex_unlock(&drvdata->idr_mutex);
1386 goto free_etr_perf_buffer;
1387 }
1388
1389
1390 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1391 mutex_unlock(&drvdata->idr_mutex);
1392
1393
1394
1395
1396
1397 if (buf && WARN_ON(buf != etr_buf))
1398 goto free_etr_perf_buffer;
1399
1400 tmc_free_etr_buf(etr_perf->etr_buf);
1401
1402free_etr_perf_buffer:
1403 kfree(etr_perf);
1404}
1405
1406
1407
1408
1409
1410static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf)
1411{
1412 long bytes, to_copy;
1413 long pg_idx, pg_offset, src_offset;
1414 unsigned long head = etr_perf->head;
1415 char **dst_pages, *src_buf;
1416 struct etr_buf *etr_buf = etr_perf->etr_buf;
1417
1418 head = etr_perf->head;
1419 pg_idx = head >> PAGE_SHIFT;
1420 pg_offset = head & (PAGE_SIZE - 1);
1421 dst_pages = (char **)etr_perf->pages;
1422 src_offset = etr_buf->offset;
1423 to_copy = etr_buf->len;
1424
1425 while (to_copy > 0) {
1426
1427
1428
1429
1430
1431
1432
1433
1434 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1435 &src_buf);
1436 if (WARN_ON_ONCE(bytes <= 0))
1437 break;
1438 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1439
1440 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1441
1442 to_copy -= bytes;
1443
1444
1445 pg_offset += bytes;
1446 if (pg_offset == PAGE_SIZE) {
1447 pg_offset = 0;
1448 if (++pg_idx == etr_perf->nr_pages)
1449 pg_idx = 0;
1450 }
1451
1452
1453 src_offset += bytes;
1454 if (src_offset >= etr_buf->size)
1455 src_offset -= etr_buf->size;
1456 }
1457}
1458
1459
1460
1461
1462
1463
1464
1465static unsigned long
1466tmc_update_etr_buffer(struct coresight_device *csdev,
1467 struct perf_output_handle *handle,
1468 void *config)
1469{
1470 bool lost = false;
1471 unsigned long flags, size = 0;
1472 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1473 struct etr_perf_buffer *etr_perf = config;
1474 struct etr_buf *etr_buf = etr_perf->etr_buf;
1475
1476 spin_lock_irqsave(&drvdata->spinlock, flags);
1477
1478
1479 if (atomic_read(csdev->refcnt) != 1) {
1480 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1481 goto out;
1482 }
1483
1484 if (WARN_ON(drvdata->perf_data != etr_perf)) {
1485 lost = true;
1486 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1487 goto out;
1488 }
1489
1490 CS_UNLOCK(drvdata->base);
1491
1492 tmc_flush_and_stop(drvdata);
1493 tmc_sync_etr_buf(drvdata);
1494
1495 CS_LOCK(drvdata->base);
1496
1497 drvdata->perf_data = NULL;
1498 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1499
1500 size = etr_buf->len;
1501 tmc_etr_sync_perf_buffer(etr_perf);
1502
1503
1504
1505
1506
1507 if (etr_perf->snapshot) {
1508 handle->head += size;
1509 if (etr_buf->full)
1510 size = etr_buf->size;
1511 }
1512
1513 lost |= etr_buf->full;
1514out:
1515 if (lost)
1516 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1517 return size;
1518}
1519
1520static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1521{
1522 int rc = 0;
1523 pid_t pid;
1524 unsigned long flags;
1525 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1526 struct perf_output_handle *handle = data;
1527 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1528
1529 spin_lock_irqsave(&drvdata->spinlock, flags);
1530
1531 if (drvdata->mode == CS_MODE_SYSFS) {
1532 rc = -EBUSY;
1533 goto unlock_out;
1534 }
1535
1536 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1537 rc = -EINVAL;
1538 goto unlock_out;
1539 }
1540
1541
1542 pid = etr_perf->pid;
1543
1544
1545 if (drvdata->pid != -1 && drvdata->pid != pid) {
1546 rc = -EBUSY;
1547 goto unlock_out;
1548 }
1549
1550 etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf);
1551 drvdata->perf_data = etr_perf;
1552
1553
1554
1555
1556
1557 if (drvdata->pid == pid) {
1558 atomic_inc(csdev->refcnt);
1559 goto unlock_out;
1560 }
1561
1562 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1563 if (!rc) {
1564
1565 drvdata->pid = pid;
1566 drvdata->mode = CS_MODE_PERF;
1567 atomic_inc(csdev->refcnt);
1568 }
1569
1570unlock_out:
1571 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1572 return rc;
1573}
1574
1575static int tmc_enable_etr_sink(struct coresight_device *csdev,
1576 u32 mode, void *data)
1577{
1578 switch (mode) {
1579 case CS_MODE_SYSFS:
1580 return tmc_enable_etr_sink_sysfs(csdev);
1581 case CS_MODE_PERF:
1582 return tmc_enable_etr_sink_perf(csdev, data);
1583 }
1584
1585
1586 return -EINVAL;
1587}
1588
1589static int tmc_disable_etr_sink(struct coresight_device *csdev)
1590{
1591 unsigned long flags;
1592 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1593
1594 spin_lock_irqsave(&drvdata->spinlock, flags);
1595
1596 if (drvdata->reading) {
1597 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1598 return -EBUSY;
1599 }
1600
1601 if (atomic_dec_return(csdev->refcnt)) {
1602 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1603 return -EBUSY;
1604 }
1605
1606
1607 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1608 tmc_etr_disable_hw(drvdata);
1609
1610 drvdata->pid = -1;
1611 drvdata->mode = CS_MODE_DISABLED;
1612
1613 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1614
1615 dev_dbg(drvdata->dev, "TMC-ETR disabled\n");
1616 return 0;
1617}
1618
1619static const struct coresight_ops_sink tmc_etr_sink_ops = {
1620 .enable = tmc_enable_etr_sink,
1621 .disable = tmc_disable_etr_sink,
1622 .alloc_buffer = tmc_alloc_etr_buffer,
1623 .update_buffer = tmc_update_etr_buffer,
1624 .free_buffer = tmc_free_etr_buffer,
1625};
1626
1627const struct coresight_ops tmc_etr_cs_ops = {
1628 .sink_ops = &tmc_etr_sink_ops,
1629};
1630
1631int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1632{
1633 int ret = 0;
1634 unsigned long flags;
1635
1636
1637 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1638 return -EINVAL;
1639
1640 spin_lock_irqsave(&drvdata->spinlock, flags);
1641 if (drvdata->reading) {
1642 ret = -EBUSY;
1643 goto out;
1644 }
1645
1646
1647
1648
1649
1650
1651 if (!drvdata->sysfs_buf) {
1652 ret = -EINVAL;
1653 goto out;
1654 }
1655
1656
1657 if (drvdata->mode == CS_MODE_SYSFS)
1658 __tmc_etr_disable_hw(drvdata);
1659
1660 drvdata->reading = true;
1661out:
1662 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1663
1664 return ret;
1665}
1666
1667int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1668{
1669 unsigned long flags;
1670 struct etr_buf *sysfs_buf = NULL;
1671
1672
1673 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1674 return -EINVAL;
1675
1676 spin_lock_irqsave(&drvdata->spinlock, flags);
1677
1678
1679 if (drvdata->mode == CS_MODE_SYSFS) {
1680
1681
1682
1683
1684
1685 __tmc_etr_enable_hw(drvdata);
1686 } else {
1687
1688
1689
1690
1691 sysfs_buf = drvdata->sysfs_buf;
1692 drvdata->sysfs_buf = NULL;
1693 }
1694
1695 drvdata->reading = false;
1696 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1697
1698
1699 if (sysfs_buf)
1700 tmc_etr_free_sysfs_buf(sysfs_buf);
1701
1702 return 0;
1703}
1704