1
2
3
4
5
6
7#include <linux/atomic.h>
8#include <linux/coresight.h>
9#include <linux/dma-mapping.h>
10#include <linux/iommu.h>
11#include <linux/idr.h>
12#include <linux/mutex.h>
13#include <linux/refcount.h>
14#include <linux/slab.h>
15#include <linux/types.h>
16#include <linux/vmalloc.h>
17#include "coresight-catu.h"
18#include "coresight-etm-perf.h"
19#include "coresight-priv.h"
20#include "coresight-tmc.h"
21
22struct etr_flat_buf {
23 struct device *dev;
24 dma_addr_t daddr;
25 void *vaddr;
26 size_t size;
27};
28
29
30
31
32
33
34
35
36
37
38
39struct etr_perf_buffer {
40 struct tmc_drvdata *drvdata;
41 struct etr_buf *etr_buf;
42 pid_t pid;
43 bool snapshot;
44 unsigned long head;
45 int nr_pages;
46 void **pages;
47};
48
49
50#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
51
52
53#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77typedef u32 sgte_t;
78
79#define ETR_SG_PAGE_SHIFT 12
80#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
81#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
82#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
83#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
84
85#define ETR_SG_ET_MASK 0x3
86#define ETR_SG_ET_LAST 0x1
87#define ETR_SG_ET_NORMAL 0x2
88#define ETR_SG_ET_LINK 0x3
89
90#define ETR_SG_ADDR_SHIFT 4
91
92#define ETR_SG_ENTRY(addr, type) \
93 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
94 (type & ETR_SG_ET_MASK))
95
96#define ETR_SG_ADDR(entry) \
97 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
98#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
99
100
101
102
103
104
105
106struct etr_sg_table {
107 struct tmc_sg_table *sg_table;
108 dma_addr_t hwaddr;
109};
110
111
112
113
114
115
116
117
118
119
120
121static inline unsigned long __attribute_const__
122tmc_etr_sg_table_entries(int nr_pages)
123{
124 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
125 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
126
127
128
129
130
131 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
132 nr_sglinks--;
133 return nr_sgpages + nr_sglinks;
134}
135
136
137
138
139
140
141static long
142tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
143{
144 int i;
145 dma_addr_t page_start;
146
147 for (i = 0; i < tmc_pages->nr_pages; i++) {
148 page_start = tmc_pages->daddrs[i];
149 if (addr >= page_start && addr < (page_start + PAGE_SIZE))
150 return i * PAGE_SIZE + (addr - page_start);
151 }
152
153 return -EINVAL;
154}
155
156
157
158
159
160
161static void tmc_pages_free(struct tmc_pages *tmc_pages,
162 struct device *dev, enum dma_data_direction dir)
163{
164 int i;
165 struct device *real_dev = dev->parent;
166
167 for (i = 0; i < tmc_pages->nr_pages; i++) {
168 if (tmc_pages->daddrs && tmc_pages->daddrs[i])
169 dma_unmap_page(real_dev, tmc_pages->daddrs[i],
170 PAGE_SIZE, dir);
171 if (tmc_pages->pages && tmc_pages->pages[i])
172 __free_page(tmc_pages->pages[i]);
173 }
174
175 kfree(tmc_pages->pages);
176 kfree(tmc_pages->daddrs);
177 tmc_pages->pages = NULL;
178 tmc_pages->daddrs = NULL;
179 tmc_pages->nr_pages = 0;
180}
181
182
183
184
185
186
187
188
189
190static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
191 struct device *dev, int node,
192 enum dma_data_direction dir, void **pages)
193{
194 int i, nr_pages;
195 dma_addr_t paddr;
196 struct page *page;
197 struct device *real_dev = dev->parent;
198
199 nr_pages = tmc_pages->nr_pages;
200 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
201 GFP_KERNEL);
202 if (!tmc_pages->daddrs)
203 return -ENOMEM;
204 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
205 GFP_KERNEL);
206 if (!tmc_pages->pages) {
207 kfree(tmc_pages->daddrs);
208 tmc_pages->daddrs = NULL;
209 return -ENOMEM;
210 }
211
212 for (i = 0; i < nr_pages; i++) {
213 if (pages && pages[i]) {
214 page = virt_to_page(pages[i]);
215
216 get_page(page);
217 } else {
218 page = alloc_pages_node(node,
219 GFP_KERNEL | __GFP_ZERO, 0);
220 }
221 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
222 if (dma_mapping_error(real_dev, paddr))
223 goto err;
224 tmc_pages->daddrs[i] = paddr;
225 tmc_pages->pages[i] = page;
226 }
227 return 0;
228err:
229 tmc_pages_free(tmc_pages, dev, dir);
230 return -ENOMEM;
231}
232
233static inline long
234tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
235{
236 return tmc_pages_get_offset(&sg_table->data_pages, addr);
237}
238
239static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
240{
241 if (sg_table->table_vaddr)
242 vunmap(sg_table->table_vaddr);
243 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
244}
245
246static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
247{
248 if (sg_table->data_vaddr)
249 vunmap(sg_table->data_vaddr);
250 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
251}
252
253void tmc_free_sg_table(struct tmc_sg_table *sg_table)
254{
255 tmc_free_table_pages(sg_table);
256 tmc_free_data_pages(sg_table);
257}
258
259
260
261
262
263
264static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
265{
266 int rc;
267 struct tmc_pages *table_pages = &sg_table->table_pages;
268
269 rc = tmc_pages_alloc(table_pages, sg_table->dev,
270 dev_to_node(sg_table->dev),
271 DMA_TO_DEVICE, NULL);
272 if (rc)
273 return rc;
274 sg_table->table_vaddr = vmap(table_pages->pages,
275 table_pages->nr_pages,
276 VM_MAP,
277 PAGE_KERNEL);
278 if (!sg_table->table_vaddr)
279 rc = -ENOMEM;
280 else
281 sg_table->table_daddr = table_pages->daddrs[0];
282 return rc;
283}
284
285static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
286{
287 int rc;
288
289
290 rc = tmc_pages_alloc(&sg_table->data_pages,
291 sg_table->dev, sg_table->node,
292 DMA_FROM_DEVICE, pages);
293 if (!rc) {
294 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
295 sg_table->data_pages.nr_pages,
296 VM_MAP,
297 PAGE_KERNEL);
298 if (!sg_table->data_vaddr)
299 rc = -ENOMEM;
300 }
301 return rc;
302}
303
304
305
306
307
308
309
310
311
312
313
314
315struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
316 int node,
317 int nr_tpages,
318 int nr_dpages,
319 void **pages)
320{
321 long rc;
322 struct tmc_sg_table *sg_table;
323
324 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
325 if (!sg_table)
326 return ERR_PTR(-ENOMEM);
327 sg_table->data_pages.nr_pages = nr_dpages;
328 sg_table->table_pages.nr_pages = nr_tpages;
329 sg_table->node = node;
330 sg_table->dev = dev;
331
332 rc = tmc_alloc_data_pages(sg_table, pages);
333 if (!rc)
334 rc = tmc_alloc_table_pages(sg_table);
335 if (rc) {
336 tmc_free_sg_table(sg_table);
337 kfree(sg_table);
338 return ERR_PTR(rc);
339 }
340
341 return sg_table;
342}
343
344
345
346
347
348void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
349 u64 offset, u64 size)
350{
351 int i, index, start;
352 int npages = DIV_ROUND_UP(size, PAGE_SIZE);
353 struct device *real_dev = table->dev->parent;
354 struct tmc_pages *data = &table->data_pages;
355
356 start = offset >> PAGE_SHIFT;
357 for (i = start; i < (start + npages); i++) {
358 index = i % data->nr_pages;
359 dma_sync_single_for_cpu(real_dev, data->daddrs[index],
360 PAGE_SIZE, DMA_FROM_DEVICE);
361 }
362}
363
364
365void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
366{
367 int i;
368 struct device *real_dev = sg_table->dev->parent;
369 struct tmc_pages *table_pages = &sg_table->table_pages;
370
371 for (i = 0; i < table_pages->nr_pages; i++)
372 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
373 PAGE_SIZE, DMA_TO_DEVICE);
374}
375
376
377
378
379
380
381
382
383
384ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
385 u64 offset, size_t len, char **bufpp)
386{
387 size_t size;
388 int pg_idx = offset >> PAGE_SHIFT;
389 int pg_offset = offset & (PAGE_SIZE - 1);
390 struct tmc_pages *data_pages = &sg_table->data_pages;
391
392 size = tmc_sg_table_buf_size(sg_table);
393 if (offset >= size)
394 return -EINVAL;
395
396
397 len = (len < (size - offset)) ? len : size - offset;
398
399 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
400 if (len > 0)
401 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
402 return len;
403}
404
405#ifdef ETR_SG_DEBUG
406
407static unsigned long
408tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
409 dma_addr_t addr, bool table)
410{
411 long offset;
412 unsigned long base;
413 struct tmc_pages *tmc_pages;
414
415 if (table) {
416 tmc_pages = &sg_table->table_pages;
417 base = (unsigned long)sg_table->table_vaddr;
418 } else {
419 tmc_pages = &sg_table->data_pages;
420 base = (unsigned long)sg_table->data_vaddr;
421 }
422
423 offset = tmc_pages_get_offset(tmc_pages, addr);
424 if (offset < 0)
425 return 0;
426 return base + offset;
427}
428
429
430static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
431{
432 sgte_t *ptr;
433 int i = 0;
434 dma_addr_t addr;
435 struct tmc_sg_table *sg_table = etr_table->sg_table;
436
437 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
438 etr_table->hwaddr, true);
439 while (ptr) {
440 addr = ETR_SG_ADDR(*ptr);
441 switch (ETR_SG_ET(*ptr)) {
442 case ETR_SG_ET_NORMAL:
443 dev_dbg(sg_table->dev,
444 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
445 ptr++;
446 break;
447 case ETR_SG_ET_LINK:
448 dev_dbg(sg_table->dev,
449 "%05d: *** %p\t:{L} 0x%llx ***\n",
450 i, ptr, addr);
451 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
452 addr, true);
453 break;
454 case ETR_SG_ET_LAST:
455 dev_dbg(sg_table->dev,
456 "%05d: ### %p\t:[L] 0x%llx ###\n",
457 i, ptr, addr);
458 return;
459 default:
460 dev_dbg(sg_table->dev,
461 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
462 i, ptr, addr);
463 return;
464 }
465 i++;
466 }
467 dev_dbg(sg_table->dev, "******* End of Table *****\n");
468}
469#else
470static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
471#endif
472
473
474
475
476
477
478
479#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
480static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
481{
482 dma_addr_t paddr;
483 int i, type, nr_entries;
484 int tpidx = 0;
485 int sgtidx = 0;
486 int sgtentry = 0;
487 int dpidx = 0;
488 int spidx = 0;
489 sgte_t *ptr;
490 struct tmc_sg_table *sg_table = etr_table->sg_table;
491 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
492 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
493
494 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
495
496
497
498 ptr = sg_table->table_vaddr;
499
500
501
502
503 for (i = 0; i < nr_entries - 1; i++) {
504 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
505
506
507
508
509
510
511
512
513 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
514 paddr = table_daddrs[tpidx + 1];
515 } else {
516 paddr = table_daddrs[tpidx] +
517 (ETR_SG_PAGE_SIZE * (sgtidx + 1));
518 }
519 type = ETR_SG_ET_LINK;
520 } else {
521
522
523
524
525 type = ETR_SG_ET_NORMAL;
526 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
527 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
528 dpidx++;
529 }
530 *ptr++ = ETR_SG_ENTRY(paddr, type);
531
532
533
534
535 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
536 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
537 tpidx++;
538 }
539 }
540
541
542 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
543 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
544}
545
546
547
548
549
550
551
552
553
554
555static struct etr_sg_table *
556tmc_init_etr_sg_table(struct device *dev, int node,
557 unsigned long size, void **pages)
558{
559 int nr_entries, nr_tpages;
560 int nr_dpages = size >> PAGE_SHIFT;
561 struct tmc_sg_table *sg_table;
562 struct etr_sg_table *etr_table;
563
564 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
565 if (!etr_table)
566 return ERR_PTR(-ENOMEM);
567 nr_entries = tmc_etr_sg_table_entries(nr_dpages);
568 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
569
570 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
571 if (IS_ERR(sg_table)) {
572 kfree(etr_table);
573 return ERR_CAST(sg_table);
574 }
575
576 etr_table->sg_table = sg_table;
577
578 etr_table->hwaddr = sg_table->table_daddr;
579 tmc_etr_sg_table_populate(etr_table);
580
581 tmc_sg_table_sync_table(sg_table);
582 tmc_etr_sg_table_dump(etr_table);
583
584 return etr_table;
585}
586
587
588
589
590static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
591 struct etr_buf *etr_buf, int node,
592 void **pages)
593{
594 struct etr_flat_buf *flat_buf;
595 struct device *real_dev = drvdata->csdev->dev.parent;
596
597
598 if (pages)
599 return -EINVAL;
600
601 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
602 if (!flat_buf)
603 return -ENOMEM;
604
605 flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size,
606 &flat_buf->daddr, GFP_KERNEL);
607 if (!flat_buf->vaddr) {
608 kfree(flat_buf);
609 return -ENOMEM;
610 }
611
612 flat_buf->size = etr_buf->size;
613 flat_buf->dev = &drvdata->csdev->dev;
614 etr_buf->hwaddr = flat_buf->daddr;
615 etr_buf->mode = ETR_MODE_FLAT;
616 etr_buf->private = flat_buf;
617 return 0;
618}
619
620static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
621{
622 struct etr_flat_buf *flat_buf = etr_buf->private;
623
624 if (flat_buf && flat_buf->daddr) {
625 struct device *real_dev = flat_buf->dev->parent;
626
627 dma_free_coherent(real_dev, flat_buf->size,
628 flat_buf->vaddr, flat_buf->daddr);
629 }
630 kfree(flat_buf);
631}
632
633static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
634{
635
636
637
638
639 etr_buf->offset = rrp - etr_buf->hwaddr;
640 if (etr_buf->full)
641 etr_buf->len = etr_buf->size;
642 else
643 etr_buf->len = rwp - rrp;
644}
645
646static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
647 u64 offset, size_t len, char **bufpp)
648{
649 struct etr_flat_buf *flat_buf = etr_buf->private;
650
651 *bufpp = (char *)flat_buf->vaddr + offset;
652
653
654
655
656 return len;
657}
658
659static const struct etr_buf_operations etr_flat_buf_ops = {
660 .alloc = tmc_etr_alloc_flat_buf,
661 .free = tmc_etr_free_flat_buf,
662 .sync = tmc_etr_sync_flat_buf,
663 .get_data = tmc_etr_get_data_flat_buf,
664};
665
666
667
668
669
670static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
671 struct etr_buf *etr_buf, int node,
672 void **pages)
673{
674 struct etr_sg_table *etr_table;
675 struct device *dev = &drvdata->csdev->dev;
676
677 etr_table = tmc_init_etr_sg_table(dev, node,
678 etr_buf->size, pages);
679 if (IS_ERR(etr_table))
680 return -ENOMEM;
681 etr_buf->hwaddr = etr_table->hwaddr;
682 etr_buf->mode = ETR_MODE_ETR_SG;
683 etr_buf->private = etr_table;
684 return 0;
685}
686
687static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
688{
689 struct etr_sg_table *etr_table = etr_buf->private;
690
691 if (etr_table) {
692 tmc_free_sg_table(etr_table->sg_table);
693 kfree(etr_table);
694 }
695}
696
697static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
698 size_t len, char **bufpp)
699{
700 struct etr_sg_table *etr_table = etr_buf->private;
701
702 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
703}
704
705static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
706{
707 long r_offset, w_offset;
708 struct etr_sg_table *etr_table = etr_buf->private;
709 struct tmc_sg_table *table = etr_table->sg_table;
710
711
712 r_offset = tmc_sg_get_data_page_offset(table, rrp);
713 if (r_offset < 0) {
714 dev_warn(table->dev,
715 "Unable to map RRP %llx to offset\n", rrp);
716 etr_buf->len = 0;
717 return;
718 }
719
720 w_offset = tmc_sg_get_data_page_offset(table, rwp);
721 if (w_offset < 0) {
722 dev_warn(table->dev,
723 "Unable to map RWP %llx to offset\n", rwp);
724 etr_buf->len = 0;
725 return;
726 }
727
728 etr_buf->offset = r_offset;
729 if (etr_buf->full)
730 etr_buf->len = etr_buf->size;
731 else
732 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
733 w_offset - r_offset;
734 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
735}
736
737static const struct etr_buf_operations etr_sg_buf_ops = {
738 .alloc = tmc_etr_alloc_sg_buf,
739 .free = tmc_etr_free_sg_buf,
740 .sync = tmc_etr_sync_sg_buf,
741 .get_data = tmc_etr_get_data_sg_buf,
742};
743
744
745
746
747
748
749
750
751
752struct coresight_device *
753tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
754{
755 int i;
756 struct coresight_device *tmp, *etr = drvdata->csdev;
757
758 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
759 return NULL;
760
761 for (i = 0; i < etr->pdata->nr_outport; i++) {
762 tmp = etr->pdata->conns[i].child_dev;
763 if (tmp && coresight_is_catu_device(tmp))
764 return tmp;
765 }
766
767 return NULL;
768}
769
770static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
771 struct etr_buf *etr_buf)
772{
773 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
774
775 if (catu && helper_ops(catu)->enable)
776 return helper_ops(catu)->enable(catu, etr_buf);
777 return 0;
778}
779
780static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
781{
782 struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
783
784 if (catu && helper_ops(catu)->disable)
785 helper_ops(catu)->disable(catu, drvdata->etr_buf);
786}
787
788static const struct etr_buf_operations *etr_buf_ops[] = {
789 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
790 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
791 [ETR_MODE_CATU] = IS_ENABLED(CONFIG_CORESIGHT_CATU)
792 ? &etr_catu_buf_ops : NULL,
793};
794
795static inline int tmc_etr_mode_alloc_buf(int mode,
796 struct tmc_drvdata *drvdata,
797 struct etr_buf *etr_buf, int node,
798 void **pages)
799{
800 int rc = -EINVAL;
801
802 switch (mode) {
803 case ETR_MODE_FLAT:
804 case ETR_MODE_ETR_SG:
805 case ETR_MODE_CATU:
806 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
807 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
808 node, pages);
809 if (!rc)
810 etr_buf->ops = etr_buf_ops[mode];
811 return rc;
812 default:
813 return -EINVAL;
814 }
815}
816
817
818
819
820
821
822
823
824
825static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
826 ssize_t size, int flags,
827 int node, void **pages)
828{
829 int rc = -ENOMEM;
830 bool has_etr_sg, has_iommu;
831 bool has_sg, has_catu;
832 struct etr_buf *etr_buf;
833 struct device *dev = &drvdata->csdev->dev;
834
835 has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
836 has_iommu = iommu_get_domain_for_dev(dev->parent);
837 has_catu = !!tmc_etr_get_catu_device(drvdata);
838
839 has_sg = has_catu || has_etr_sg;
840
841 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
842 if (!etr_buf)
843 return ERR_PTR(-ENOMEM);
844
845 etr_buf->size = size;
846
847
848
849
850
851
852
853
854
855
856
857
858
859 if (!pages &&
860 (!has_sg || has_iommu || size < SZ_1M))
861 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
862 etr_buf, node, pages);
863 if (rc && has_etr_sg)
864 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
865 etr_buf, node, pages);
866 if (rc && has_catu)
867 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
868 etr_buf, node, pages);
869 if (rc) {
870 kfree(etr_buf);
871 return ERR_PTR(rc);
872 }
873
874 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
875 (unsigned long)size >> 10, etr_buf->mode);
876 return etr_buf;
877}
878
879static void tmc_free_etr_buf(struct etr_buf *etr_buf)
880{
881 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
882 etr_buf->ops->free(etr_buf);
883 kfree(etr_buf);
884}
885
886
887
888
889
890
891
892static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
893 u64 offset, size_t len, char **bufpp)
894{
895
896 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
897
898 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
899}
900
901static inline s64
902tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
903{
904 ssize_t len;
905 char *bufp;
906
907 len = tmc_etr_buf_get_data(etr_buf, offset,
908 CORESIGHT_BARRIER_PKT_SIZE, &bufp);
909 if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
910 return -EINVAL;
911 coresight_insert_barrier_packet(bufp);
912 return offset + CORESIGHT_BARRIER_PKT_SIZE;
913}
914
915
916
917
918
919
920
921static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
922{
923 struct etr_buf *etr_buf = drvdata->etr_buf;
924 u64 rrp, rwp;
925 u32 status;
926
927 rrp = tmc_read_rrp(drvdata);
928 rwp = tmc_read_rwp(drvdata);
929 status = readl_relaxed(drvdata->base + TMC_STS);
930 etr_buf->full = status & TMC_STS_FULL;
931
932 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
933
934 etr_buf->ops->sync(etr_buf, rrp, rwp);
935
936
937 if (etr_buf->full)
938 tmc_etr_buf_insert_barrier_packet(etr_buf, etr_buf->offset);
939}
940
941static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
942{
943 u32 axictl, sts;
944 struct etr_buf *etr_buf = drvdata->etr_buf;
945
946 CS_UNLOCK(drvdata->base);
947
948
949 tmc_wait_for_tmcready(drvdata);
950
951 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
952 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
953
954 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
955 axictl &= ~TMC_AXICTL_CLEAR_MASK;
956 axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
957 axictl |= TMC_AXICTL_AXCACHE_OS;
958
959 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
960 axictl &= ~TMC_AXICTL_ARCACHE_MASK;
961 axictl |= TMC_AXICTL_ARCACHE_OS;
962 }
963
964 if (etr_buf->mode == ETR_MODE_ETR_SG)
965 axictl |= TMC_AXICTL_SCT_GAT_MODE;
966
967 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
968 tmc_write_dba(drvdata, etr_buf->hwaddr);
969
970
971
972
973
974 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
975 tmc_write_rrp(drvdata, etr_buf->hwaddr);
976 tmc_write_rwp(drvdata, etr_buf->hwaddr);
977 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
978 writel_relaxed(sts, drvdata->base + TMC_STS);
979 }
980
981 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
982 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
983 TMC_FFCR_TRIGON_TRIGIN,
984 drvdata->base + TMC_FFCR);
985 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
986 tmc_enable_hw(drvdata);
987
988 CS_LOCK(drvdata->base);
989}
990
991static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
992 struct etr_buf *etr_buf)
993{
994 int rc;
995
996
997 if (WARN_ON(!etr_buf))
998 return -EINVAL;
999
1000 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1001 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1002 return -EINVAL;
1003
1004 if (WARN_ON(drvdata->etr_buf))
1005 return -EBUSY;
1006
1007
1008
1009
1010
1011 rc = tmc_etr_enable_catu(drvdata, etr_buf);
1012 if (rc)
1013 return rc;
1014 rc = coresight_claim_device(drvdata->base);
1015 if (!rc) {
1016 drvdata->etr_buf = etr_buf;
1017 __tmc_etr_enable_hw(drvdata);
1018 }
1019
1020 return rc;
1021}
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1034 loff_t pos, size_t len, char **bufpp)
1035{
1036 s64 offset;
1037 ssize_t actual = len;
1038 struct etr_buf *etr_buf = drvdata->sysfs_buf;
1039
1040 if (pos + actual > etr_buf->len)
1041 actual = etr_buf->len - pos;
1042 if (actual <= 0)
1043 return actual;
1044
1045
1046 offset = etr_buf->offset + pos;
1047 if (offset >= etr_buf->size)
1048 offset -= etr_buf->size;
1049 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1050}
1051
1052static struct etr_buf *
1053tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1054{
1055 return tmc_alloc_etr_buf(drvdata, drvdata->size,
1056 0, cpu_to_node(0), NULL);
1057}
1058
1059static void
1060tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1061{
1062 if (buf)
1063 tmc_free_etr_buf(buf);
1064}
1065
1066static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1067{
1068 struct etr_buf *etr_buf = drvdata->etr_buf;
1069
1070 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1071 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1072 drvdata->sysfs_buf = NULL;
1073 } else {
1074 tmc_sync_etr_buf(drvdata);
1075 }
1076}
1077
1078static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1079{
1080 CS_UNLOCK(drvdata->base);
1081
1082 tmc_flush_and_stop(drvdata);
1083
1084
1085
1086
1087 if (drvdata->mode == CS_MODE_SYSFS)
1088 tmc_etr_sync_sysfs_buf(drvdata);
1089
1090 tmc_disable_hw(drvdata);
1091
1092 CS_LOCK(drvdata->base);
1093
1094}
1095
1096static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1097{
1098 __tmc_etr_disable_hw(drvdata);
1099
1100 tmc_etr_disable_catu(drvdata);
1101 coresight_disclaim_device(drvdata->base);
1102
1103 drvdata->etr_buf = NULL;
1104}
1105
1106static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1107{
1108 int ret = 0;
1109 unsigned long flags;
1110 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1111 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121 spin_lock_irqsave(&drvdata->spinlock, flags);
1122 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1123 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1124 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1125
1126
1127 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1128 if (IS_ERR(new_buf))
1129 return PTR_ERR(new_buf);
1130
1131
1132 spin_lock_irqsave(&drvdata->spinlock, flags);
1133 }
1134
1135 if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1136 ret = -EBUSY;
1137 goto out;
1138 }
1139
1140
1141
1142
1143
1144
1145 if (drvdata->mode == CS_MODE_SYSFS) {
1146 atomic_inc(csdev->refcnt);
1147 goto out;
1148 }
1149
1150
1151
1152
1153
1154 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1155 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1156 free_buf = sysfs_buf;
1157 drvdata->sysfs_buf = new_buf;
1158 }
1159
1160 ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1161 if (!ret) {
1162 drvdata->mode = CS_MODE_SYSFS;
1163 atomic_inc(csdev->refcnt);
1164 }
1165out:
1166 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1167
1168
1169 if (free_buf)
1170 tmc_etr_free_sysfs_buf(free_buf);
1171
1172 if (!ret)
1173 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1174
1175 return ret;
1176}
1177
1178
1179
1180
1181
1182
1183
1184
1185static struct etr_buf *
1186alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1187 int nr_pages, void **pages, bool snapshot)
1188{
1189 int node;
1190 struct etr_buf *etr_buf;
1191 unsigned long size;
1192
1193 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1194
1195
1196
1197
1198 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1199 etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1200 0, node, NULL);
1201 if (!IS_ERR(etr_buf))
1202 goto done;
1203 }
1204
1205
1206
1207
1208
1209 size = drvdata->size;
1210 do {
1211 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1212 if (!IS_ERR(etr_buf))
1213 goto done;
1214 size /= 2;
1215 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1216
1217 return ERR_PTR(-ENOMEM);
1218
1219done:
1220 return etr_buf;
1221}
1222
1223static struct etr_buf *
1224get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1225 struct perf_event *event, int nr_pages,
1226 void **pages, bool snapshot)
1227{
1228 int ret;
1229 pid_t pid = task_pid_nr(event->owner);
1230 struct etr_buf *etr_buf;
1231
1232retry:
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251 mutex_lock(&drvdata->idr_mutex);
1252 etr_buf = idr_find(&drvdata->idr, pid);
1253 if (etr_buf) {
1254 refcount_inc(&etr_buf->refcount);
1255 mutex_unlock(&drvdata->idr_mutex);
1256 return etr_buf;
1257 }
1258
1259
1260 mutex_unlock(&drvdata->idr_mutex);
1261
1262 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1263 if (IS_ERR(etr_buf))
1264 return etr_buf;
1265
1266 refcount_set(&etr_buf->refcount, 1);
1267
1268
1269 mutex_lock(&drvdata->idr_mutex);
1270 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1271 mutex_unlock(&drvdata->idr_mutex);
1272
1273
1274 if (ret == -ENOSPC) {
1275 tmc_free_etr_buf(etr_buf);
1276 goto retry;
1277 }
1278
1279
1280 if (ret == -ENOMEM) {
1281 tmc_free_etr_buf(etr_buf);
1282 return ERR_PTR(ret);
1283 }
1284
1285
1286 return etr_buf;
1287}
1288
1289static struct etr_buf *
1290get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1291 struct perf_event *event, int nr_pages,
1292 void **pages, bool snapshot)
1293{
1294 struct etr_buf *etr_buf;
1295
1296
1297
1298
1299
1300 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1301 if (IS_ERR(etr_buf))
1302 goto out;
1303
1304 refcount_set(&etr_buf->refcount, 1);
1305out:
1306 return etr_buf;
1307}
1308
1309static struct etr_buf *
1310get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1311 int nr_pages, void **pages, bool snapshot)
1312{
1313 if (event->cpu == -1)
1314 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1315 pages, snapshot);
1316
1317 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1318 pages, snapshot);
1319}
1320
1321static struct etr_perf_buffer *
1322tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1323 int nr_pages, void **pages, bool snapshot)
1324{
1325 int node;
1326 struct etr_buf *etr_buf;
1327 struct etr_perf_buffer *etr_perf;
1328
1329 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1330
1331 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1332 if (!etr_perf)
1333 return ERR_PTR(-ENOMEM);
1334
1335 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1336 if (!IS_ERR(etr_buf))
1337 goto done;
1338
1339 kfree(etr_perf);
1340 return ERR_PTR(-ENOMEM);
1341
1342done:
1343
1344
1345
1346
1347 etr_perf->drvdata = drvdata;
1348 etr_perf->etr_buf = etr_buf;
1349
1350 return etr_perf;
1351}
1352
1353
1354static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1355 struct perf_event *event, void **pages,
1356 int nr_pages, bool snapshot)
1357{
1358 struct etr_perf_buffer *etr_perf;
1359 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1360
1361 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1362 nr_pages, pages, snapshot);
1363 if (IS_ERR(etr_perf)) {
1364 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1365 return NULL;
1366 }
1367
1368 etr_perf->pid = task_pid_nr(event->owner);
1369 etr_perf->snapshot = snapshot;
1370 etr_perf->nr_pages = nr_pages;
1371 etr_perf->pages = pages;
1372
1373 return etr_perf;
1374}
1375
1376static void tmc_free_etr_buffer(void *config)
1377{
1378 struct etr_perf_buffer *etr_perf = config;
1379 struct tmc_drvdata *drvdata = etr_perf->drvdata;
1380 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1381
1382 if (!etr_buf)
1383 goto free_etr_perf_buffer;
1384
1385 mutex_lock(&drvdata->idr_mutex);
1386
1387 if (!refcount_dec_and_test(&etr_buf->refcount)) {
1388 mutex_unlock(&drvdata->idr_mutex);
1389 goto free_etr_perf_buffer;
1390 }
1391
1392
1393 buf = idr_remove(&drvdata->idr, etr_perf->pid);
1394 mutex_unlock(&drvdata->idr_mutex);
1395
1396
1397
1398
1399
1400 if (buf && WARN_ON(buf != etr_buf))
1401 goto free_etr_perf_buffer;
1402
1403 tmc_free_etr_buf(etr_perf->etr_buf);
1404
1405free_etr_perf_buffer:
1406 kfree(etr_perf);
1407}
1408
1409
1410
1411
1412
1413static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf)
1414{
1415 long bytes, to_copy;
1416 long pg_idx, pg_offset, src_offset;
1417 unsigned long head = etr_perf->head;
1418 char **dst_pages, *src_buf;
1419 struct etr_buf *etr_buf = etr_perf->etr_buf;
1420
1421 head = etr_perf->head;
1422 pg_idx = head >> PAGE_SHIFT;
1423 pg_offset = head & (PAGE_SIZE - 1);
1424 dst_pages = (char **)etr_perf->pages;
1425 src_offset = etr_buf->offset;
1426 to_copy = etr_buf->len;
1427
1428 while (to_copy > 0) {
1429
1430
1431
1432
1433
1434
1435
1436
1437 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1438 &src_buf);
1439 if (WARN_ON_ONCE(bytes <= 0))
1440 break;
1441 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1442
1443 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1444
1445 to_copy -= bytes;
1446
1447
1448 pg_offset += bytes;
1449 if (pg_offset == PAGE_SIZE) {
1450 pg_offset = 0;
1451 if (++pg_idx == etr_perf->nr_pages)
1452 pg_idx = 0;
1453 }
1454
1455
1456 src_offset += bytes;
1457 if (src_offset >= etr_buf->size)
1458 src_offset -= etr_buf->size;
1459 }
1460}
1461
1462
1463
1464
1465
1466
1467
1468static unsigned long
1469tmc_update_etr_buffer(struct coresight_device *csdev,
1470 struct perf_output_handle *handle,
1471 void *config)
1472{
1473 bool lost = false;
1474 unsigned long flags, size = 0;
1475 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1476 struct etr_perf_buffer *etr_perf = config;
1477 struct etr_buf *etr_buf = etr_perf->etr_buf;
1478
1479 spin_lock_irqsave(&drvdata->spinlock, flags);
1480
1481
1482 if (atomic_read(csdev->refcnt) != 1) {
1483 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1484 goto out;
1485 }
1486
1487 if (WARN_ON(drvdata->perf_data != etr_perf)) {
1488 lost = true;
1489 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1490 goto out;
1491 }
1492
1493 CS_UNLOCK(drvdata->base);
1494
1495 tmc_flush_and_stop(drvdata);
1496 tmc_sync_etr_buf(drvdata);
1497
1498 CS_LOCK(drvdata->base);
1499
1500 drvdata->perf_data = NULL;
1501 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1502
1503 size = etr_buf->len;
1504 tmc_etr_sync_perf_buffer(etr_perf);
1505
1506
1507
1508
1509
1510
1511
1512 if (etr_perf->snapshot)
1513 handle->head += size;
1514
1515 lost |= etr_buf->full;
1516out:
1517
1518
1519
1520
1521
1522
1523 if (!etr_perf->snapshot && lost)
1524 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1525 return size;
1526}
1527
1528static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1529{
1530 int rc = 0;
1531 pid_t pid;
1532 unsigned long flags;
1533 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1534 struct perf_output_handle *handle = data;
1535 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1536
1537 spin_lock_irqsave(&drvdata->spinlock, flags);
1538
1539 if (drvdata->mode == CS_MODE_SYSFS) {
1540 rc = -EBUSY;
1541 goto unlock_out;
1542 }
1543
1544 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1545 rc = -EINVAL;
1546 goto unlock_out;
1547 }
1548
1549
1550 pid = etr_perf->pid;
1551
1552
1553 if (drvdata->pid != -1 && drvdata->pid != pid) {
1554 rc = -EBUSY;
1555 goto unlock_out;
1556 }
1557
1558 etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf);
1559 drvdata->perf_data = etr_perf;
1560
1561
1562
1563
1564
1565 if (drvdata->pid == pid) {
1566 atomic_inc(csdev->refcnt);
1567 goto unlock_out;
1568 }
1569
1570 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1571 if (!rc) {
1572
1573 drvdata->pid = pid;
1574 drvdata->mode = CS_MODE_PERF;
1575 atomic_inc(csdev->refcnt);
1576 }
1577
1578unlock_out:
1579 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1580 return rc;
1581}
1582
1583static int tmc_enable_etr_sink(struct coresight_device *csdev,
1584 u32 mode, void *data)
1585{
1586 switch (mode) {
1587 case CS_MODE_SYSFS:
1588 return tmc_enable_etr_sink_sysfs(csdev);
1589 case CS_MODE_PERF:
1590 return tmc_enable_etr_sink_perf(csdev, data);
1591 }
1592
1593
1594 return -EINVAL;
1595}
1596
1597static int tmc_disable_etr_sink(struct coresight_device *csdev)
1598{
1599 unsigned long flags;
1600 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1601
1602 spin_lock_irqsave(&drvdata->spinlock, flags);
1603
1604 if (drvdata->reading) {
1605 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1606 return -EBUSY;
1607 }
1608
1609 if (atomic_dec_return(csdev->refcnt)) {
1610 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1611 return -EBUSY;
1612 }
1613
1614
1615 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1616 tmc_etr_disable_hw(drvdata);
1617
1618 drvdata->pid = -1;
1619 drvdata->mode = CS_MODE_DISABLED;
1620
1621 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1622
1623 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1624 return 0;
1625}
1626
1627static const struct coresight_ops_sink tmc_etr_sink_ops = {
1628 .enable = tmc_enable_etr_sink,
1629 .disable = tmc_disable_etr_sink,
1630 .alloc_buffer = tmc_alloc_etr_buffer,
1631 .update_buffer = tmc_update_etr_buffer,
1632 .free_buffer = tmc_free_etr_buffer,
1633};
1634
1635const struct coresight_ops tmc_etr_cs_ops = {
1636 .sink_ops = &tmc_etr_sink_ops,
1637};
1638
1639int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1640{
1641 int ret = 0;
1642 unsigned long flags;
1643
1644
1645 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1646 return -EINVAL;
1647
1648 spin_lock_irqsave(&drvdata->spinlock, flags);
1649 if (drvdata->reading) {
1650 ret = -EBUSY;
1651 goto out;
1652 }
1653
1654
1655
1656
1657
1658
1659 if (!drvdata->sysfs_buf) {
1660 ret = -EINVAL;
1661 goto out;
1662 }
1663
1664
1665 if (drvdata->mode == CS_MODE_SYSFS)
1666 __tmc_etr_disable_hw(drvdata);
1667
1668 drvdata->reading = true;
1669out:
1670 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1671
1672 return ret;
1673}
1674
1675int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1676{
1677 unsigned long flags;
1678 struct etr_buf *sysfs_buf = NULL;
1679
1680
1681 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1682 return -EINVAL;
1683
1684 spin_lock_irqsave(&drvdata->spinlock, flags);
1685
1686
1687 if (drvdata->mode == CS_MODE_SYSFS) {
1688
1689
1690
1691
1692
1693 __tmc_etr_enable_hw(drvdata);
1694 } else {
1695
1696
1697
1698
1699 sysfs_buf = drvdata->sysfs_buf;
1700 drvdata->sysfs_buf = NULL;
1701 }
1702
1703 drvdata->reading = false;
1704 spin_unlock_irqrestore(&drvdata->spinlock, flags);
1705
1706
1707 if (sysfs_buf)
1708 tmc_etr_free_sysfs_buf(sysfs_buf);
1709
1710 return 0;
1711}
1712