1
2
3
4
5
6
7
8#include <linux/dma-mapping.h>
9#include <linux/io.h>
10#include <linux/init.h>
11#include <linux/of.h>
12#include <linux/platform_device.h>
13#include <linux/sys_soc.h>
14#include <linux/soc/ti/k3-ringacc.h>
15#include <linux/soc/ti/ti_sci_protocol.h>
16#include <linux/soc/ti/ti_sci_inta_msi.h>
17#include <linux/of_irq.h>
18#include <linux/irqdomain.h>
19
20static LIST_HEAD(k3_ringacc_list);
21static DEFINE_MUTEX(k3_ringacc_list_lock);
22
23#define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
24
25
26
27
28
29
30
31
32
33
34
35
36struct k3_ring_rt_regs {
37 u32 resv_16[4];
38 u32 db;
39 u32 resv_4[1];
40 u32 occ;
41 u32 indx;
42 u32 hwocc;
43 u32 hwindx;
44};
45
46#define K3_RINGACC_RT_REGS_STEP 0x1000
47
48
49
50
51
52
53
54
55
56struct k3_ring_fifo_regs {
57 u32 head_data[128];
58 u32 tail_data[128];
59 u32 peek_head_data[128];
60 u32 peek_tail_data[128];
61};
62
63
64
65
66
67
68
69struct k3_ringacc_proxy_gcfg_regs {
70 u32 revision;
71 u32 config;
72};
73
74#define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
75
76
77
78
79
80
81
82
83
84struct k3_ringacc_proxy_target_regs {
85 u32 control;
86 u32 status;
87 u8 resv_512[504];
88 u32 data[128];
89};
90
91#define K3_RINGACC_PROXY_TARGET_STEP 0x1000
92#define K3_RINGACC_PROXY_NOT_USED (-1)
93
94enum k3_ringacc_proxy_access_mode {
95 PROXY_ACCESS_MODE_HEAD = 0,
96 PROXY_ACCESS_MODE_TAIL = 1,
97 PROXY_ACCESS_MODE_PEEK_HEAD = 2,
98 PROXY_ACCESS_MODE_PEEK_TAIL = 3,
99};
100
101#define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
102#define K3_RINGACC_FIFO_REGS_STEP 0x1000
103#define K3_RINGACC_MAX_DB_RING_CNT (127U)
104
105struct k3_ring_ops {
106 int (*push_tail)(struct k3_ring *ring, void *elm);
107 int (*push_head)(struct k3_ring *ring, void *elm);
108 int (*pop_tail)(struct k3_ring *ring, void *elm);
109 int (*pop_head)(struct k3_ring *ring, void *elm);
110};
111
112
113
114
115
116
117
118
119
120struct k3_ring_state {
121 u32 free;
122 u32 occ;
123 u32 windex;
124 u32 rindex;
125};
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145struct k3_ring {
146 struct k3_ring_rt_regs __iomem *rt;
147 struct k3_ring_fifo_regs __iomem *fifos;
148 struct k3_ringacc_proxy_target_regs __iomem *proxy;
149 dma_addr_t ring_mem_dma;
150 void *ring_mem_virt;
151 struct k3_ring_ops *ops;
152 u32 size;
153 enum k3_ring_size elm_size;
154 enum k3_ring_mode mode;
155 u32 flags;
156#define K3_RING_FLAG_BUSY BIT(1)
157#define K3_RING_FLAG_SHARED BIT(2)
158 struct k3_ring_state state;
159 u32 ring_id;
160 struct k3_ringacc *parent;
161 u32 use_count;
162 int proxy_id;
163};
164
165struct k3_ringacc_ops {
166 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
167};
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189struct k3_ringacc {
190 struct device *dev;
191 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
192 void __iomem *proxy_target_base;
193 u32 num_rings;
194 unsigned long *rings_inuse;
195 struct ti_sci_resource *rm_gp_range;
196
197 bool dma_ring_reset_quirk;
198 u32 num_proxies;
199 unsigned long *proxy_inuse;
200
201 struct k3_ring *rings;
202 struct list_head list;
203 struct mutex req_lock;
204
205 const struct ti_sci_handle *tisci;
206 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
207 u32 tisci_dev_id;
208
209 const struct k3_ringacc_ops *ops;
210};
211
212
213
214
215
216
217struct k3_ringacc_soc_data {
218 unsigned dma_ring_reset_quirk:1;
219};
220
221static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
222{
223 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
224 (4 << ring->elm_size);
225}
226
227static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
228{
229 return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
230}
231
232static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
233static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
234
235static struct k3_ring_ops k3_ring_mode_ring_ops = {
236 .push_tail = k3_ringacc_ring_push_mem,
237 .pop_head = k3_ringacc_ring_pop_mem,
238};
239
240static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
241static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
242static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
243static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
244
245static struct k3_ring_ops k3_ring_mode_msg_ops = {
246 .push_tail = k3_ringacc_ring_push_io,
247 .push_head = k3_ringacc_ring_push_head_io,
248 .pop_tail = k3_ringacc_ring_pop_tail_io,
249 .pop_head = k3_ringacc_ring_pop_io,
250};
251
252static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
253static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
254static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
255static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
256
257static struct k3_ring_ops k3_ring_mode_proxy_ops = {
258 .push_tail = k3_ringacc_ring_push_tail_proxy,
259 .push_head = k3_ringacc_ring_push_head_proxy,
260 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
261 .pop_head = k3_ringacc_ring_pop_head_proxy,
262};
263
264static void k3_ringacc_ring_dump(struct k3_ring *ring)
265{
266 struct device *dev = ring->parent->dev;
267
268 dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
269 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
270 &ring->ring_mem_dma);
271 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
272 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
273 dev_dbg(dev, "dump flags %08X\n", ring->flags);
274
275 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
276 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
277 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
278 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
279 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
280
281 if (ring->ring_mem_virt)
282 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
283 16, 1, ring->ring_mem_virt, 16 * 8, false);
284}
285
286struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
287 int id, u32 flags)
288{
289 int proxy_id = K3_RINGACC_PROXY_NOT_USED;
290
291 mutex_lock(&ringacc->req_lock);
292
293 if (id == K3_RINGACC_RING_ID_ANY) {
294
295 struct ti_sci_resource_desc *gp_rings =
296 &ringacc->rm_gp_range->desc[0];
297 unsigned long size;
298
299 size = gp_rings->start + gp_rings->num;
300 id = find_next_zero_bit(ringacc->rings_inuse, size,
301 gp_rings->start);
302 if (id == size)
303 goto error;
304 } else if (id < 0) {
305 goto error;
306 }
307
308 if (test_bit(id, ringacc->rings_inuse) &&
309 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
310 goto error;
311 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
312 goto out;
313
314 if (flags & K3_RINGACC_RING_USE_PROXY) {
315 proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
316 ringacc->num_proxies, 0);
317 if (proxy_id == ringacc->num_proxies)
318 goto error;
319 }
320
321 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
322 set_bit(proxy_id, ringacc->proxy_inuse);
323 ringacc->rings[id].proxy_id = proxy_id;
324 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
325 proxy_id);
326 } else {
327 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
328 }
329
330 set_bit(id, ringacc->rings_inuse);
331out:
332 ringacc->rings[id].use_count++;
333 mutex_unlock(&ringacc->req_lock);
334 return &ringacc->rings[id];
335
336error:
337 mutex_unlock(&ringacc->req_lock);
338 return NULL;
339}
340EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
341
342int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
343 int fwd_id, int compl_id,
344 struct k3_ring **fwd_ring,
345 struct k3_ring **compl_ring)
346{
347 int ret = 0;
348
349 if (!fwd_ring || !compl_ring)
350 return -EINVAL;
351
352 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
353 if (!(*fwd_ring))
354 return -ENODEV;
355
356 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
357 if (!(*compl_ring)) {
358 k3_ringacc_ring_free(*fwd_ring);
359 ret = -ENODEV;
360 }
361
362 return ret;
363}
364EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
365
366static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
367{
368 struct k3_ringacc *ringacc = ring->parent;
369 int ret;
370
371 ret = ringacc->tisci_ring_ops->config(
372 ringacc->tisci,
373 TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
374 ringacc->tisci_dev_id,
375 ring->ring_id,
376 0,
377 0,
378 ring->size,
379 0,
380 0,
381 0);
382 if (ret)
383 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
384 ret, ring->ring_id);
385}
386
387void k3_ringacc_ring_reset(struct k3_ring *ring)
388{
389 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
390 return;
391
392 memset(&ring->state, 0, sizeof(ring->state));
393
394 k3_ringacc_ring_reset_sci(ring);
395}
396EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
397
398static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
399 enum k3_ring_mode mode)
400{
401 struct k3_ringacc *ringacc = ring->parent;
402 int ret;
403
404 ret = ringacc->tisci_ring_ops->config(
405 ringacc->tisci,
406 TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
407 ringacc->tisci_dev_id,
408 ring->ring_id,
409 0,
410 0,
411 0,
412 mode,
413 0,
414 0);
415 if (ret)
416 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
417 ret, ring->ring_id);
418}
419
420void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
421{
422 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
423 return;
424
425 if (!ring->parent->dma_ring_reset_quirk)
426 goto reset;
427
428 if (!occ)
429 occ = readl(&ring->rt->occ);
430
431 if (occ) {
432 u32 db_ring_cnt, db_ring_cnt_cur;
433
434 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
435 ring->ring_id, occ);
436
437 k3_ringacc_ring_reset_sci(ring);
438
439
440
441
442
443 if (ring->mode != K3_RINGACC_RING_MODE_RING)
444 k3_ringacc_ring_reconfig_qmode_sci(
445 ring, K3_RINGACC_RING_MODE_RING);
446
447
448
449
450
451 db_ring_cnt = (1U << 22) - occ;
452
453 while (db_ring_cnt != 0) {
454
455
456
457
458
459 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
460 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
461 else
462 db_ring_cnt_cur = db_ring_cnt;
463
464 writel(db_ring_cnt_cur, &ring->rt->db);
465 db_ring_cnt -= db_ring_cnt_cur;
466 }
467
468
469 if (ring->mode != K3_RINGACC_RING_MODE_RING)
470 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
471 }
472
473reset:
474
475 k3_ringacc_ring_reset(ring);
476}
477EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
478
479static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
480{
481 struct k3_ringacc *ringacc = ring->parent;
482 int ret;
483
484 ret = ringacc->tisci_ring_ops->config(
485 ringacc->tisci,
486 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
487 ringacc->tisci_dev_id,
488 ring->ring_id,
489 0,
490 0,
491 0,
492 0,
493 0,
494 0);
495 if (ret)
496 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
497 ret, ring->ring_id);
498}
499
500int k3_ringacc_ring_free(struct k3_ring *ring)
501{
502 struct k3_ringacc *ringacc;
503
504 if (!ring)
505 return -EINVAL;
506
507 ringacc = ring->parent;
508
509 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
510
511 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
512 return -EINVAL;
513
514 mutex_lock(&ringacc->req_lock);
515
516 if (--ring->use_count)
517 goto out;
518
519 if (!(ring->flags & K3_RING_FLAG_BUSY))
520 goto no_init;
521
522 k3_ringacc_ring_free_sci(ring);
523
524 dma_free_coherent(ringacc->dev,
525 ring->size * (4 << ring->elm_size),
526 ring->ring_mem_virt, ring->ring_mem_dma);
527 ring->flags = 0;
528 ring->ops = NULL;
529 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
530 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
531 ring->proxy = NULL;
532 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
533 }
534
535no_init:
536 clear_bit(ring->ring_id, ringacc->rings_inuse);
537
538out:
539 mutex_unlock(&ringacc->req_lock);
540 return 0;
541}
542EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
543
544u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
545{
546 if (!ring)
547 return -EINVAL;
548
549 return ring->ring_id;
550}
551EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
552
553u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
554{
555 if (!ring)
556 return -EINVAL;
557
558 return ring->parent->tisci_dev_id;
559}
560EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
561
562int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
563{
564 int irq_num;
565
566 if (!ring)
567 return -EINVAL;
568
569 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id);
570 if (irq_num <= 0)
571 irq_num = -EINVAL;
572 return irq_num;
573}
574EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
575
576static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
577{
578 struct k3_ringacc *ringacc = ring->parent;
579 u32 ring_idx;
580 int ret;
581
582 if (!ringacc->tisci)
583 return -EINVAL;
584
585 ring_idx = ring->ring_id;
586 ret = ringacc->tisci_ring_ops->config(
587 ringacc->tisci,
588 TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
589 ringacc->tisci_dev_id,
590 ring_idx,
591 lower_32_bits(ring->ring_mem_dma),
592 upper_32_bits(ring->ring_mem_dma),
593 ring->size,
594 ring->mode,
595 ring->elm_size,
596 0);
597 if (ret)
598 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
599 ret, ring_idx);
600
601 return ret;
602}
603
604int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
605{
606 struct k3_ringacc *ringacc;
607 int ret = 0;
608
609 if (!ring || !cfg)
610 return -EINVAL;
611 ringacc = ring->parent;
612
613 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
614 cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
615 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
616 !test_bit(ring->ring_id, ringacc->rings_inuse))
617 return -EINVAL;
618
619 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
620 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
621 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
622 dev_err(ringacc->dev,
623 "Message mode must use proxy for %u element size\n",
624 4 << ring->elm_size);
625 return -EINVAL;
626 }
627
628
629
630
631
632
633
634
635
636 if (ring->use_count != 1)
637 return 0;
638
639 ring->size = cfg->size;
640 ring->elm_size = cfg->elm_size;
641 ring->mode = cfg->mode;
642 memset(&ring->state, 0, sizeof(ring->state));
643
644 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
645 ring->proxy = ringacc->proxy_target_base +
646 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
647
648 switch (ring->mode) {
649 case K3_RINGACC_RING_MODE_RING:
650 ring->ops = &k3_ring_mode_ring_ops;
651 break;
652 case K3_RINGACC_RING_MODE_MESSAGE:
653 if (ring->proxy)
654 ring->ops = &k3_ring_mode_proxy_ops;
655 else
656 ring->ops = &k3_ring_mode_msg_ops;
657 break;
658 default:
659 ring->ops = NULL;
660 ret = -EINVAL;
661 goto err_free_proxy;
662 }
663
664 ring->ring_mem_virt = dma_alloc_coherent(ringacc->dev,
665 ring->size * (4 << ring->elm_size),
666 &ring->ring_mem_dma, GFP_KERNEL);
667 if (!ring->ring_mem_virt) {
668 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
669 ret = -ENOMEM;
670 goto err_free_ops;
671 }
672
673 ret = k3_ringacc_ring_cfg_sci(ring);
674
675 if (ret)
676 goto err_free_mem;
677
678 ring->flags |= K3_RING_FLAG_BUSY;
679 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
680 K3_RING_FLAG_SHARED : 0;
681
682 k3_ringacc_ring_dump(ring);
683
684 return 0;
685
686err_free_mem:
687 dma_free_coherent(ringacc->dev,
688 ring->size * (4 << ring->elm_size),
689 ring->ring_mem_virt,
690 ring->ring_mem_dma);
691err_free_ops:
692 ring->ops = NULL;
693err_free_proxy:
694 ring->proxy = NULL;
695 return ret;
696}
697EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
698
699u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
700{
701 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
702 return -EINVAL;
703
704 return ring->size;
705}
706EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
707
708u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
709{
710 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
711 return -EINVAL;
712
713 if (!ring->state.free)
714 ring->state.free = ring->size - readl(&ring->rt->occ);
715
716 return ring->state.free;
717}
718EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
719
720u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
721{
722 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
723 return -EINVAL;
724
725 return readl(&ring->rt->occ);
726}
727EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
728
729u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
730{
731 return !k3_ringacc_ring_get_free(ring);
732}
733EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
734
735enum k3_ringacc_access_mode {
736 K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
737 K3_RINGACC_ACCESS_MODE_POP_HEAD,
738 K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
739 K3_RINGACC_ACCESS_MODE_POP_TAIL,
740 K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
741 K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
742};
743
744#define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16)
745#define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24)
746static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
747 enum k3_ringacc_proxy_access_mode mode)
748{
749 u32 val;
750
751 val = ring->ring_id;
752 val |= K3_RINGACC_PROXY_MODE(mode);
753 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
754 writel(val, &ring->proxy->control);
755 return 0;
756}
757
758static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
759 enum k3_ringacc_access_mode access_mode)
760{
761 void __iomem *ptr;
762
763 ptr = (void __iomem *)&ring->proxy->data;
764
765 switch (access_mode) {
766 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
767 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
768 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
769 break;
770 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
771 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
772 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
773 break;
774 default:
775 return -EINVAL;
776 }
777
778 ptr += k3_ringacc_ring_get_fifo_pos(ring);
779
780 switch (access_mode) {
781 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
782 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
783 dev_dbg(ring->parent->dev,
784 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
785 access_mode);
786 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
787 ring->state.occ--;
788 break;
789 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
790 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
791 dev_dbg(ring->parent->dev,
792 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
793 access_mode);
794 memcpy_toio(ptr, elem, (4 << ring->elm_size));
795 ring->state.free--;
796 break;
797 default:
798 return -EINVAL;
799 }
800
801 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
802 ring->state.occ);
803 return 0;
804}
805
806static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
807{
808 return k3_ringacc_ring_access_proxy(ring, elem,
809 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
810}
811
812static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
813{
814 return k3_ringacc_ring_access_proxy(ring, elem,
815 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
816}
817
818static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
819{
820 return k3_ringacc_ring_access_proxy(ring, elem,
821 K3_RINGACC_ACCESS_MODE_POP_HEAD);
822}
823
824static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
825{
826 return k3_ringacc_ring_access_proxy(ring, elem,
827 K3_RINGACC_ACCESS_MODE_POP_HEAD);
828}
829
830static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
831 enum k3_ringacc_access_mode access_mode)
832{
833 void __iomem *ptr;
834
835 switch (access_mode) {
836 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
837 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
838 ptr = (void __iomem *)&ring->fifos->head_data;
839 break;
840 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
841 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
842 ptr = (void __iomem *)&ring->fifos->tail_data;
843 break;
844 default:
845 return -EINVAL;
846 }
847
848 ptr += k3_ringacc_ring_get_fifo_pos(ring);
849
850 switch (access_mode) {
851 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
852 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
853 dev_dbg(ring->parent->dev,
854 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
855 access_mode);
856 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
857 ring->state.occ--;
858 break;
859 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
860 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
861 dev_dbg(ring->parent->dev,
862 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
863 access_mode);
864 memcpy_toio(ptr, elem, (4 << ring->elm_size));
865 ring->state.free--;
866 break;
867 default:
868 return -EINVAL;
869 }
870
871 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
872 ring->state.free, ring->state.windex, ring->state.occ,
873 ring->state.rindex);
874 return 0;
875}
876
877static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
878{
879 return k3_ringacc_ring_access_io(ring, elem,
880 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
881}
882
883static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
884{
885 return k3_ringacc_ring_access_io(ring, elem,
886 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
887}
888
889static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
890{
891 return k3_ringacc_ring_access_io(ring, elem,
892 K3_RINGACC_ACCESS_MODE_POP_HEAD);
893}
894
895static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
896{
897 return k3_ringacc_ring_access_io(ring, elem,
898 K3_RINGACC_ACCESS_MODE_POP_HEAD);
899}
900
901static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
902{
903 void *elem_ptr;
904
905 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
906
907 memcpy(elem_ptr, elem, (4 << ring->elm_size));
908
909 ring->state.windex = (ring->state.windex + 1) % ring->size;
910 ring->state.free--;
911 writel(1, &ring->rt->db);
912
913 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
914 ring->state.free, ring->state.windex);
915
916 return 0;
917}
918
919static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
920{
921 void *elem_ptr;
922
923 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
924
925 memcpy(elem, elem_ptr, (4 << ring->elm_size));
926
927 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
928 ring->state.occ--;
929 writel(-1, &ring->rt->db);
930
931 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
932 ring->state.occ, ring->state.rindex, elem_ptr);
933 return 0;
934}
935
936int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
937{
938 int ret = -EOPNOTSUPP;
939
940 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
941 return -EINVAL;
942
943 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
944 ring->state.free, ring->state.windex);
945
946 if (k3_ringacc_ring_is_full(ring))
947 return -ENOMEM;
948
949 if (ring->ops && ring->ops->push_tail)
950 ret = ring->ops->push_tail(ring, elem);
951
952 return ret;
953}
954EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
955
956int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
957{
958 int ret = -EOPNOTSUPP;
959
960 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
961 return -EINVAL;
962
963 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
964 ring->state.free, ring->state.windex);
965
966 if (k3_ringacc_ring_is_full(ring))
967 return -ENOMEM;
968
969 if (ring->ops && ring->ops->push_head)
970 ret = ring->ops->push_head(ring, elem);
971
972 return ret;
973}
974EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
975
976int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
977{
978 int ret = -EOPNOTSUPP;
979
980 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
981 return -EINVAL;
982
983 if (!ring->state.occ)
984 ring->state.occ = k3_ringacc_ring_get_occ(ring);
985
986 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
987 ring->state.rindex);
988
989 if (!ring->state.occ)
990 return -ENODATA;
991
992 if (ring->ops && ring->ops->pop_head)
993 ret = ring->ops->pop_head(ring, elem);
994
995 return ret;
996}
997EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
998
999int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1000{
1001 int ret = -EOPNOTSUPP;
1002
1003 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1004 return -EINVAL;
1005
1006 if (!ring->state.occ)
1007 ring->state.occ = k3_ringacc_ring_get_occ(ring);
1008
1009 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1010 ring->state.occ, ring->state.rindex);
1011
1012 if (!ring->state.occ)
1013 return -ENODATA;
1014
1015 if (ring->ops && ring->ops->pop_tail)
1016 ret = ring->ops->pop_tail(ring, elem);
1017
1018 return ret;
1019}
1020EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1021
1022struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1023 const char *property)
1024{
1025 struct device_node *ringacc_np;
1026 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1027 struct k3_ringacc *entry;
1028
1029 ringacc_np = of_parse_phandle(np, property, 0);
1030 if (!ringacc_np)
1031 return ERR_PTR(-ENODEV);
1032
1033 mutex_lock(&k3_ringacc_list_lock);
1034 list_for_each_entry(entry, &k3_ringacc_list, list)
1035 if (entry->dev->of_node == ringacc_np) {
1036 ringacc = entry;
1037 break;
1038 }
1039 mutex_unlock(&k3_ringacc_list_lock);
1040 of_node_put(ringacc_np);
1041
1042 return ringacc;
1043}
1044EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1045
1046static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1047{
1048 struct device_node *node = ringacc->dev->of_node;
1049 struct device *dev = ringacc->dev;
1050 struct platform_device *pdev = to_platform_device(dev);
1051 int ret;
1052
1053 if (!node) {
1054 dev_err(dev, "device tree info unavailable\n");
1055 return -ENODEV;
1056 }
1057
1058 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1059 if (ret) {
1060 dev_err(dev, "ti,num-rings read failure %d\n", ret);
1061 return ret;
1062 }
1063
1064 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1065 if (IS_ERR(ringacc->tisci)) {
1066 ret = PTR_ERR(ringacc->tisci);
1067 if (ret != -EPROBE_DEFER)
1068 dev_err(dev, "ti,sci read fail %d\n", ret);
1069 ringacc->tisci = NULL;
1070 return ret;
1071 }
1072
1073 ret = of_property_read_u32(node, "ti,sci-dev-id",
1074 &ringacc->tisci_dev_id);
1075 if (ret) {
1076 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1077 return ret;
1078 }
1079
1080 pdev->id = ringacc->tisci_dev_id;
1081
1082 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1083 ringacc->tisci_dev_id,
1084 "ti,sci-rm-range-gp-rings");
1085 if (IS_ERR(ringacc->rm_gp_range)) {
1086 dev_err(dev, "Failed to allocate MSI interrupts\n");
1087 return PTR_ERR(ringacc->rm_gp_range);
1088 }
1089
1090 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1091 ringacc->rm_gp_range);
1092}
1093
1094static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = {
1095 .dma_ring_reset_quirk = 1,
1096};
1097
1098static const struct soc_device_attribute k3_ringacc_socinfo[] = {
1099 { .family = "AM65X",
1100 .revision = "SR1.0",
1101 .data = &k3_ringacc_soc_data_sr1
1102 },
1103 {}
1104};
1105
1106static int k3_ringacc_init(struct platform_device *pdev,
1107 struct k3_ringacc *ringacc)
1108{
1109 const struct soc_device_attribute *soc;
1110 void __iomem *base_fifo, *base_rt;
1111 struct device *dev = &pdev->dev;
1112 struct resource *res;
1113 int ret, i;
1114
1115 dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
1116 DOMAIN_BUS_TI_SCI_INTA_MSI);
1117 if (!dev->msi_domain) {
1118 dev_err(dev, "Failed to get MSI domain\n");
1119 return -EPROBE_DEFER;
1120 }
1121
1122 ret = k3_ringacc_probe_dt(ringacc);
1123 if (ret)
1124 return ret;
1125
1126 soc = soc_device_match(k3_ringacc_socinfo);
1127 if (soc && soc->data) {
1128 const struct k3_ringacc_soc_data *soc_data = soc->data;
1129
1130 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;
1131 }
1132
1133 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1134 base_rt = devm_ioremap_resource(dev, res);
1135 if (IS_ERR(base_rt))
1136 return PTR_ERR(base_rt);
1137
1138 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1139 base_fifo = devm_ioremap_resource(dev, res);
1140 if (IS_ERR(base_fifo))
1141 return PTR_ERR(base_fifo);
1142
1143 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1144 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1145 if (IS_ERR(ringacc->proxy_gcfg))
1146 return PTR_ERR(ringacc->proxy_gcfg);
1147
1148 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1149 "proxy_target");
1150 ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1151 if (IS_ERR(ringacc->proxy_target_base))
1152 return PTR_ERR(ringacc->proxy_target_base);
1153
1154 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1155 K3_RINGACC_PROXY_CFG_THREADS_MASK;
1156
1157 ringacc->rings = devm_kzalloc(dev,
1158 sizeof(*ringacc->rings) *
1159 ringacc->num_rings,
1160 GFP_KERNEL);
1161 ringacc->rings_inuse = devm_kcalloc(dev,
1162 BITS_TO_LONGS(ringacc->num_rings),
1163 sizeof(unsigned long), GFP_KERNEL);
1164 ringacc->proxy_inuse = devm_kcalloc(dev,
1165 BITS_TO_LONGS(ringacc->num_proxies),
1166 sizeof(unsigned long), GFP_KERNEL);
1167
1168 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1169 return -ENOMEM;
1170
1171 for (i = 0; i < ringacc->num_rings; i++) {
1172 ringacc->rings[i].rt = base_rt +
1173 K3_RINGACC_RT_REGS_STEP * i;
1174 ringacc->rings[i].fifos = base_fifo +
1175 K3_RINGACC_FIFO_REGS_STEP * i;
1176 ringacc->rings[i].parent = ringacc;
1177 ringacc->rings[i].ring_id = i;
1178 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1179 }
1180
1181 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1182
1183 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1184 ringacc->num_rings,
1185 ringacc->rm_gp_range->desc[0].start,
1186 ringacc->rm_gp_range->desc[0].num,
1187 ringacc->tisci_dev_id);
1188 dev_info(dev, "dma-ring-reset-quirk: %s\n",
1189 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1190 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1191 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1192
1193 return 0;
1194}
1195
1196struct ringacc_match_data {
1197 struct k3_ringacc_ops ops;
1198};
1199
1200static struct ringacc_match_data k3_ringacc_data = {
1201 .ops = {
1202 .init = k3_ringacc_init,
1203 },
1204};
1205
1206
1207static const struct of_device_id k3_ringacc_of_match[] = {
1208 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
1209 {},
1210};
1211
1212static int k3_ringacc_probe(struct platform_device *pdev)
1213{
1214 const struct ringacc_match_data *match_data;
1215 const struct of_device_id *match;
1216 struct device *dev = &pdev->dev;
1217 struct k3_ringacc *ringacc;
1218 int ret;
1219
1220 match = of_match_node(k3_ringacc_of_match, dev->of_node);
1221 if (!match)
1222 return -ENODEV;
1223 match_data = match->data;
1224
1225 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1226 if (!ringacc)
1227 return -ENOMEM;
1228
1229 ringacc->dev = dev;
1230 mutex_init(&ringacc->req_lock);
1231 ringacc->ops = &match_data->ops;
1232
1233 ret = ringacc->ops->init(pdev, ringacc);
1234 if (ret)
1235 return ret;
1236
1237 dev_set_drvdata(dev, ringacc);
1238
1239 mutex_lock(&k3_ringacc_list_lock);
1240 list_add_tail(&ringacc->list, &k3_ringacc_list);
1241 mutex_unlock(&k3_ringacc_list_lock);
1242
1243 return 0;
1244}
1245
1246static struct platform_driver k3_ringacc_driver = {
1247 .probe = k3_ringacc_probe,
1248 .driver = {
1249 .name = "k3-ringacc",
1250 .of_match_table = k3_ringacc_of_match,
1251 .suppress_bind_attrs = true,
1252 },
1253};
1254builtin_platform_driver(k3_ringacc_driver);
1255
