1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <soc/tegra/ivc.h>
15
16#define TEGRA_IVC_ALIGN 64
17
18
19
20
21
22
23enum tegra_ivc_state {
24
25
26
27
28
29
30
31
32
33
34
35 TEGRA_IVC_STATE_ESTABLISHED = 0,
36
37
38
39
40
41
42
43 TEGRA_IVC_STATE_SYNC,
44
45
46
47
48
49
50
51 TEGRA_IVC_STATE_ACK
52};
53
54
55
56
57
58
59
60
61struct tegra_ivc_header {
62 union {
63 struct {
64
65 u32 count;
66 u32 state;
67 };
68
69 u8 pad[TEGRA_IVC_ALIGN];
70 } tx;
71
72 union {
73
74 u32 count;
75 u8 pad[TEGRA_IVC_ALIGN];
76 } rx;
77};
78
79static inline void tegra_ivc_invalidate(struct tegra_ivc *ivc, dma_addr_t phys)
80{
81 if (!ivc->peer)
82 return;
83
84 dma_sync_single_for_cpu(ivc->peer, phys, TEGRA_IVC_ALIGN,
85 DMA_FROM_DEVICE);
86}
87
88static inline void tegra_ivc_flush(struct tegra_ivc *ivc, dma_addr_t phys)
89{
90 if (!ivc->peer)
91 return;
92
93 dma_sync_single_for_device(ivc->peer, phys, TEGRA_IVC_ALIGN,
94 DMA_TO_DEVICE);
95}
96
97static inline bool tegra_ivc_empty(struct tegra_ivc *ivc,
98 struct tegra_ivc_header *header)
99{
100
101
102
103
104
105 u32 tx = READ_ONCE(header->tx.count);
106 u32 rx = READ_ONCE(header->rx.count);
107
108
109
110
111
112
113
114
115
116
117
118 if (tx - rx > ivc->num_frames)
119 return true;
120
121 return tx == rx;
122}
123
124static inline bool tegra_ivc_full(struct tegra_ivc *ivc,
125 struct tegra_ivc_header *header)
126{
127 u32 tx = READ_ONCE(header->tx.count);
128 u32 rx = READ_ONCE(header->rx.count);
129
130
131
132
133
134 return tx - rx >= ivc->num_frames;
135}
136
137static inline u32 tegra_ivc_available(struct tegra_ivc *ivc,
138 struct tegra_ivc_header *header)
139{
140 u32 tx = READ_ONCE(header->tx.count);
141 u32 rx = READ_ONCE(header->rx.count);
142
143
144
145
146
147
148
149 return tx - rx;
150}
151
152static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)
153{
154 WRITE_ONCE(ivc->tx.channel->tx.count,
155 READ_ONCE(ivc->tx.channel->tx.count) + 1);
156
157 if (ivc->tx.position == ivc->num_frames - 1)
158 ivc->tx.position = 0;
159 else
160 ivc->tx.position++;
161}
162
163static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)
164{
165 WRITE_ONCE(ivc->rx.channel->rx.count,
166 READ_ONCE(ivc->rx.channel->rx.count) + 1);
167
168 if (ivc->rx.position == ivc->num_frames - 1)
169 ivc->rx.position = 0;
170 else
171 ivc->rx.position++;
172}
173
174static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)
175{
176 unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);
177
178
179
180
181
182
183
184
185
186 if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED)
187 return -ECONNRESET;
188
189
190
191
192
193
194
195
196 if (!tegra_ivc_empty(ivc, ivc->rx.channel))
197 return 0;
198
199 tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);
200
201 if (tegra_ivc_empty(ivc, ivc->rx.channel))
202 return -ENOSPC;
203
204 return 0;
205}
206
207static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)
208{
209 unsigned int offset = offsetof(struct tegra_ivc_header, rx.count);
210
211 if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED)
212 return -ECONNRESET;
213
214 if (!tegra_ivc_full(ivc, ivc->tx.channel))
215 return 0;
216
217 tegra_ivc_invalidate(ivc, ivc->tx.phys + offset);
218
219 if (tegra_ivc_full(ivc, ivc->tx.channel))
220 return -ENOSPC;
221
222 return 0;
223}
224
225static void *tegra_ivc_frame_virt(struct tegra_ivc *ivc,
226 struct tegra_ivc_header *header,
227 unsigned int frame)
228{
229 if (WARN_ON(frame >= ivc->num_frames))
230 return ERR_PTR(-EINVAL);
231
232 return (void *)(header + 1) + ivc->frame_size * frame;
233}
234
235static inline dma_addr_t tegra_ivc_frame_phys(struct tegra_ivc *ivc,
236 dma_addr_t phys,
237 unsigned int frame)
238{
239 unsigned long offset;
240
241 offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame;
242
243 return phys + offset;
244}
245
246static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,
247 dma_addr_t phys,
248 unsigned int frame,
249 unsigned int offset,
250 size_t size)
251{
252 if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))
253 return;
254
255 phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;
256
257 dma_sync_single_for_cpu(ivc->peer, phys, size, DMA_FROM_DEVICE);
258}
259
260static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,
261 dma_addr_t phys,
262 unsigned int frame,
263 unsigned int offset,
264 size_t size)
265{
266 if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))
267 return;
268
269 phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;
270
271 dma_sync_single_for_device(ivc->peer, phys, size, DMA_TO_DEVICE);
272}
273
274
275void *tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc)
276{
277 int err;
278
279 if (WARN_ON(ivc == NULL))
280 return ERR_PTR(-EINVAL);
281
282 err = tegra_ivc_check_read(ivc);
283 if (err < 0)
284 return ERR_PTR(err);
285
286
287
288
289
290 smp_rmb();
291
292 tegra_ivc_invalidate_frame(ivc, ivc->rx.phys, ivc->rx.position, 0,
293 ivc->frame_size);
294
295 return tegra_ivc_frame_virt(ivc, ivc->rx.channel, ivc->rx.position);
296}
297EXPORT_SYMBOL(tegra_ivc_read_get_next_frame);
298
299int tegra_ivc_read_advance(struct tegra_ivc *ivc)
300{
301 unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);
302 unsigned int tx = offsetof(struct tegra_ivc_header, tx.count);
303 int err;
304
305
306
307
308
309
310 err = tegra_ivc_check_read(ivc);
311 if (err < 0)
312 return err;
313
314 tegra_ivc_advance_rx(ivc);
315
316 tegra_ivc_flush(ivc, ivc->rx.phys + rx);
317
318
319
320
321
322 smp_mb();
323
324
325
326
327
328
329 tegra_ivc_invalidate(ivc, ivc->rx.phys + tx);
330
331 if (tegra_ivc_available(ivc, ivc->rx.channel) == ivc->num_frames - 1)
332 ivc->notify(ivc, ivc->notify_data);
333
334 return 0;
335}
336EXPORT_SYMBOL(tegra_ivc_read_advance);
337
338
339void *tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc)
340{
341 int err;
342
343 err = tegra_ivc_check_write(ivc);
344 if (err < 0)
345 return ERR_PTR(err);
346
347 return tegra_ivc_frame_virt(ivc, ivc->tx.channel, ivc->tx.position);
348}
349EXPORT_SYMBOL(tegra_ivc_write_get_next_frame);
350
351
352int tegra_ivc_write_advance(struct tegra_ivc *ivc)
353{
354 unsigned int tx = offsetof(struct tegra_ivc_header, tx.count);
355 unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);
356 int err;
357
358 err = tegra_ivc_check_write(ivc);
359 if (err < 0)
360 return err;
361
362 tegra_ivc_flush_frame(ivc, ivc->tx.phys, ivc->tx.position, 0,
363 ivc->frame_size);
364
365
366
367
368
369 smp_wmb();
370
371 tegra_ivc_advance_tx(ivc);
372 tegra_ivc_flush(ivc, ivc->tx.phys + tx);
373
374
375
376
377
378 smp_mb();
379
380
381
382
383
384
385 tegra_ivc_invalidate(ivc, ivc->tx.phys + rx);
386
387 if (tegra_ivc_available(ivc, ivc->tx.channel) == 1)
388 ivc->notify(ivc, ivc->notify_data);
389
390 return 0;
391}
392EXPORT_SYMBOL(tegra_ivc_write_advance);
393
394void tegra_ivc_reset(struct tegra_ivc *ivc)
395{
396 unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);
397
398 ivc->tx.channel->tx.state = TEGRA_IVC_STATE_SYNC;
399 tegra_ivc_flush(ivc, ivc->tx.phys + offset);
400 ivc->notify(ivc, ivc->notify_data);
401}
402EXPORT_SYMBOL(tegra_ivc_reset);
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424int tegra_ivc_notified(struct tegra_ivc *ivc)
425{
426 unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);
427 enum tegra_ivc_state state;
428
429
430 tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);
431 state = READ_ONCE(ivc->rx.channel->tx.state);
432
433 if (state == TEGRA_IVC_STATE_SYNC) {
434 offset = offsetof(struct tegra_ivc_header, tx.count);
435
436
437
438
439
440 smp_rmb();
441
442
443
444
445
446
447 ivc->tx.channel->tx.count = 0;
448 ivc->rx.channel->rx.count = 0;
449
450 ivc->tx.position = 0;
451 ivc->rx.position = 0;
452
453
454
455
456
457 smp_wmb();
458
459
460
461
462
463 ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ACK;
464 tegra_ivc_flush(ivc, ivc->tx.phys + offset);
465
466
467
468
469 ivc->notify(ivc, ivc->notify_data);
470
471 } else if (ivc->tx.channel->tx.state == TEGRA_IVC_STATE_SYNC &&
472 state == TEGRA_IVC_STATE_ACK) {
473 offset = offsetof(struct tegra_ivc_header, tx.count);
474
475
476
477
478
479 smp_rmb();
480
481
482
483
484
485
486 ivc->tx.channel->tx.count = 0;
487 ivc->rx.channel->rx.count = 0;
488
489 ivc->tx.position = 0;
490 ivc->rx.position = 0;
491
492
493
494
495
496 smp_wmb();
497
498
499
500
501
502
503 ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ESTABLISHED;
504 tegra_ivc_flush(ivc, ivc->tx.phys + offset);
505
506
507
508
509 ivc->notify(ivc, ivc->notify_data);
510
511 } else if (ivc->tx.channel->tx.state == TEGRA_IVC_STATE_ACK) {
512 offset = offsetof(struct tegra_ivc_header, tx.count);
513
514
515
516
517
518
519 smp_rmb();
520
521
522
523
524
525
526
527 ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ESTABLISHED;
528 tegra_ivc_flush(ivc, ivc->tx.phys + offset);
529
530
531
532
533 ivc->notify(ivc, ivc->notify_data);
534
535 } else {
536
537
538
539
540
541
542 }
543
544 if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED)
545 return -EAGAIN;
546
547 return 0;
548}
549EXPORT_SYMBOL(tegra_ivc_notified);
550
551size_t tegra_ivc_align(size_t size)
552{
553 return ALIGN(size, TEGRA_IVC_ALIGN);
554}
555EXPORT_SYMBOL(tegra_ivc_align);
556
557unsigned tegra_ivc_total_queue_size(unsigned queue_size)
558{
559 if (!IS_ALIGNED(queue_size, TEGRA_IVC_ALIGN)) {
560 pr_err("%s: queue_size (%u) must be %u-byte aligned\n",
561 __func__, queue_size, TEGRA_IVC_ALIGN);
562 return 0;
563 }
564
565 return queue_size + sizeof(struct tegra_ivc_header);
566}
567EXPORT_SYMBOL(tegra_ivc_total_queue_size);
568
569static int tegra_ivc_check_params(unsigned long rx, unsigned long tx,
570 unsigned int num_frames, size_t frame_size)
571{
572 BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, tx.count),
573 TEGRA_IVC_ALIGN));
574 BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, rx.count),
575 TEGRA_IVC_ALIGN));
576 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct tegra_ivc_header),
577 TEGRA_IVC_ALIGN));
578
579 if ((uint64_t)num_frames * (uint64_t)frame_size >= 0x100000000UL) {
580 pr_err("num_frames * frame_size overflows\n");
581 return -EINVAL;
582 }
583
584 if (!IS_ALIGNED(frame_size, TEGRA_IVC_ALIGN)) {
585 pr_err("frame size not adequately aligned: %zu\n", frame_size);
586 return -EINVAL;
587 }
588
589
590
591
592
593 if (!IS_ALIGNED(rx, TEGRA_IVC_ALIGN)) {
594 pr_err("IVC channel start not aligned: %#lx\n", rx);
595 return -EINVAL;
596 }
597
598 if (!IS_ALIGNED(tx, TEGRA_IVC_ALIGN)) {
599 pr_err("IVC channel start not aligned: %#lx\n", tx);
600 return -EINVAL;
601 }
602
603 if (rx < tx) {
604 if (rx + frame_size * num_frames > tx) {
605 pr_err("queue regions overlap: %#lx + %zx > %#lx\n",
606 rx, frame_size * num_frames, tx);
607 return -EINVAL;
608 }
609 } else {
610 if (tx + frame_size * num_frames > rx) {
611 pr_err("queue regions overlap: %#lx + %zx > %#lx\n",
612 tx, frame_size * num_frames, rx);
613 return -EINVAL;
614 }
615 }
616
617 return 0;
618}
619
620int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, void *rx,
621 dma_addr_t rx_phys, void *tx, dma_addr_t tx_phys,
622 unsigned int num_frames, size_t frame_size,
623 void (*notify)(struct tegra_ivc *ivc, void *data),
624 void *data)
625{
626 size_t queue_size;
627 int err;
628
629 if (WARN_ON(!ivc || !notify))
630 return -EINVAL;
631
632
633
634
635
636 if (frame_size > INT_MAX)
637 return -E2BIG;
638
639 err = tegra_ivc_check_params((unsigned long)rx, (unsigned long)tx,
640 num_frames, frame_size);
641 if (err < 0)
642 return err;
643
644 queue_size = tegra_ivc_total_queue_size(num_frames * frame_size);
645
646 if (peer) {
647 ivc->rx.phys = dma_map_single(peer, rx, queue_size,
648 DMA_BIDIRECTIONAL);
649 if (dma_mapping_error(peer, ivc->rx.phys))
650 return -ENOMEM;
651
652 ivc->tx.phys = dma_map_single(peer, tx, queue_size,
653 DMA_BIDIRECTIONAL);
654 if (dma_mapping_error(peer, ivc->tx.phys)) {
655 dma_unmap_single(peer, ivc->rx.phys, queue_size,
656 DMA_BIDIRECTIONAL);
657 return -ENOMEM;
658 }
659 } else {
660 ivc->rx.phys = rx_phys;
661 ivc->tx.phys = tx_phys;
662 }
663
664 ivc->rx.channel = rx;
665 ivc->tx.channel = tx;
666 ivc->peer = peer;
667 ivc->notify = notify;
668 ivc->notify_data = data;
669 ivc->frame_size = frame_size;
670 ivc->num_frames = num_frames;
671
672
673
674
675
676 ivc->tx.position = 0;
677 ivc->rx.position = 0;
678
679 return 0;
680}
681EXPORT_SYMBOL(tegra_ivc_init);
682
683void tegra_ivc_cleanup(struct tegra_ivc *ivc)
684{
685 if (ivc->peer) {
686 size_t size = tegra_ivc_total_queue_size(ivc->num_frames *
687 ivc->frame_size);
688
689 dma_unmap_single(ivc->peer, ivc->rx.phys, size,
690 DMA_BIDIRECTIONAL);
691 dma_unmap_single(ivc->peer, ivc->tx.phys, size,
692 DMA_BIDIRECTIONAL);
693 }
694}
695EXPORT_SYMBOL(tegra_ivc_cleanup);
696