1
2
3
4
5
6#include <common.h>
7#include <cpu_func.h>
8#include <asm/io.h>
9#include <asm/arch-tegra/ivc.h>
10#include <linux/bug.h>
11
12#define TEGRA_IVC_ALIGN 64
13
14
15
16
17
18
19enum ivc_state {
20
21
22
23
24
25
26
27
28
29
30
31 ivc_state_established = 0,
32
33
34
35
36
37
38
39 ivc_state_sync,
40
41
42
43
44
45
46
47 ivc_state_ack
48};
49
50
51
52
53
54
55
56
57struct tegra_ivc_channel_header {
58 union {
59
60 struct {
61 uint32_t w_count;
62 uint32_t state;
63 };
64 uint8_t w_align[TEGRA_IVC_ALIGN];
65 };
66 union {
67
68 uint32_t r_count;
69 uint8_t r_align[TEGRA_IVC_ALIGN];
70 };
71};
72
73static inline void tegra_ivc_invalidate_counter(struct tegra_ivc *ivc,
74 struct tegra_ivc_channel_header *h,
75 ulong offset)
76{
77 ulong base = ((ulong)h) + offset;
78 invalidate_dcache_range(base, base + TEGRA_IVC_ALIGN);
79}
80
81static inline void tegra_ivc_flush_counter(struct tegra_ivc *ivc,
82 struct tegra_ivc_channel_header *h,
83 ulong offset)
84{
85 ulong base = ((ulong)h) + offset;
86 flush_dcache_range(base, base + TEGRA_IVC_ALIGN);
87}
88
89static inline ulong tegra_ivc_frame_addr(struct tegra_ivc *ivc,
90 struct tegra_ivc_channel_header *h,
91 uint32_t frame)
92{
93 BUG_ON(frame >= ivc->nframes);
94
95 return ((ulong)h) + sizeof(struct tegra_ivc_channel_header) +
96 (ivc->frame_size * frame);
97}
98
99static inline void *tegra_ivc_frame_pointer(struct tegra_ivc *ivc,
100 struct tegra_ivc_channel_header *ch,
101 uint32_t frame)
102{
103 return (void *)tegra_ivc_frame_addr(ivc, ch, frame);
104}
105
106static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,
107 struct tegra_ivc_channel_header *h,
108 unsigned frame)
109{
110 ulong base = tegra_ivc_frame_addr(ivc, h, frame);
111 invalidate_dcache_range(base, base + ivc->frame_size);
112}
113
114static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,
115 struct tegra_ivc_channel_header *h,
116 unsigned frame)
117{
118 ulong base = tegra_ivc_frame_addr(ivc, h, frame);
119 flush_dcache_range(base, base + ivc->frame_size);
120}
121
122static inline int tegra_ivc_channel_empty(struct tegra_ivc *ivc,
123 struct tegra_ivc_channel_header *ch)
124{
125
126
127
128
129
130 uint32_t w_count = READ_ONCE(ch->w_count);
131 uint32_t r_count = READ_ONCE(ch->r_count);
132
133
134
135
136
137
138
139
140
141
142
143 if (w_count - r_count > ivc->nframes)
144 return 1;
145
146 return w_count == r_count;
147}
148
149static inline int tegra_ivc_channel_full(struct tegra_ivc *ivc,
150 struct tegra_ivc_channel_header *ch)
151{
152
153
154
155
156 return (READ_ONCE(ch->w_count) - READ_ONCE(ch->r_count)) >=
157 ivc->nframes;
158}
159
160static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)
161{
162 WRITE_ONCE(ivc->rx_channel->r_count,
163 READ_ONCE(ivc->rx_channel->r_count) + 1);
164
165 if (ivc->r_pos == ivc->nframes - 1)
166 ivc->r_pos = 0;
167 else
168 ivc->r_pos++;
169}
170
171static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)
172{
173 WRITE_ONCE(ivc->tx_channel->w_count,
174 READ_ONCE(ivc->tx_channel->w_count) + 1);
175
176 if (ivc->w_pos == ivc->nframes - 1)
177 ivc->w_pos = 0;
178 else
179 ivc->w_pos++;
180}
181
182static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)
183{
184 ulong offset;
185
186
187
188
189
190
191
192
193
194 if (ivc->tx_channel->state != ivc_state_established)
195 return -ECONNRESET;
196
197
198
199
200
201
202
203 if (!tegra_ivc_channel_empty(ivc, ivc->rx_channel))
204 return 0;
205
206 offset = offsetof(struct tegra_ivc_channel_header, w_count);
207 tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
208 return tegra_ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;
209}
210
211static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)
212{
213 ulong offset;
214
215 if (ivc->tx_channel->state != ivc_state_established)
216 return -ECONNRESET;
217
218 if (!tegra_ivc_channel_full(ivc, ivc->tx_channel))
219 return 0;
220
221 offset = offsetof(struct tegra_ivc_channel_header, r_count);
222 tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
223 return tegra_ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;
224}
225
226static inline uint32_t tegra_ivc_channel_avail_count(struct tegra_ivc *ivc,
227 struct tegra_ivc_channel_header *ch)
228{
229
230
231
232
233
234
235 return READ_ONCE(ch->w_count) - READ_ONCE(ch->r_count);
236}
237
238int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, void **frame)
239{
240 int result = tegra_ivc_check_read(ivc);
241 if (result < 0)
242 return result;
243
244
245
246
247
248 mb();
249
250 tegra_ivc_invalidate_frame(ivc, ivc->rx_channel, ivc->r_pos);
251 *frame = tegra_ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
252
253 return 0;
254}
255
256int tegra_ivc_read_advance(struct tegra_ivc *ivc)
257{
258 ulong offset;
259 int result;
260
261
262
263
264
265
266 result = tegra_ivc_check_read(ivc);
267 if (result)
268 return result;
269
270 tegra_ivc_advance_rx(ivc);
271 offset = offsetof(struct tegra_ivc_channel_header, r_count);
272 tegra_ivc_flush_counter(ivc, ivc->rx_channel, offset);
273
274
275
276
277 mb();
278
279 offset = offsetof(struct tegra_ivc_channel_header, w_count);
280 tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
281
282 if (tegra_ivc_channel_avail_count(ivc, ivc->rx_channel) ==
283 ivc->nframes - 1)
284 ivc->notify(ivc);
285
286 return 0;
287}
288
289int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, void **frame)
290{
291 int result = tegra_ivc_check_write(ivc);
292 if (result)
293 return result;
294
295 *frame = tegra_ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
296
297 return 0;
298}
299
300int tegra_ivc_write_advance(struct tegra_ivc *ivc)
301{
302 ulong offset;
303 int result;
304
305 result = tegra_ivc_check_write(ivc);
306 if (result)
307 return result;
308
309 tegra_ivc_flush_frame(ivc, ivc->tx_channel, ivc->w_pos);
310
311
312
313
314 mb();
315
316 tegra_ivc_advance_tx(ivc);
317 offset = offsetof(struct tegra_ivc_channel_header, w_count);
318 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
319
320
321
322
323 mb();
324
325 offset = offsetof(struct tegra_ivc_channel_header, r_count);
326 tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
327
328 if (tegra_ivc_channel_avail_count(ivc, ivc->tx_channel) == 1)
329 ivc->notify(ivc);
330
331 return 0;
332}
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353int tegra_ivc_channel_notified(struct tegra_ivc *ivc)
354{
355 ulong offset;
356 enum ivc_state peer_state;
357
358
359 offset = offsetof(struct tegra_ivc_channel_header, w_count);
360 tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
361 peer_state = READ_ONCE(ivc->rx_channel->state);
362
363 if (peer_state == ivc_state_sync) {
364
365
366
367
368 mb();
369
370
371
372
373
374
375 ivc->tx_channel->w_count = 0;
376 ivc->rx_channel->r_count = 0;
377
378 ivc->w_pos = 0;
379 ivc->r_pos = 0;
380
381
382
383
384
385 mb();
386
387
388
389
390
391 ivc->tx_channel->state = ivc_state_ack;
392 offset = offsetof(struct tegra_ivc_channel_header, w_count);
393 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
394
395
396
397
398 ivc->notify(ivc);
399 } else if (ivc->tx_channel->state == ivc_state_sync &&
400 peer_state == ivc_state_ack) {
401
402
403
404
405 mb();
406
407
408
409
410
411
412 ivc->tx_channel->w_count = 0;
413 ivc->rx_channel->r_count = 0;
414
415 ivc->w_pos = 0;
416 ivc->r_pos = 0;
417
418
419
420
421
422 mb();
423
424
425
426
427
428
429 ivc->tx_channel->state = ivc_state_established;
430 offset = offsetof(struct tegra_ivc_channel_header, w_count);
431 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
432
433
434
435
436 ivc->notify(ivc);
437 } else if (ivc->tx_channel->state == ivc_state_ack) {
438
439
440
441
442
443 mb();
444
445
446
447
448
449
450
451 ivc->tx_channel->state = ivc_state_established;
452 offset = offsetof(struct tegra_ivc_channel_header, w_count);
453 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
454
455
456
457
458 ivc->notify(ivc);
459 } else {
460
461
462
463
464
465
466 }
467
468 if (ivc->tx_channel->state != ivc_state_established)
469 return -EAGAIN;
470
471 return 0;
472}
473
474void tegra_ivc_channel_reset(struct tegra_ivc *ivc)
475{
476 ulong offset;
477
478 ivc->tx_channel->state = ivc_state_sync;
479 offset = offsetof(struct tegra_ivc_channel_header, w_count);
480 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
481 ivc->notify(ivc);
482}
483
484static int check_ivc_params(ulong qbase1, ulong qbase2, uint32_t nframes,
485 uint32_t frame_size)
486{
487 int ret = 0;
488
489 BUG_ON(offsetof(struct tegra_ivc_channel_header, w_count) &
490 (TEGRA_IVC_ALIGN - 1));
491 BUG_ON(offsetof(struct tegra_ivc_channel_header, r_count) &
492 (TEGRA_IVC_ALIGN - 1));
493 BUG_ON(sizeof(struct tegra_ivc_channel_header) &
494 (TEGRA_IVC_ALIGN - 1));
495
496 if ((uint64_t)nframes * (uint64_t)frame_size >= 0x100000000) {
497 pr_err("tegra_ivc: nframes * frame_size overflows\n");
498 return -EINVAL;
499 }
500
501
502
503
504
505 if ((qbase1 & (TEGRA_IVC_ALIGN - 1)) ||
506 (qbase2 & (TEGRA_IVC_ALIGN - 1))) {
507 pr_err("tegra_ivc: channel start not aligned\n");
508 return -EINVAL;
509 }
510
511 if (frame_size & (TEGRA_IVC_ALIGN - 1)) {
512 pr_err("tegra_ivc: frame size not adequately aligned\n");
513 return -EINVAL;
514 }
515
516 if (qbase1 < qbase2) {
517 if (qbase1 + frame_size * nframes > qbase2)
518 ret = -EINVAL;
519 } else {
520 if (qbase2 + frame_size * nframes > qbase1)
521 ret = -EINVAL;
522 }
523
524 if (ret) {
525 pr_err("tegra_ivc: queue regions overlap\n");
526 return ret;
527 }
528
529 return 0;
530}
531
532int tegra_ivc_init(struct tegra_ivc *ivc, ulong rx_base, ulong tx_base,
533 uint32_t nframes, uint32_t frame_size,
534 void (*notify)(struct tegra_ivc *))
535{
536 int ret;
537
538 if (!ivc)
539 return -EINVAL;
540
541 ret = check_ivc_params(rx_base, tx_base, nframes, frame_size);
542 if (ret)
543 return ret;
544
545 ivc->rx_channel = (struct tegra_ivc_channel_header *)rx_base;
546 ivc->tx_channel = (struct tegra_ivc_channel_header *)tx_base;
547 ivc->w_pos = 0;
548 ivc->r_pos = 0;
549 ivc->nframes = nframes;
550 ivc->frame_size = frame_size;
551 ivc->notify = notify;
552
553 return 0;
554}
555