1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#ifndef VCHIQ_CORE_H
35#define VCHIQ_CORE_H
36
37#include <linux/mutex.h>
38#include <linux/semaphore.h>
39#include <linux/kthread.h>
40
41#include "vchiq_cfg.h"
42
43#include "vchiq.h"
44
45
46#define VCHIQ_LOG_DEFAULT 4
47#define VCHIQ_LOG_ERROR 3
48#define VCHIQ_LOG_WARNING 4
49#define VCHIQ_LOG_INFO 6
50#define VCHIQ_LOG_TRACE 7
51
52#define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: "
53
54#ifndef vchiq_log_error
55#define vchiq_log_error(cat, fmt, ...) \
56 do { if (cat >= VCHIQ_LOG_ERROR) \
57 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
58#endif
59#ifndef vchiq_log_warning
60#define vchiq_log_warning(cat, fmt, ...) \
61 do { if (cat >= VCHIQ_LOG_WARNING) \
62 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
63#endif
64#ifndef vchiq_log_info
65#define vchiq_log_info(cat, fmt, ...) \
66 do { if (cat >= VCHIQ_LOG_INFO) \
67 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
68#endif
69#ifndef vchiq_log_trace
70#define vchiq_log_trace(cat, fmt, ...) \
71 do { if (cat >= VCHIQ_LOG_TRACE) \
72 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
73#endif
74
75#define vchiq_loud_error(...) \
76 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
77
78#ifndef vchiq_static_assert
79#define vchiq_static_assert(cond) __attribute__((unused)) \
80 extern int vchiq_static_assert[(cond) ? 1 : -1]
81#endif
82
83#define IS_POW2(x) (x && ((x & (x - 1)) == 0))
84
85
86vchiq_static_assert(IS_POW2(VCHIQ_SLOT_SIZE));
87vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS));
88vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS_PER_SIDE));
89
90#define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1)
91#define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
92#define VCHIQ_SLOT_ZERO_SLOTS ((sizeof(VCHIQ_SLOT_ZERO_T) + \
93 VCHIQ_SLOT_SIZE - 1) / VCHIQ_SLOT_SIZE)
94
95#define VCHIQ_MSG_PADDING 0
96#define VCHIQ_MSG_CONNECT 1
97#define VCHIQ_MSG_OPEN 2
98#define VCHIQ_MSG_OPENACK 3
99#define VCHIQ_MSG_CLOSE 4
100#define VCHIQ_MSG_DATA 5
101#define VCHIQ_MSG_BULK_RX 6
102#define VCHIQ_MSG_BULK_TX 7
103#define VCHIQ_MSG_BULK_RX_DONE 8
104#define VCHIQ_MSG_BULK_TX_DONE 9
105#define VCHIQ_MSG_PAUSE 10
106#define VCHIQ_MSG_RESUME 11
107#define VCHIQ_MSG_REMOTE_USE 12
108#define VCHIQ_MSG_REMOTE_RELEASE 13
109#define VCHIQ_MSG_REMOTE_USE_ACTIVE 14
110
111#define VCHIQ_PORT_MAX (VCHIQ_MAX_SERVICES - 1)
112#define VCHIQ_PORT_FREE 0x1000
113#define VCHIQ_PORT_IS_VALID(port) (port < VCHIQ_PORT_FREE)
114#define VCHIQ_MAKE_MSG(type, srcport, dstport) \
115 ((type<<24) | (srcport<<12) | (dstport<<0))
116#define VCHIQ_MSG_TYPE(msgid) ((unsigned int)msgid >> 24)
117#define VCHIQ_MSG_SRCPORT(msgid) \
118 (unsigned short)(((unsigned int)msgid >> 12) & 0xfff)
119#define VCHIQ_MSG_DSTPORT(msgid) \
120 ((unsigned short)msgid & 0xfff)
121
122#define VCHIQ_FOURCC_AS_4CHARS(fourcc) \
123 ((fourcc) >> 24) & 0xff, \
124 ((fourcc) >> 16) & 0xff, \
125 ((fourcc) >> 8) & 0xff, \
126 (fourcc) & 0xff
127
128
129vchiq_static_assert(VCHIQ_MSG_SRCPORT(VCHIQ_MAKE_MSG(0, 0, VCHIQ_PORT_MAX))
130 == 0);
131vchiq_static_assert(VCHIQ_MSG_TYPE(VCHIQ_MAKE_MSG(0, VCHIQ_PORT_MAX, 0)) == 0);
132vchiq_static_assert((unsigned int)VCHIQ_PORT_MAX <
133 (unsigned int)VCHIQ_PORT_FREE);
134
135#define VCHIQ_MSGID_PADDING VCHIQ_MAKE_MSG(VCHIQ_MSG_PADDING, 0, 0)
136#define VCHIQ_MSGID_CLAIMED 0x40000000
137
138#define VCHIQ_FOURCC_INVALID 0x00000000
139#define VCHIQ_FOURCC_IS_LEGAL(fourcc) (fourcc != VCHIQ_FOURCC_INVALID)
140
141#define VCHIQ_BULK_ACTUAL_ABORTED -1
142
143typedef uint32_t BITSET_T;
144
145vchiq_static_assert((sizeof(BITSET_T) * 8) == 32);
146
147#define BITSET_SIZE(b) ((b + 31) >> 5)
148#define BITSET_WORD(b) (b >> 5)
149#define BITSET_BIT(b) (1 << (b & 31))
150#define BITSET_ZERO(bs) memset(bs, 0, sizeof(bs))
151#define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b))
152#define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
153#define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b))
154
155#if VCHIQ_ENABLE_STATS
156#define VCHIQ_STATS_INC(state, stat) (state->stats. stat++)
157#define VCHIQ_SERVICE_STATS_INC(service, stat) (service->stats. stat++)
158#define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) \
159 (service->stats. stat += addend)
160#else
161#define VCHIQ_STATS_INC(state, stat) ((void)0)
162#define VCHIQ_SERVICE_STATS_INC(service, stat) ((void)0)
163#define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) ((void)0)
164#endif
165
166enum {
167 DEBUG_ENTRIES,
168#if VCHIQ_ENABLE_DEBUG
169 DEBUG_SLOT_HANDLER_COUNT,
170 DEBUG_SLOT_HANDLER_LINE,
171 DEBUG_PARSE_LINE,
172 DEBUG_PARSE_HEADER,
173 DEBUG_PARSE_MSGID,
174 DEBUG_AWAIT_COMPLETION_LINE,
175 DEBUG_DEQUEUE_MESSAGE_LINE,
176 DEBUG_SERVICE_CALLBACK_LINE,
177 DEBUG_MSG_QUEUE_FULL_COUNT,
178 DEBUG_COMPLETION_QUEUE_FULL_COUNT,
179#endif
180 DEBUG_MAX
181};
182
183#if VCHIQ_ENABLE_DEBUG
184
185#define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
186#define DEBUG_TRACE(d) \
187 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(); } while (0)
188#define DEBUG_VALUE(d, v) \
189 do { debug_ptr[DEBUG_ ## d] = (v); dsb(); } while (0)
190#define DEBUG_COUNT(d) \
191 do { debug_ptr[DEBUG_ ## d]++; dsb(); } while (0)
192
193#else
194
195#define DEBUG_INITIALISE(local)
196#define DEBUG_TRACE(d)
197#define DEBUG_VALUE(d, v)
198#define DEBUG_COUNT(d)
199
200#endif
201
202typedef enum {
203 VCHIQ_CONNSTATE_DISCONNECTED,
204 VCHIQ_CONNSTATE_CONNECTING,
205 VCHIQ_CONNSTATE_CONNECTED,
206 VCHIQ_CONNSTATE_PAUSING,
207 VCHIQ_CONNSTATE_PAUSE_SENT,
208 VCHIQ_CONNSTATE_PAUSED,
209 VCHIQ_CONNSTATE_RESUMING,
210 VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
211 VCHIQ_CONNSTATE_RESUME_TIMEOUT
212} VCHIQ_CONNSTATE_T;
213
214enum {
215 VCHIQ_SRVSTATE_FREE,
216 VCHIQ_SRVSTATE_HIDDEN,
217 VCHIQ_SRVSTATE_LISTENING,
218 VCHIQ_SRVSTATE_OPENING,
219 VCHIQ_SRVSTATE_OPEN,
220 VCHIQ_SRVSTATE_OPENSYNC,
221 VCHIQ_SRVSTATE_CLOSESENT,
222 VCHIQ_SRVSTATE_CLOSERECVD,
223 VCHIQ_SRVSTATE_CLOSEWAIT,
224 VCHIQ_SRVSTATE_CLOSED
225};
226
227enum {
228 VCHIQ_POLL_TERMINATE,
229 VCHIQ_POLL_REMOVE,
230 VCHIQ_POLL_TXNOTIFY,
231 VCHIQ_POLL_RXNOTIFY,
232 VCHIQ_POLL_COUNT
233};
234
235typedef enum {
236 VCHIQ_BULK_TRANSMIT,
237 VCHIQ_BULK_RECEIVE
238} VCHIQ_BULK_DIR_T;
239
240typedef void (*VCHIQ_USERDATA_TERM_T)(void *userdata);
241
242typedef struct vchiq_bulk_struct {
243 short mode;
244 short dir;
245 void *userdata;
246 VCHI_MEM_HANDLE_T handle;
247 void *data;
248 int size;
249 void *remote_data;
250 int remote_size;
251 int actual;
252} VCHIQ_BULK_T;
253
254typedef struct vchiq_bulk_queue_struct {
255 int local_insert;
256 int remote_insert;
257 int process;
258 int remote_notify;
259 int remove;
260
261 VCHIQ_BULK_T bulks[VCHIQ_NUM_SERVICE_BULKS];
262} VCHIQ_BULK_QUEUE_T;
263
264typedef struct remote_event_struct {
265 int armed;
266 int fired;
267 struct semaphore *event;
268} REMOTE_EVENT_T;
269
270typedef struct opaque_platform_state_t *VCHIQ_PLATFORM_STATE_T;
271
272typedef struct vchiq_state_struct VCHIQ_STATE_T;
273
274typedef struct vchiq_slot_struct {
275 char data[VCHIQ_SLOT_SIZE];
276} VCHIQ_SLOT_T;
277
278typedef struct vchiq_slot_info_struct {
279
280 short use_count;
281 short release_count;
282} VCHIQ_SLOT_INFO_T;
283
284typedef struct vchiq_service_struct {
285 VCHIQ_SERVICE_BASE_T base;
286 VCHIQ_SERVICE_HANDLE_T handle;
287 unsigned int ref_count;
288 int srvstate;
289 VCHIQ_USERDATA_TERM_T userdata_term;
290 unsigned int localport;
291 unsigned int remoteport;
292 int public_fourcc;
293 int client_id;
294 char auto_close;
295 char sync;
296 char closing;
297 char trace;
298 atomic_t poll_flags;
299 short version;
300 short version_min;
301 short peer_version;
302
303 VCHIQ_STATE_T *state;
304 VCHIQ_INSTANCE_T instance;
305
306 int service_use_count;
307
308 VCHIQ_BULK_QUEUE_T bulk_tx;
309 VCHIQ_BULK_QUEUE_T bulk_rx;
310
311 struct semaphore remove_event;
312 struct semaphore bulk_remove_event;
313 struct mutex bulk_mutex;
314
315 struct service_stats_struct {
316 int quota_stalls;
317 int slot_stalls;
318 int bulk_stalls;
319 int error_count;
320 int ctrl_tx_count;
321 int ctrl_rx_count;
322 int bulk_tx_count;
323 int bulk_rx_count;
324 int bulk_aborted_count;
325 uint64_t ctrl_tx_bytes;
326 uint64_t ctrl_rx_bytes;
327 uint64_t bulk_tx_bytes;
328 uint64_t bulk_rx_bytes;
329 } stats;
330} VCHIQ_SERVICE_T;
331
332
333
334
335
336typedef struct vchiq_service_quota_struct {
337 unsigned short slot_quota;
338 unsigned short slot_use_count;
339 unsigned short message_quota;
340 unsigned short message_use_count;
341 struct semaphore quota_event;
342 int previous_tx_index;
343} VCHIQ_SERVICE_QUOTA_T;
344
345typedef struct vchiq_shared_state_struct {
346
347
348 int initialised;
349
350
351 int slot_first;
352 int slot_last;
353
354
355 int slot_sync;
356
357
358
359 REMOTE_EVENT_T trigger;
360
361
362
363
364 int tx_pos;
365
366
367 REMOTE_EVENT_T recycle;
368
369
370 int slot_queue_recycle;
371
372
373 REMOTE_EVENT_T sync_trigger;
374
375
376
377 REMOTE_EVENT_T sync_release;
378
379
380 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
381
382
383 int debug[DEBUG_MAX];
384} VCHIQ_SHARED_STATE_T;
385
386typedef struct vchiq_slot_zero_struct {
387 int magic;
388 short version;
389 short version_min;
390 int slot_zero_size;
391 int slot_size;
392 int max_slots;
393 int max_slots_per_side;
394 int platform_data[2];
395 VCHIQ_SHARED_STATE_T master;
396 VCHIQ_SHARED_STATE_T slave;
397 VCHIQ_SLOT_INFO_T slots[VCHIQ_MAX_SLOTS];
398} VCHIQ_SLOT_ZERO_T;
399
400struct vchiq_state_struct {
401 int id;
402 int initialised;
403 VCHIQ_CONNSTATE_T conn_state;
404 int is_master;
405 short version_common;
406
407 VCHIQ_SHARED_STATE_T *local;
408 VCHIQ_SHARED_STATE_T *remote;
409 VCHIQ_SLOT_T *slot_data;
410
411 unsigned short default_slot_quota;
412 unsigned short default_message_quota;
413
414
415 struct semaphore connect;
416
417
418 struct mutex mutex;
419 VCHIQ_INSTANCE_T *instance;
420
421
422 struct task_struct *slot_handler_thread;
423
424
425 struct task_struct *recycle_thread;
426
427
428 struct task_struct *sync_thread;
429
430
431 struct semaphore trigger_event;
432
433
434 struct semaphore recycle_event;
435
436
437 struct semaphore sync_trigger_event;
438
439
440 struct semaphore sync_release_event;
441
442 char *tx_data;
443 char *rx_data;
444 VCHIQ_SLOT_INFO_T *rx_info;
445
446 struct mutex slot_mutex;
447
448 struct mutex recycle_mutex;
449
450 struct mutex sync_mutex;
451
452 struct mutex bulk_transfer_mutex;
453
454
455
456
457
458 int rx_pos;
459
460
461
462 int local_tx_pos;
463
464
465 int slot_queue_available;
466
467
468 int poll_needed;
469
470
471 int previous_data_index;
472
473
474 unsigned short data_use_count;
475
476
477 unsigned short data_quota;
478
479
480 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
481
482
483 int unused_service;
484
485
486 struct semaphore slot_available_event;
487
488 struct semaphore slot_remove_event;
489
490
491 struct semaphore data_quota_event;
492
493
494
495 int deferred_bulks;
496
497 struct state_stats_struct {
498 int slot_stalls;
499 int data_stalls;
500 int ctrl_tx_count;
501 int ctrl_rx_count;
502 int error_count;
503 } stats;
504
505 VCHIQ_SERVICE_T * services[VCHIQ_MAX_SERVICES];
506 VCHIQ_SERVICE_QUOTA_T service_quotas[VCHIQ_MAX_SERVICES];
507 VCHIQ_SLOT_INFO_T slot_info[VCHIQ_MAX_SLOTS];
508
509 VCHIQ_PLATFORM_STATE_T platform_state;
510};
511
512struct bulk_waiter {
513 VCHIQ_BULK_T *bulk;
514 struct semaphore event;
515 int actual;
516};
517
518extern spinlock_t bulk_waiter_spinlock;
519
520extern int vchiq_core_log_level;
521extern int vchiq_core_msg_log_level;
522extern int vchiq_sync_log_level;
523
524extern VCHIQ_STATE_T *vchiq_states[VCHIQ_MAX_STATES];
525
526extern const char *
527get_conn_state_name(VCHIQ_CONNSTATE_T conn_state);
528
529extern VCHIQ_SLOT_ZERO_T *
530vchiq_init_slots(void *mem_base, int mem_size);
531
532extern VCHIQ_STATUS_T
533vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
534 int is_master);
535
536extern VCHIQ_STATUS_T
537vchiq_connect_internal(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance);
538
539extern VCHIQ_SERVICE_T *
540vchiq_add_service_internal(VCHIQ_STATE_T *state,
541 const VCHIQ_SERVICE_PARAMS_T *params, int srvstate,
542 VCHIQ_INSTANCE_T instance, VCHIQ_USERDATA_TERM_T userdata_term);
543
544extern VCHIQ_STATUS_T
545vchiq_open_service_internal(VCHIQ_SERVICE_T *service, int client_id);
546
547extern VCHIQ_STATUS_T
548vchiq_close_service_internal(VCHIQ_SERVICE_T *service, int close_recvd);
549
550extern void
551vchiq_terminate_service_internal(VCHIQ_SERVICE_T *service);
552
553extern void
554vchiq_free_service_internal(VCHIQ_SERVICE_T *service);
555
556extern VCHIQ_STATUS_T
557vchiq_shutdown_internal(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance);
558
559extern VCHIQ_STATUS_T
560vchiq_pause_internal(VCHIQ_STATE_T *state);
561
562extern VCHIQ_STATUS_T
563vchiq_resume_internal(VCHIQ_STATE_T *state);
564
565extern void
566remote_event_pollall(VCHIQ_STATE_T *state);
567
568extern VCHIQ_STATUS_T
569vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
570 VCHI_MEM_HANDLE_T memhandle, void *offset, int size, void *userdata,
571 VCHIQ_BULK_MODE_T mode, VCHIQ_BULK_DIR_T dir);
572
573extern void
574vchiq_dump_state(void *dump_context, VCHIQ_STATE_T *state);
575
576extern void
577vchiq_dump_service_state(void *dump_context, VCHIQ_SERVICE_T *service);
578
579extern void
580vchiq_loud_error_header(void);
581
582extern void
583vchiq_loud_error_footer(void);
584
585extern void
586request_poll(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service, int poll_type);
587
588static inline VCHIQ_SERVICE_T *
589handle_to_service(VCHIQ_SERVICE_HANDLE_T handle)
590{
591 VCHIQ_STATE_T *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) &
592 (VCHIQ_MAX_STATES - 1)];
593 if (!state)
594 return NULL;
595
596 return state->services[handle & (VCHIQ_MAX_SERVICES - 1)];
597}
598
599extern VCHIQ_SERVICE_T *
600find_service_by_handle(VCHIQ_SERVICE_HANDLE_T handle);
601
602extern VCHIQ_SERVICE_T *
603find_service_by_port(VCHIQ_STATE_T *state, int localport);
604
605extern VCHIQ_SERVICE_T *
606find_service_for_instance(VCHIQ_INSTANCE_T instance,
607 VCHIQ_SERVICE_HANDLE_T handle);
608
609extern VCHIQ_SERVICE_T *
610find_closed_service_for_instance(VCHIQ_INSTANCE_T instance,
611 VCHIQ_SERVICE_HANDLE_T handle);
612
613extern VCHIQ_SERVICE_T *
614next_service_by_instance(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance,
615 int *pidx);
616
617extern void
618lock_service(VCHIQ_SERVICE_T *service);
619
620extern void
621unlock_service(VCHIQ_SERVICE_T *service);
622
623
624
625
626extern VCHIQ_STATUS_T
627vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk,
628 VCHI_MEM_HANDLE_T memhandle, void *offset, int size, int dir);
629
630extern void
631vchiq_transfer_bulk(VCHIQ_BULK_T *bulk);
632
633extern void
634vchiq_complete_bulk(VCHIQ_BULK_T *bulk);
635
636extern VCHIQ_STATUS_T
637vchiq_copy_from_user(void *dst, const void *src, int size);
638
639extern void
640remote_event_signal(REMOTE_EVENT_T *event);
641
642void
643vchiq_platform_check_suspend(VCHIQ_STATE_T *state);
644
645extern void
646vchiq_platform_paused(VCHIQ_STATE_T *state);
647
648extern VCHIQ_STATUS_T
649vchiq_platform_resume(VCHIQ_STATE_T *state);
650
651extern void
652vchiq_platform_resumed(VCHIQ_STATE_T *state);
653
654extern void
655vchiq_dump(void *dump_context, const char *str, int len);
656
657extern void
658vchiq_dump_platform_state(void *dump_context);
659
660extern void
661vchiq_dump_platform_instances(void *dump_context);
662
663extern void
664vchiq_dump_platform_service_state(void *dump_context,
665 VCHIQ_SERVICE_T *service);
666
667extern VCHIQ_STATUS_T
668vchiq_use_service_internal(VCHIQ_SERVICE_T *service);
669
670extern VCHIQ_STATUS_T
671vchiq_release_service_internal(VCHIQ_SERVICE_T *service);
672
673extern void
674vchiq_on_remote_use(VCHIQ_STATE_T *state);
675
676extern void
677vchiq_on_remote_release(VCHIQ_STATE_T *state);
678
679extern VCHIQ_STATUS_T
680vchiq_platform_init_state(VCHIQ_STATE_T *state);
681
682extern VCHIQ_STATUS_T
683vchiq_check_service(VCHIQ_SERVICE_T *service);
684
685extern void
686vchiq_on_remote_use_active(VCHIQ_STATE_T *state);
687
688extern VCHIQ_STATUS_T
689vchiq_send_remote_use(VCHIQ_STATE_T *state);
690
691extern VCHIQ_STATUS_T
692vchiq_send_remote_release(VCHIQ_STATE_T *state);
693
694extern VCHIQ_STATUS_T
695vchiq_send_remote_use_active(VCHIQ_STATE_T *state);
696
697extern void
698vchiq_platform_conn_state_changed(VCHIQ_STATE_T *state,
699 VCHIQ_CONNSTATE_T oldstate, VCHIQ_CONNSTATE_T newstate);
700
701extern void
702vchiq_platform_handle_timeout(VCHIQ_STATE_T *state);
703
704extern void
705vchiq_set_conn_state(VCHIQ_STATE_T *state, VCHIQ_CONNSTATE_T newstate);
706
707
708extern void
709vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
710 size_t numBytes);
711
712#endif
713