1
2
3
4#ifndef VCHIQ_CORE_H
5#define VCHIQ_CORE_H
6
7#include <linux/mutex.h>
8#include <linux/completion.h>
9#include <linux/kthread.h>
10#include <linux/kref.h>
11#include <linux/rcupdate.h>
12#include <linux/wait.h>
13
14#include "vchiq_cfg.h"
15
16#include "vchiq.h"
17
18
19#define VCHIQ_LOG_DEFAULT 4
20#define VCHIQ_LOG_ERROR 3
21#define VCHIQ_LOG_WARNING 4
22#define VCHIQ_LOG_INFO 6
23#define VCHIQ_LOG_TRACE 7
24
25#define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: "
26
27#ifndef vchiq_log_error
28#define vchiq_log_error(cat, fmt, ...) \
29 do { if (cat >= VCHIQ_LOG_ERROR) \
30 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
31#endif
32#ifndef vchiq_log_warning
33#define vchiq_log_warning(cat, fmt, ...) \
34 do { if (cat >= VCHIQ_LOG_WARNING) \
35 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
36#endif
37#ifndef vchiq_log_info
38#define vchiq_log_info(cat, fmt, ...) \
39 do { if (cat >= VCHIQ_LOG_INFO) \
40 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
41#endif
42#ifndef vchiq_log_trace
43#define vchiq_log_trace(cat, fmt, ...) \
44 do { if (cat >= VCHIQ_LOG_TRACE) \
45 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
46#endif
47
48#define vchiq_loud_error(...) \
49 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
50
51#ifndef vchiq_static_assert
52#define vchiq_static_assert(cond) __attribute__((unused)) \
53 extern int vchiq_static_assert[(cond) ? 1 : -1]
54#endif
55
56#define IS_POW2(x) (x && ((x & (x - 1)) == 0))
57
58
59vchiq_static_assert(IS_POW2(VCHIQ_SLOT_SIZE));
60vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS));
61vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS_PER_SIDE));
62
63#define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1)
64#define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
65#define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \
66 VCHIQ_SLOT_SIZE)
67
68#define VCHIQ_MSG_PADDING 0
69#define VCHIQ_MSG_CONNECT 1
70#define VCHIQ_MSG_OPEN 2
71#define VCHIQ_MSG_OPENACK 3
72#define VCHIQ_MSG_CLOSE 4
73#define VCHIQ_MSG_DATA 5
74#define VCHIQ_MSG_BULK_RX 6
75#define VCHIQ_MSG_BULK_TX 7
76#define VCHIQ_MSG_BULK_RX_DONE 8
77#define VCHIQ_MSG_BULK_TX_DONE 9
78#define VCHIQ_MSG_PAUSE 10
79#define VCHIQ_MSG_RESUME 11
80#define VCHIQ_MSG_REMOTE_USE 12
81#define VCHIQ_MSG_REMOTE_RELEASE 13
82#define VCHIQ_MSG_REMOTE_USE_ACTIVE 14
83
84#define VCHIQ_PORT_MAX (VCHIQ_MAX_SERVICES - 1)
85#define VCHIQ_PORT_FREE 0x1000
86#define VCHIQ_PORT_IS_VALID(port) (port < VCHIQ_PORT_FREE)
87#define VCHIQ_MAKE_MSG(type, srcport, dstport) \
88 ((type<<24) | (srcport<<12) | (dstport<<0))
89#define VCHIQ_MSG_TYPE(msgid) ((unsigned int)msgid >> 24)
90#define VCHIQ_MSG_SRCPORT(msgid) \
91 (unsigned short)(((unsigned int)msgid >> 12) & 0xfff)
92#define VCHIQ_MSG_DSTPORT(msgid) \
93 ((unsigned short)msgid & 0xfff)
94
95#define VCHIQ_FOURCC_AS_4CHARS(fourcc) \
96 ((fourcc) >> 24) & 0xff, \
97 ((fourcc) >> 16) & 0xff, \
98 ((fourcc) >> 8) & 0xff, \
99 (fourcc) & 0xff
100
101
102vchiq_static_assert(VCHIQ_MSG_SRCPORT(VCHIQ_MAKE_MSG(0, 0, VCHIQ_PORT_MAX))
103 == 0);
104vchiq_static_assert(VCHIQ_MSG_TYPE(VCHIQ_MAKE_MSG(0, VCHIQ_PORT_MAX, 0)) == 0);
105vchiq_static_assert((unsigned int)VCHIQ_PORT_MAX <
106 (unsigned int)VCHIQ_PORT_FREE);
107
108#define VCHIQ_MSGID_PADDING VCHIQ_MAKE_MSG(VCHIQ_MSG_PADDING, 0, 0)
109#define VCHIQ_MSGID_CLAIMED 0x40000000
110
111#define VCHIQ_FOURCC_INVALID 0x00000000
112#define VCHIQ_FOURCC_IS_LEGAL(fourcc) (fourcc != VCHIQ_FOURCC_INVALID)
113
114#define VCHIQ_BULK_ACTUAL_ABORTED -1
115
116typedef uint32_t BITSET_T;
117
118vchiq_static_assert((sizeof(BITSET_T) * 8) == 32);
119
120#define BITSET_SIZE(b) ((b + 31) >> 5)
121#define BITSET_WORD(b) (b >> 5)
122#define BITSET_BIT(b) (1 << (b & 31))
123#define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b))
124#define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
125#define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b))
126
127#if VCHIQ_ENABLE_STATS
128#define VCHIQ_STATS_INC(state, stat) (state->stats. stat++)
129#define VCHIQ_SERVICE_STATS_INC(service, stat) (service->stats. stat++)
130#define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) \
131 (service->stats. stat += addend)
132#else
133#define VCHIQ_STATS_INC(state, stat) ((void)0)
134#define VCHIQ_SERVICE_STATS_INC(service, stat) ((void)0)
135#define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) ((void)0)
136#endif
137
138enum {
139 DEBUG_ENTRIES,
140#if VCHIQ_ENABLE_DEBUG
141 DEBUG_SLOT_HANDLER_COUNT,
142 DEBUG_SLOT_HANDLER_LINE,
143 DEBUG_PARSE_LINE,
144 DEBUG_PARSE_HEADER,
145 DEBUG_PARSE_MSGID,
146 DEBUG_AWAIT_COMPLETION_LINE,
147 DEBUG_DEQUEUE_MESSAGE_LINE,
148 DEBUG_SERVICE_CALLBACK_LINE,
149 DEBUG_MSG_QUEUE_FULL_COUNT,
150 DEBUG_COMPLETION_QUEUE_FULL_COUNT,
151#endif
152 DEBUG_MAX
153};
154
155#if VCHIQ_ENABLE_DEBUG
156
157#define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
158#define DEBUG_TRACE(d) \
159 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
160#define DEBUG_VALUE(d, v) \
161 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
162#define DEBUG_COUNT(d) \
163 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
164
165#else
166
167#define DEBUG_INITIALISE(local)
168#define DEBUG_TRACE(d)
169#define DEBUG_VALUE(d, v)
170#define DEBUG_COUNT(d)
171
172#endif
173
174enum vchiq_connstate {
175 VCHIQ_CONNSTATE_DISCONNECTED,
176 VCHIQ_CONNSTATE_CONNECTING,
177 VCHIQ_CONNSTATE_CONNECTED,
178 VCHIQ_CONNSTATE_PAUSING,
179 VCHIQ_CONNSTATE_PAUSE_SENT,
180 VCHIQ_CONNSTATE_PAUSED,
181 VCHIQ_CONNSTATE_RESUMING,
182 VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
183 VCHIQ_CONNSTATE_RESUME_TIMEOUT
184};
185
186enum {
187 VCHIQ_SRVSTATE_FREE,
188 VCHIQ_SRVSTATE_HIDDEN,
189 VCHIQ_SRVSTATE_LISTENING,
190 VCHIQ_SRVSTATE_OPENING,
191 VCHIQ_SRVSTATE_OPEN,
192 VCHIQ_SRVSTATE_OPENSYNC,
193 VCHIQ_SRVSTATE_CLOSESENT,
194 VCHIQ_SRVSTATE_CLOSERECVD,
195 VCHIQ_SRVSTATE_CLOSEWAIT,
196 VCHIQ_SRVSTATE_CLOSED
197};
198
199enum {
200 VCHIQ_POLL_TERMINATE,
201 VCHIQ_POLL_REMOVE,
202 VCHIQ_POLL_TXNOTIFY,
203 VCHIQ_POLL_RXNOTIFY,
204 VCHIQ_POLL_COUNT
205};
206
207enum vchiq_bulk_dir {
208 VCHIQ_BULK_TRANSMIT,
209 VCHIQ_BULK_RECEIVE
210};
211
212typedef void (*vchiq_userdata_term)(void *userdata);
213
214struct vchiq_bulk {
215 short mode;
216 short dir;
217 void *userdata;
218 void *data;
219 int size;
220 void *remote_data;
221 int remote_size;
222 int actual;
223};
224
225struct vchiq_bulk_queue {
226 int local_insert;
227 int remote_insert;
228 int process;
229 int remote_notify;
230 int remove;
231
232 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS];
233};
234
235struct remote_event {
236 int armed;
237 int fired;
238 u32 __unused;
239};
240
241struct opaque_platform_state;
242
243struct vchiq_slot {
244 char data[VCHIQ_SLOT_SIZE];
245};
246
247struct vchiq_slot_info {
248
249 short use_count;
250 short release_count;
251};
252
253struct vchiq_service {
254 struct vchiq_service_base base;
255 unsigned int handle;
256 struct kref ref_count;
257 struct rcu_head rcu;
258 int srvstate;
259 vchiq_userdata_term userdata_term;
260 unsigned int localport;
261 unsigned int remoteport;
262 int public_fourcc;
263 int client_id;
264 char auto_close;
265 char sync;
266 char closing;
267 char trace;
268 atomic_t poll_flags;
269 short version;
270 short version_min;
271 short peer_version;
272
273 struct vchiq_state *state;
274 struct vchiq_instance *instance;
275
276 int service_use_count;
277
278 struct vchiq_bulk_queue bulk_tx;
279 struct vchiq_bulk_queue bulk_rx;
280
281 struct completion remove_event;
282 struct completion bulk_remove_event;
283 struct mutex bulk_mutex;
284
285 struct service_stats_struct {
286 int quota_stalls;
287 int slot_stalls;
288 int bulk_stalls;
289 int error_count;
290 int ctrl_tx_count;
291 int ctrl_rx_count;
292 int bulk_tx_count;
293 int bulk_rx_count;
294 int bulk_aborted_count;
295 uint64_t ctrl_tx_bytes;
296 uint64_t ctrl_rx_bytes;
297 uint64_t bulk_tx_bytes;
298 uint64_t bulk_rx_bytes;
299 } stats;
300};
301
302
303
304
305
306struct vchiq_service_quota {
307 unsigned short slot_quota;
308 unsigned short slot_use_count;
309 unsigned short message_quota;
310 unsigned short message_use_count;
311 struct completion quota_event;
312 int previous_tx_index;
313};
314
315struct vchiq_shared_state {
316
317
318 int initialised;
319
320
321 int slot_first;
322 int slot_last;
323
324
325 int slot_sync;
326
327
328
329 struct remote_event trigger;
330
331
332
333
334 int tx_pos;
335
336
337 struct remote_event recycle;
338
339
340 int slot_queue_recycle;
341
342
343 struct remote_event sync_trigger;
344
345
346
347 struct remote_event sync_release;
348
349
350 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
351
352
353 int debug[DEBUG_MAX];
354};
355
356struct vchiq_slot_zero {
357 int magic;
358 short version;
359 short version_min;
360 int slot_zero_size;
361 int slot_size;
362 int max_slots;
363 int max_slots_per_side;
364 int platform_data[2];
365 struct vchiq_shared_state master;
366 struct vchiq_shared_state slave;
367 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS];
368};
369
370struct vchiq_state {
371 int id;
372 int initialised;
373 enum vchiq_connstate conn_state;
374 short version_common;
375
376 struct vchiq_shared_state *local;
377 struct vchiq_shared_state *remote;
378 struct vchiq_slot *slot_data;
379
380 unsigned short default_slot_quota;
381 unsigned short default_message_quota;
382
383
384 struct completion connect;
385
386
387 struct mutex mutex;
388 struct vchiq_instance **instance;
389
390
391 struct task_struct *slot_handler_thread;
392
393
394 struct task_struct *recycle_thread;
395
396
397 struct task_struct *sync_thread;
398
399
400 wait_queue_head_t trigger_event;
401
402
403 wait_queue_head_t recycle_event;
404
405
406 wait_queue_head_t sync_trigger_event;
407
408
409 wait_queue_head_t sync_release_event;
410
411 char *tx_data;
412 char *rx_data;
413 struct vchiq_slot_info *rx_info;
414
415 struct mutex slot_mutex;
416
417 struct mutex recycle_mutex;
418
419 struct mutex sync_mutex;
420
421 struct mutex bulk_transfer_mutex;
422
423
424
425
426
427 int rx_pos;
428
429
430
431 int local_tx_pos;
432
433
434 int slot_queue_available;
435
436
437 int poll_needed;
438
439
440 int previous_data_index;
441
442
443 unsigned short data_use_count;
444
445
446 unsigned short data_quota;
447
448
449 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
450
451
452 int unused_service;
453
454
455 struct completion slot_available_event;
456
457 struct completion slot_remove_event;
458
459
460 struct completion data_quota_event;
461
462 struct state_stats_struct {
463 int slot_stalls;
464 int data_stalls;
465 int ctrl_tx_count;
466 int ctrl_rx_count;
467 int error_count;
468 } stats;
469
470 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES];
471 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES];
472 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS];
473
474 struct opaque_platform_state *platform_state;
475};
476
477struct bulk_waiter {
478 struct vchiq_bulk *bulk;
479 struct completion event;
480 int actual;
481};
482
483extern spinlock_t bulk_waiter_spinlock;
484
485extern int vchiq_core_log_level;
486extern int vchiq_core_msg_log_level;
487extern int vchiq_sync_log_level;
488
489extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES];
490
491extern const char *
492get_conn_state_name(enum vchiq_connstate conn_state);
493
494extern struct vchiq_slot_zero *
495vchiq_init_slots(void *mem_base, int mem_size);
496
497extern enum vchiq_status
498vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero);
499
500extern enum vchiq_status
501vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance);
502
503extern struct vchiq_service *
504vchiq_add_service_internal(struct vchiq_state *state,
505 const struct vchiq_service_params *params,
506 int srvstate, struct vchiq_instance *instance,
507 vchiq_userdata_term userdata_term);
508
509extern enum vchiq_status
510vchiq_open_service_internal(struct vchiq_service *service, int client_id);
511
512extern enum vchiq_status
513vchiq_close_service_internal(struct vchiq_service *service, int close_recvd);
514
515extern void
516vchiq_terminate_service_internal(struct vchiq_service *service);
517
518extern void
519vchiq_free_service_internal(struct vchiq_service *service);
520
521extern enum vchiq_status
522vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance);
523
524extern void
525remote_event_pollall(struct vchiq_state *state);
526
527extern enum vchiq_status
528vchiq_bulk_transfer(unsigned int handle, void *offset, int size,
529 void *userdata, enum vchiq_bulk_mode mode,
530 enum vchiq_bulk_dir dir);
531
532extern int
533vchiq_dump_state(void *dump_context, struct vchiq_state *state);
534
535extern int
536vchiq_dump_service_state(void *dump_context, struct vchiq_service *service);
537
538extern void
539vchiq_loud_error_header(void);
540
541extern void
542vchiq_loud_error_footer(void);
543
544extern void
545request_poll(struct vchiq_state *state, struct vchiq_service *service,
546 int poll_type);
547
548static inline struct vchiq_service *
549handle_to_service(unsigned int handle)
550{
551 int idx = handle & (VCHIQ_MAX_SERVICES - 1);
552 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) &
553 (VCHIQ_MAX_STATES - 1)];
554
555 if (!state)
556 return NULL;
557 return rcu_dereference(state->services[idx]);
558}
559
560extern struct vchiq_service *
561find_service_by_handle(unsigned int handle);
562
563extern struct vchiq_service *
564find_service_by_port(struct vchiq_state *state, int localport);
565
566extern struct vchiq_service *
567find_service_for_instance(struct vchiq_instance *instance,
568 unsigned int handle);
569
570extern struct vchiq_service *
571find_closed_service_for_instance(struct vchiq_instance *instance,
572 unsigned int handle);
573
574extern struct vchiq_service *
575__next_service_by_instance(struct vchiq_state *state,
576 struct vchiq_instance *instance,
577 int *pidx);
578
579extern struct vchiq_service *
580next_service_by_instance(struct vchiq_state *state,
581 struct vchiq_instance *instance,
582 int *pidx);
583
584extern void
585lock_service(struct vchiq_service *service);
586
587extern void
588unlock_service(struct vchiq_service *service);
589
590
591
592
593extern enum vchiq_status
594vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
595 int dir);
596
597extern void
598vchiq_complete_bulk(struct vchiq_bulk *bulk);
599
600extern void
601remote_event_signal(struct remote_event *event);
602
603extern int
604vchiq_dump(void *dump_context, const char *str, int len);
605
606extern int
607vchiq_dump_platform_state(void *dump_context);
608
609extern int
610vchiq_dump_platform_instances(void *dump_context);
611
612extern int
613vchiq_dump_platform_service_state(void *dump_context,
614 struct vchiq_service *service);
615
616extern enum vchiq_status
617vchiq_use_service_internal(struct vchiq_service *service);
618
619extern enum vchiq_status
620vchiq_release_service_internal(struct vchiq_service *service);
621
622extern void
623vchiq_on_remote_use(struct vchiq_state *state);
624
625extern void
626vchiq_on_remote_release(struct vchiq_state *state);
627
628extern enum vchiq_status
629vchiq_platform_init_state(struct vchiq_state *state);
630
631extern enum vchiq_status
632vchiq_check_service(struct vchiq_service *service);
633
634extern void
635vchiq_on_remote_use_active(struct vchiq_state *state);
636
637extern enum vchiq_status
638vchiq_send_remote_use(struct vchiq_state *state);
639
640extern enum vchiq_status
641vchiq_send_remote_use_active(struct vchiq_state *state);
642
643extern void
644vchiq_platform_conn_state_changed(struct vchiq_state *state,
645 enum vchiq_connstate oldstate,
646 enum vchiq_connstate newstate);
647
648extern void
649vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate);
650
651extern void
652vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
653 size_t numBytes);
654
655#endif
656