1
2
3
4
5
6
7
8
9
10
11#include "mali_timeline.h"
12#include "mali_kernel_common.h"
13#include "mali_scheduler.h"
14#include "mali_soft_job.h"
15#include "mali_timeline_fence_wait.h"
16#include "mali_timeline_sync_fence.h"
17#include "mali_executor.h"
18#include "mali_pp_job.h"
19
20#define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
21
22
23
24
25
26
27
28_mali_osk_atomic_t gp_tracker_count;
29_mali_osk_atomic_t phy_pp_tracker_count;
30_mali_osk_atomic_t virt_pp_tracker_count;
31
32static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
33 struct mali_timeline_waiter *waiter);
34
35#if defined(CONFIG_SYNC)
36#include <linux/version.h>
37#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
38#include <linux/list.h>
39#include <linux/workqueue.h>
40#include <linux/spinlock.h>
41
42struct mali_deferred_fence_put_entry {
43 struct hlist_node list;
44 struct sync_fence *fence;
45};
46
47static HLIST_HEAD(mali_timeline_sync_fence_to_free_list);
48static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock);
49
50static void put_sync_fences(struct work_struct *ignore)
51{
52 struct hlist_head list;
53 struct hlist_node *tmp, *pos;
54 unsigned long flags;
55 struct mali_deferred_fence_put_entry *o;
56
57 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
58 hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list);
59 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
60
61 hlist_for_each_entry_safe(o, pos, tmp, &list, list) {
62 sync_fence_put(o->fence);
63 kfree(o);
64 }
65}
66
67static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences);
68#endif
69
70
71static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
72{
73 struct mali_timeline_system *system;
74 struct mali_timeline_waiter *waiter;
75 struct mali_timeline_tracker *tracker;
76 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
77 u32 tid = _mali_osk_get_tid();
78 mali_bool is_aborting = MALI_FALSE;
79#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
80 int fence_status = sync_fence->status;
81#else
82 int fence_status = atomic_read(&sync_fence->status);
83#endif
84
85 MALI_DEBUG_ASSERT_POINTER(sync_fence);
86 MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
87
88 tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
89 MALI_DEBUG_ASSERT_POINTER(tracker);
90
91 system = tracker->system;
92 MALI_DEBUG_ASSERT_POINTER(system);
93 MALI_DEBUG_ASSERT_POINTER(system->session);
94
95 mali_spinlock_reentrant_wait(system->spinlock, tid);
96
97 is_aborting = system->session->is_aborting;
98 if (!is_aborting && (0 > fence_status)) {
99 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
100 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
101 }
102
103 waiter = tracker->waiter_sync;
104 MALI_DEBUG_ASSERT_POINTER(waiter);
105
106 tracker->sync_fence = NULL;
107 tracker->fence.sync_fd = -1;
108
109 schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
110
111
112 if (is_aborting) {
113 _mali_osk_wait_queue_wake_up(system->wait_queue);
114 }
115
116 mali_spinlock_reentrant_signal(system->spinlock, tid);
117
118
119
120
121
122
123#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
124 {
125 struct mali_deferred_fence_put_entry *obj;
126
127 obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC);
128 if (obj) {
129 unsigned long flags;
130 mali_bool schedule = MALI_FALSE;
131
132 obj->fence = sync_fence;
133
134 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
135 if (hlist_empty(&mali_timeline_sync_fence_to_free_list))
136 schedule = MALI_TRUE;
137 hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list);
138 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
139
140 if (schedule)
141 schedule_delayed_work(&delayed_sync_fence_put, 0);
142 }
143 }
144#else
145 sync_fence_put(sync_fence);
146#endif
147
148 if (!is_aborting) {
149 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
150 }
151}
152#endif
153
154static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
155{
156 MALI_DEBUG_ASSERT_POINTER(tracker);
157 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
158
159 return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
160}
161
162static void mali_timeline_timer_callback(void *data)
163{
164 struct mali_timeline_system *system;
165 struct mali_timeline_tracker *tracker;
166 struct mali_timeline *timeline;
167 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
168 u32 tid = _mali_osk_get_tid();
169
170 timeline = (struct mali_timeline *) data;
171 MALI_DEBUG_ASSERT_POINTER(timeline);
172
173 system = timeline->system;
174 MALI_DEBUG_ASSERT_POINTER(system);
175
176 mali_spinlock_reentrant_wait(system->spinlock, tid);
177
178 if (!system->timer_enabled) {
179 mali_spinlock_reentrant_signal(system->spinlock, tid);
180 return;
181 }
182
183 tracker = timeline->tracker_tail;
184 timeline->timer_active = MALI_FALSE;
185
186 if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
187
188 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
189 mali_spinlock_reentrant_signal(system->spinlock, tid);
190 return;
191 }
192
193 schedule_mask = mali_timeline_tracker_time_out(tracker);
194 tracker->timer_active = MALI_FALSE;
195 } else {
196 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
197 }
198
199 mali_spinlock_reentrant_signal(system->spinlock, tid);
200
201 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
202}
203
204void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
205{
206 u32 i;
207 u32 tid = _mali_osk_get_tid();
208
209 MALI_DEBUG_ASSERT_POINTER(system);
210
211 mali_spinlock_reentrant_wait(system->spinlock, tid);
212 system->timer_enabled = MALI_FALSE;
213 mali_spinlock_reentrant_signal(system->spinlock, tid);
214
215 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
216 struct mali_timeline *timeline = system->timelines[i];
217
218 MALI_DEBUG_ASSERT_POINTER(timeline);
219
220 if (NULL != timeline->delayed_work) {
221 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
222 timeline->timer_active = MALI_FALSE;
223 }
224 }
225}
226
227static void mali_timeline_destroy(struct mali_timeline *timeline)
228{
229 MALI_DEBUG_ASSERT_POINTER(timeline);
230 if (NULL != timeline) {
231
232 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
233 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
234 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
235 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
236 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
237 MALI_DEBUG_ASSERT(NULL != timeline->system);
238 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
239
240#if defined(CONFIG_SYNC)
241 if (NULL != timeline->sync_tl) {
242 sync_timeline_destroy(timeline->sync_tl);
243 }
244#endif
245
246 if (NULL != timeline->delayed_work) {
247 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
248 _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
249 }
250
251 _mali_osk_free(timeline);
252 }
253}
254
255static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
256{
257 struct mali_timeline *timeline;
258
259 MALI_DEBUG_ASSERT_POINTER(system);
260 MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
261
262 timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
263 if (NULL == timeline) {
264 return NULL;
265 }
266
267
268#if defined(MALI_TIMELINE_DEBUG_START_POINT)
269
270 timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
271#else
272 timeline->point_next = 1;
273#endif
274 timeline->point_oldest = timeline->point_next;
275
276
277
278 timeline->system = system;
279 timeline->id = id;
280
281 timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
282 if (NULL == timeline->delayed_work) {
283 mali_timeline_destroy(timeline);
284 return NULL;
285 }
286
287 timeline->timer_active = MALI_FALSE;
288
289#if defined(CONFIG_SYNC)
290 {
291 char timeline_name[32];
292
293 switch (id) {
294 case MALI_TIMELINE_GP:
295 _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
296 break;
297 case MALI_TIMELINE_PP:
298 _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
299 break;
300 case MALI_TIMELINE_SOFT:
301 _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
302 break;
303 default:
304 MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
305 mali_timeline_destroy(timeline);
306 return NULL;
307 }
308
309 timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name);
310 if (NULL == timeline->sync_tl) {
311 mali_timeline_destroy(timeline);
312 return NULL;
313 }
314 }
315#endif
316
317 return timeline;
318}
319
320static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
321{
322 MALI_DEBUG_ASSERT_POINTER(timeline);
323 MALI_DEBUG_ASSERT_POINTER(tracker);
324
325 if (mali_timeline_is_full(timeline)) {
326
327 tracker->point = MALI_TIMELINE_NO_POINT;
328 return;
329 }
330
331 tracker->timeline = timeline;
332 tracker->point = timeline->point_next;
333
334
335 timeline->point_next++;
336 if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
337 timeline->point_next++;
338 }
339
340 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
341
342 if (MALI_TIMELINE_TRACKER_GP == tracker->type) {
343 _mali_osk_atomic_inc(&gp_tracker_count);
344 } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) {
345 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
346 _mali_osk_atomic_inc(&virt_pp_tracker_count);
347 } else {
348 _mali_osk_atomic_inc(&phy_pp_tracker_count);
349 }
350 }
351
352
353 if (NULL == timeline->tracker_head) {
354
355 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
356
357 timeline->tracker_tail = tracker;
358
359 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
360 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
361 } else {
362 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
363
364 tracker->timeline_prev = timeline->tracker_head;
365 timeline->tracker_head->timeline_next = tracker;
366
367 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
368 }
369 timeline->tracker_head = tracker;
370
371 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
372 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
373}
374
375
376static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
377{
378 struct mali_timeline_waiter *waiter_prev;
379 struct mali_timeline_waiter *waiter_next;
380
381
382
383 MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
384 MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
385
386
387 waiter_prev = timeline->waiter_head;
388 waiter_next = NULL;
389
390
391
392 while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
393 waiter_next = waiter_prev;
394 waiter_prev = waiter_prev->timeline_prev;
395 }
396
397 if (NULL == waiter_prev && NULL == waiter_next) {
398
399 timeline->waiter_head = waiter_new;
400 timeline->waiter_tail = waiter_new;
401 } else if (NULL == waiter_next) {
402
403 waiter_new->timeline_prev = timeline->waiter_head;
404 timeline->waiter_head->timeline_next = waiter_new;
405 timeline->waiter_head = waiter_new;
406 } else if (NULL == waiter_prev) {
407
408 waiter_new->timeline_next = timeline->waiter_tail;
409 timeline->waiter_tail->timeline_prev = waiter_new;
410 timeline->waiter_tail = waiter_new;
411 } else {
412
413 waiter_new->timeline_next = waiter_next;
414 waiter_new->timeline_prev = waiter_prev;
415 waiter_next->timeline_prev = waiter_new;
416 waiter_prev->timeline_next = waiter_new;
417 }
418}
419
420static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
421{
422 struct mali_timeline_system *system;
423 struct mali_timeline_tracker *oldest_tracker;
424
425 MALI_DEBUG_ASSERT_POINTER(timeline);
426 MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
427
428 system = timeline->system;
429 MALI_DEBUG_ASSERT_POINTER(system);
430
431 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
432
433
434 if (!system->timer_enabled) return;
435
436 oldest_tracker = timeline->tracker_tail;
437 if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
438 if (MALI_FALSE == oldest_tracker->timer_active) {
439 if (MALI_TRUE == timeline->timer_active) {
440 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
441 }
442 _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
443 oldest_tracker->timer_active = MALI_TRUE;
444 timeline->timer_active = MALI_TRUE;
445 }
446 } else if (MALI_TRUE == timeline->timer_active) {
447 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
448 timeline->timer_active = MALI_FALSE;
449 }
450}
451
452static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
453{
454 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
455
456 MALI_DEBUG_ASSERT_POINTER(timeline);
457
458 MALI_DEBUG_CODE({
459 struct mali_timeline_system *system = timeline->system;
460 MALI_DEBUG_ASSERT_POINTER(system);
461
462 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
463 });
464
465 if (NULL != timeline->tracker_tail) {
466
467 timeline->point_oldest = timeline->tracker_tail->point;
468 } else {
469
470 timeline->point_oldest = timeline->point_next;
471 }
472
473
474
475
476 while (NULL != timeline->waiter_tail) {
477 u32 waiter_time_relative;
478 u32 time_head_relative;
479 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
480
481 time_head_relative = timeline->point_next - timeline->point_oldest;
482 waiter_time_relative = waiter->point - timeline->point_oldest;
483
484 if (waiter_time_relative < time_head_relative) {
485
486 break;
487 }
488
489
490 if (NULL != waiter->timeline_next) {
491 waiter->timeline_next->timeline_prev = NULL;
492 } else {
493
494 timeline->waiter_head = NULL;
495 }
496 timeline->waiter_tail = waiter->timeline_next;
497
498
499
500 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
501 }
502
503 return schedule_mask;
504}
505
506void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
507 mali_timeline_tracker_type type,
508 struct mali_timeline_fence *fence,
509 void *job)
510{
511 MALI_DEBUG_ASSERT_POINTER(tracker);
512 MALI_DEBUG_ASSERT_POINTER(job);
513
514 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
515
516
517 _mali_osk_memset(tracker, 0, sizeof(*tracker));
518
519 tracker->type = type;
520 tracker->job = job;
521 tracker->trigger_ref_count = 1;
522 tracker->os_tick_create = _mali_osk_time_tickcount();
523 MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
524
525 tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
526
527
528 if (NULL != fence) {
529 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
530 }
531}
532
533mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
534{
535 struct mali_timeline *timeline;
536 struct mali_timeline_system *system;
537 struct mali_timeline_tracker *tracker_next, *tracker_prev;
538 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
539 u32 tid = _mali_osk_get_tid();
540
541
542 MALI_DEBUG_ASSERT_POINTER(tracker);
543 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
544
545
546 MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
547
548
549 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
550 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
551
552 MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
553
554 timeline = tracker->timeline;
555 if (NULL == timeline) {
556
557 return MALI_SCHEDULER_MASK_EMPTY;
558 }
559
560 system = timeline->system;
561 MALI_DEBUG_ASSERT_POINTER(system);
562
563 mali_spinlock_reentrant_wait(system->spinlock, tid);
564
565
566 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
567 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point));
568
569
570 MALI_DEBUG_CODE(tracker->magic = 0);
571
572 tracker_next = tracker->timeline_next;
573 tracker_prev = tracker->timeline_prev;
574 tracker->timeline_next = NULL;
575 tracker->timeline_prev = NULL;
576
577
578 if (NULL == tracker_next) {
579
580 timeline->tracker_head = tracker_prev;
581 } else {
582 tracker_next->timeline_prev = tracker_prev;
583 }
584
585 if (NULL == tracker_prev) {
586
587 timeline->tracker_tail = tracker_next;
588 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
589
590 schedule_mask |= mali_timeline_update_oldest_point(timeline);
591 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
592 } else {
593 tracker_prev->timeline_next = tracker_next;
594 }
595
596 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
597
598
599 if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
600 mali_timeline_update_delayed_work(tracker->timeline);
601 }
602
603 mali_spinlock_reentrant_signal(system->spinlock, tid);
604
605 return schedule_mask;
606}
607
608void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
609 struct mali_timeline_waiter *tail,
610 struct mali_timeline_waiter *head)
611{
612 MALI_DEBUG_ASSERT_POINTER(system);
613 MALI_DEBUG_ASSERT_POINTER(head);
614 MALI_DEBUG_ASSERT_POINTER(tail);
615 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
616
617 head->tracker_next = system->waiter_empty_list;
618 system->waiter_empty_list = tail;
619}
620
621static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
622{
623 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
624 struct mali_timeline_system *system;
625 struct mali_timeline *timeline;
626 u32 tid = _mali_osk_get_tid();
627
628 MALI_DEBUG_ASSERT_POINTER(tracker);
629 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
630
631 system = tracker->system;
632 MALI_DEBUG_ASSERT_POINTER(system);
633 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
634
635 tracker->os_tick_activate = _mali_osk_time_tickcount();
636
637 if (NULL != tracker->waiter_head) {
638 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
639 tracker->waiter_head = NULL;
640 tracker->waiter_tail = NULL;
641 }
642
643 switch (tracker->type) {
644 case MALI_TIMELINE_TRACKER_GP:
645 schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job);
646
647 _mali_osk_atomic_dec(&gp_tracker_count);
648 break;
649 case MALI_TIMELINE_TRACKER_PP:
650 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
651 _mali_osk_atomic_dec(&virt_pp_tracker_count);
652 } else {
653 _mali_osk_atomic_dec(&phy_pp_tracker_count);
654 }
655 schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job);
656 break;
657 case MALI_TIMELINE_TRACKER_SOFT:
658 timeline = tracker->timeline;
659 MALI_DEBUG_ASSERT_POINTER(timeline);
660
661 schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
662
663
664 mali_spinlock_reentrant_wait(system->spinlock, tid);
665 mali_timeline_update_delayed_work(timeline);
666 mali_spinlock_reentrant_signal(system->spinlock, tid);
667 break;
668 case MALI_TIMELINE_TRACKER_WAIT:
669 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
670 break;
671 case MALI_TIMELINE_TRACKER_SYNC:
672#if defined(CONFIG_SYNC)
673 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
674#else
675 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
676#endif
677 break;
678 default:
679 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
680 break;
681 }
682
683 return schedule_mask;
684}
685
686void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
687{
688 u32 tid = _mali_osk_get_tid();
689
690 MALI_DEBUG_ASSERT_POINTER(tracker);
691 MALI_DEBUG_ASSERT_POINTER(system);
692
693 mali_spinlock_reentrant_wait(system->spinlock, tid);
694
695 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
696 tracker->trigger_ref_count++;
697
698 mali_spinlock_reentrant_signal(system->spinlock, tid);
699}
700
701mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
702{
703 u32 tid = _mali_osk_get_tid();
704 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
705
706 MALI_DEBUG_ASSERT_POINTER(tracker);
707 MALI_DEBUG_ASSERT_POINTER(system);
708
709 mali_spinlock_reentrant_wait(system->spinlock, tid);
710
711 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
712 tracker->trigger_ref_count--;
713
714 tracker->activation_error |= activation_error;
715
716 if (0 == tracker->trigger_ref_count) {
717 schedule_mask |= mali_timeline_tracker_activate(tracker);
718 tracker = NULL;
719 }
720
721 mali_spinlock_reentrant_signal(system->spinlock, tid);
722
723 return schedule_mask;
724}
725
726void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
727{
728 u32 i;
729
730 MALI_DEBUG_ASSERT_POINTER(fence);
731 MALI_DEBUG_ASSERT_POINTER(uk_fence);
732
733 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
734 fence->points[i] = uk_fence->points[i];
735 }
736
737 fence->sync_fd = uk_fence->sync_fd;
738}
739
740struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
741{
742 u32 i;
743 struct mali_timeline_system *system;
744
745 MALI_DEBUG_ASSERT_POINTER(session);
746 MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
747
748 system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
749 if (NULL == system) {
750 return NULL;
751 }
752
753 system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
754 if (NULL == system->spinlock) {
755 mali_timeline_system_destroy(system);
756 return NULL;
757 }
758
759 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
760 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
761 if (NULL == system->timelines[i]) {
762 mali_timeline_system_destroy(system);
763 return NULL;
764 }
765 }
766
767#if defined(CONFIG_SYNC)
768 system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled");
769 if (NULL == system->signaled_sync_tl) {
770 mali_timeline_system_destroy(system);
771 return NULL;
772 }
773#endif
774
775 system->waiter_empty_list = NULL;
776 system->session = session;
777 system->timer_enabled = MALI_TRUE;
778
779 system->wait_queue = _mali_osk_wait_queue_init();
780 if (NULL == system->wait_queue) {
781 mali_timeline_system_destroy(system);
782 return NULL;
783 }
784
785 return system;
786}
787
788#if defined(CONFIG_SYNC)
789
790
791
792
793
794
795
796
797
798static mali_bool mali_timeline_has_no_trackers(void *data)
799{
800 struct mali_timeline *timeline = (struct mali_timeline *) data;
801
802 MALI_DEBUG_ASSERT_POINTER(timeline);
803
804 return mali_timeline_is_empty(timeline);
805}
806
807
808
809
810
811
812
813
814static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
815{
816 u32 i;
817 u32 tid = _mali_osk_get_tid();
818 struct mali_timeline_tracker *tracker, *tracker_next;
819 _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
820
821 MALI_DEBUG_ASSERT_POINTER(system);
822 MALI_DEBUG_ASSERT_POINTER(system->session);
823 MALI_DEBUG_ASSERT(system->session->is_aborting);
824
825 mali_spinlock_reentrant_wait(system->spinlock, tid);
826
827
828 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
829 struct mali_timeline *timeline = system->timelines[i];
830
831 MALI_DEBUG_ASSERT_POINTER(timeline);
832
833 tracker_next = timeline->tracker_tail;
834 while (NULL != tracker_next) {
835 tracker = tracker_next;
836 tracker_next = tracker->timeline_next;
837
838 if (NULL == tracker->sync_fence) continue;
839
840 MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
841
842
843 if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
844
845 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
846 }
847 }
848 }
849
850 mali_spinlock_reentrant_signal(system->spinlock, tid);
851
852
853 _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
854 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
855 }
856
857
858 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
859 struct mali_timeline *timeline = system->timelines[i];
860
861 MALI_DEBUG_ASSERT_POINTER(timeline);
862
863 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
864 }
865}
866
867#endif
868
869void mali_timeline_system_abort(struct mali_timeline_system *system)
870{
871 MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
872
873 MALI_DEBUG_ASSERT_POINTER(system);
874 MALI_DEBUG_ASSERT_POINTER(system->session);
875 MALI_DEBUG_ASSERT(system->session->is_aborting);
876
877 MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
878
879#if defined(CONFIG_SYNC)
880 mali_timeline_cancel_sync_fence_waiters(system);
881#endif
882
883
884 MALI_DEBUG_CODE({
885 u32 i;
886 mali_spinlock_reentrant_wait(system->spinlock, tid);
887 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
888 {
889 struct mali_timeline *timeline = system->timelines[i];
890 MALI_DEBUG_ASSERT_POINTER(timeline);
891 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
892 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
893 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
894 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
895 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
896 }
897 mali_spinlock_reentrant_signal(system->spinlock, tid);
898 });
899}
900
901void mali_timeline_system_destroy(struct mali_timeline_system *system)
902{
903 u32 i;
904 struct mali_timeline_waiter *waiter, *next;
905
906 MALI_DEBUG_ASSERT_POINTER(system);
907 MALI_DEBUG_ASSERT_POINTER(system->session);
908
909 MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
910
911 if (NULL != system) {
912
913 if (NULL != system->wait_queue) {
914 _mali_osk_wait_queue_term(system->wait_queue);
915 system->wait_queue = NULL;
916 }
917
918
919 waiter = system->waiter_empty_list;
920 while (NULL != waiter) {
921 next = waiter->tracker_next;
922 _mali_osk_free(waiter);
923 waiter = next;
924 }
925
926#if defined(CONFIG_SYNC)
927 if (NULL != system->signaled_sync_tl) {
928 sync_timeline_destroy(system->signaled_sync_tl);
929 }
930#endif
931
932 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
933 if (NULL != system->timelines[i]) {
934 mali_timeline_destroy(system->timelines[i]);
935 }
936 }
937 if (NULL != system->spinlock) {
938 mali_spinlock_reentrant_term(system->spinlock);
939 }
940
941 _mali_osk_free(system);
942 }
943}
944
945
946
947
948
949
950
951static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
952{
953 u32 i, num_waiters = 0;
954
955 MALI_DEBUG_ASSERT_POINTER(fence);
956
957 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
958 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
959 ++num_waiters;
960 }
961 }
962
963#if defined(CONFIG_SYNC)
964 if (-1 != fence->sync_fd) ++num_waiters;
965#endif
966
967 return num_waiters;
968}
969
970static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
971{
972 struct mali_timeline_waiter *waiter;
973
974 MALI_DEBUG_ASSERT_POINTER(system);
975 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
976
977 waiter = system->waiter_empty_list;
978 if (NULL != waiter) {
979
980 system->waiter_empty_list = waiter->tracker_next;
981 _mali_osk_memset(waiter, 0, sizeof(*waiter));
982 }
983
984
985 return waiter;
986}
987
988static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
989 struct mali_timeline_waiter **tail,
990 struct mali_timeline_waiter **head,
991 int max_num_waiters)
992{
993 u32 i, tid = _mali_osk_get_tid();
994 mali_bool do_alloc;
995 struct mali_timeline_waiter *waiter;
996
997 MALI_DEBUG_ASSERT_POINTER(system);
998 MALI_DEBUG_ASSERT_POINTER(tail);
999 MALI_DEBUG_ASSERT_POINTER(head);
1000
1001 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1002
1003 *head = *tail = NULL;
1004 do_alloc = MALI_FALSE;
1005 i = 0;
1006 while (i < max_num_waiters) {
1007 if (MALI_FALSE == do_alloc) {
1008 waiter = mali_timeline_system_get_zeroed_waiter(system);
1009 if (NULL == waiter) {
1010 do_alloc = MALI_TRUE;
1011 mali_spinlock_reentrant_signal(system->spinlock, tid);
1012 continue;
1013 }
1014 } else {
1015 waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
1016 if (NULL == waiter) break;
1017 }
1018 ++i;
1019 if (NULL == *tail) {
1020 *tail = waiter;
1021 *head = waiter;
1022 } else {
1023 (*head)->tracker_next = waiter;
1024 *head = waiter;
1025 }
1026 }
1027 if (MALI_TRUE == do_alloc) {
1028 mali_spinlock_reentrant_wait(system->spinlock, tid);
1029 }
1030}
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
1043 struct mali_timeline_tracker *tracker,
1044 struct mali_timeline_waiter *waiter_tail,
1045 struct mali_timeline_waiter *waiter_head)
1046{
1047 int i;
1048 u32 tid = _mali_osk_get_tid();
1049 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1050#if defined(CONFIG_SYNC)
1051 struct sync_fence *sync_fence = NULL;
1052#endif
1053
1054 MALI_DEBUG_ASSERT_POINTER(system);
1055 MALI_DEBUG_ASSERT_POINTER(tracker);
1056
1057 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1058
1059 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
1060 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1061 MALI_DEBUG_ASSERT(NULL != tracker->job);
1062
1063
1064
1065 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1066 mali_timeline_point point;
1067 struct mali_timeline *timeline;
1068 struct mali_timeline_waiter *waiter;
1069
1070
1071 point = tracker->fence.points[i];
1072
1073 if (likely(MALI_TIMELINE_NO_POINT == point)) {
1074
1075 continue;
1076 }
1077
1078 timeline = system->timelines[i];
1079 MALI_DEBUG_ASSERT_POINTER(timeline);
1080
1081 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
1082 MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
1083 point, timeline->point_oldest, timeline->point_next));
1084 continue;
1085 }
1086
1087 if (likely(mali_timeline_is_point_released(timeline, point))) {
1088
1089
1090 continue;
1091 }
1092
1093
1094 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
1095
1096
1097 if (likely(NULL != waiter_tail)) {
1098 waiter = waiter_tail;
1099 waiter_tail = waiter_tail->tracker_next;
1100 } else {
1101 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1102 continue;
1103 }
1104
1105
1106 tracker->trigger_ref_count++;
1107
1108 waiter->point = point;
1109 waiter->tracker = tracker;
1110
1111
1112 if (NULL == tracker->waiter_head) {
1113
1114 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1115 tracker->waiter_tail = waiter;
1116 } else {
1117 tracker->waiter_head->tracker_next = waiter;
1118 }
1119 tracker->waiter_head = waiter;
1120
1121
1122 mali_timeline_insert_waiter(timeline, waiter);
1123 }
1124#if defined(CONFIG_SYNC)
1125 if (-1 != tracker->fence.sync_fd) {
1126 int ret;
1127 struct mali_timeline_waiter *waiter;
1128
1129 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1130 if (unlikely(NULL == sync_fence)) {
1131 MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1132 goto exit;
1133 }
1134
1135
1136 if (unlikely(NULL == waiter_tail)) {
1137 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1138 goto exit;
1139 }
1140
1141
1142 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1143 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1144 if (1 == ret) {
1145
1146 tracker->fence.sync_fd = -1;
1147 goto exit;
1148 } else if (0 != ret) {
1149 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1150 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1151 goto exit;
1152 }
1153
1154
1155 waiter = waiter_tail;
1156 waiter_tail = waiter_tail->tracker_next;
1157
1158
1159 tracker->trigger_ref_count++;
1160
1161 waiter->point = MALI_TIMELINE_NO_POINT;
1162 waiter->tracker = tracker;
1163
1164
1165 if (NULL == tracker->waiter_head) {
1166
1167 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1168 tracker->waiter_tail = waiter;
1169 } else {
1170 tracker->waiter_head->tracker_next = waiter;
1171 }
1172 tracker->waiter_head = waiter;
1173
1174
1175 tracker->waiter_sync = waiter;
1176
1177
1178 tracker->sync_fence = sync_fence;
1179
1180 sync_fence = NULL;
1181 }
1182exit:
1183#endif
1184
1185 if (NULL != waiter_tail) {
1186 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1187 }
1188
1189
1190 tracker->trigger_ref_count--;
1191
1192
1193 if (0 == tracker->trigger_ref_count) {
1194 schedule_mask |= mali_timeline_tracker_activate(tracker);
1195 }
1196
1197 mali_spinlock_reentrant_signal(system->spinlock, tid);
1198
1199#if defined(CONFIG_SYNC)
1200 if (NULL != sync_fence) {
1201 sync_fence_put(sync_fence);
1202 }
1203#endif
1204
1205 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1206}
1207
1208mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
1209 struct mali_timeline_tracker *tracker,
1210 enum mali_timeline_id timeline_id)
1211{
1212 int num_waiters = 0;
1213 struct mali_timeline_waiter *waiter_tail, *waiter_head;
1214 u32 tid = _mali_osk_get_tid();
1215 mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1216
1217 MALI_DEBUG_ASSERT_POINTER(system);
1218 MALI_DEBUG_ASSERT_POINTER(system->session);
1219 MALI_DEBUG_ASSERT_POINTER(tracker);
1220
1221 MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1222 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1223 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1224
1225 MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1226
1227 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1228 tracker->system = system;
1229
1230 mali_spinlock_reentrant_wait(system->spinlock, tid);
1231
1232 num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1233
1234
1235 mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1236 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1237
1238
1239
1240
1241
1242
1243
1244 MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1245 if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1246 struct mali_timeline *timeline = system->timelines[timeline_id];
1247 mali_timeline_insert_tracker(timeline, tracker);
1248 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1249 }
1250
1251 point = tracker->point;
1252
1253
1254
1255 mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1256 tracker = NULL;
1257
1258
1259
1260
1261
1262
1263
1264 return point;
1265}
1266
1267static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
1268 struct mali_timeline_waiter *waiter)
1269{
1270 struct mali_timeline_tracker *tracker;
1271 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1272
1273 MALI_DEBUG_ASSERT_POINTER(system);
1274 MALI_DEBUG_ASSERT_POINTER(waiter);
1275
1276 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1277
1278 tracker = waiter->tracker;
1279 MALI_DEBUG_ASSERT_POINTER(tracker);
1280
1281
1282
1283
1284
1285 waiter->point = MALI_TIMELINE_NO_POINT;
1286 waiter->tracker = NULL;
1287
1288 tracker->trigger_ref_count--;
1289 if (0 == tracker->trigger_ref_count) {
1290
1291 schedule_mask |= mali_timeline_tracker_activate(tracker);
1292 tracker = NULL;
1293 }
1294
1295 return schedule_mask;
1296}
1297
1298mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
1299 enum mali_timeline_id timeline_id)
1300{
1301 mali_timeline_point point;
1302 struct mali_timeline *timeline;
1303 u32 tid = _mali_osk_get_tid();
1304
1305 MALI_DEBUG_ASSERT_POINTER(system);
1306
1307 if (MALI_TIMELINE_MAX <= timeline_id) {
1308 return MALI_TIMELINE_NO_POINT;
1309 }
1310
1311 mali_spinlock_reentrant_wait(system->spinlock, tid);
1312
1313 timeline = system->timelines[timeline_id];
1314 MALI_DEBUG_ASSERT_POINTER(timeline);
1315
1316 point = MALI_TIMELINE_NO_POINT;
1317 if (timeline->point_oldest != timeline->point_next) {
1318 point = timeline->point_next - 1;
1319 if (MALI_TIMELINE_NO_POINT == point) point--;
1320 }
1321
1322 mali_spinlock_reentrant_signal(system->spinlock, tid);
1323
1324 return point;
1325}
1326
1327void mali_timeline_initialize(void)
1328{
1329 _mali_osk_atomic_init(&gp_tracker_count, 0);
1330 _mali_osk_atomic_init(&phy_pp_tracker_count, 0);
1331 _mali_osk_atomic_init(&virt_pp_tracker_count, 0);
1332}
1333
1334void mali_timeline_terminate(void)
1335{
1336 _mali_osk_atomic_term(&gp_tracker_count);
1337 _mali_osk_atomic_term(&phy_pp_tracker_count);
1338 _mali_osk_atomic_term(&virt_pp_tracker_count);
1339}
1340
1341#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1342
1343static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1344{
1345 struct mali_timeline *timeline;
1346 struct mali_timeline_system *system;
1347
1348 MALI_DEBUG_ASSERT_POINTER(tracker);
1349
1350 MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1351 timeline = tracker->timeline;
1352
1353 MALI_DEBUG_ASSERT_POINTER(timeline->system);
1354 system = timeline->system;
1355
1356 if (MALI_TIMELINE_MAX > id) {
1357 if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) {
1358 return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1359 } else {
1360 return MALI_FALSE;
1361 }
1362 } else {
1363 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1364 return MALI_FALSE;
1365 }
1366}
1367
1368static const char *timeline_id_to_string(enum mali_timeline_id id)
1369{
1370 switch (id) {
1371 case MALI_TIMELINE_GP:
1372 return "GP";
1373 case MALI_TIMELINE_PP:
1374 return "PP";
1375 case MALI_TIMELINE_SOFT:
1376 return "SOFT";
1377 default:
1378 return "NONE";
1379 }
1380}
1381
1382static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1383{
1384 switch (type) {
1385 case MALI_TIMELINE_TRACKER_GP:
1386 return "GP";
1387 case MALI_TIMELINE_TRACKER_PP:
1388 return "PP";
1389 case MALI_TIMELINE_TRACKER_SOFT:
1390 return "SOFT";
1391 case MALI_TIMELINE_TRACKER_WAIT:
1392 return "WAIT";
1393 case MALI_TIMELINE_TRACKER_SYNC:
1394 return "SYNC";
1395 default:
1396 return "INVALID";
1397 }
1398}
1399
1400mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1401{
1402 struct mali_timeline *timeline = NULL;
1403
1404 MALI_DEBUG_ASSERT_POINTER(tracker);
1405 timeline = tracker->timeline;
1406
1407 if (0 != tracker->trigger_ref_count) {
1408 return MALI_TIMELINE_TS_WAITING;
1409 }
1410
1411 if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1412 return MALI_TIMELINE_TS_ACTIVE;
1413 }
1414
1415 if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1416 return MALI_TIMELINE_TS_INIT;
1417 }
1418
1419 return MALI_TIMELINE_TS_FINISH;
1420}
1421
1422void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx)
1423{
1424 const char *tracker_state = "IWAF";
1425 char state_char = 'I';
1426 char tracker_type[32] = {0};
1427
1428 MALI_DEBUG_ASSERT_POINTER(tracker);
1429
1430 state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1431 _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1432
1433#if defined(CONFIG_SYNC)
1434 if (0 != tracker->trigger_ref_count) {
1435 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1436 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1437 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1438 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1439 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1440 tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1441 } else {
1442 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1443 tracker_type, tracker->point, state_char,
1444 tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1445 }
1446#else
1447 if (0 != tracker->trigger_ref_count) {
1448 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1449 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1450 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1451 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1452 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1453 tracker->job);
1454 } else {
1455 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n",
1456 tracker_type, tracker->point, state_char,
1457 tracker->job);
1458 }
1459#endif
1460}
1461
1462void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx)
1463{
1464 struct mali_timeline_tracker *tracker = NULL;
1465
1466 MALI_DEBUG_ASSERT_POINTER(timeline);
1467
1468 tracker = timeline->tracker_tail;
1469 while (NULL != tracker) {
1470 mali_timeline_debug_print_tracker(tracker, print_ctx);
1471 tracker = tracker->timeline_next;
1472 }
1473}
1474
1475void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx)
1476{
1477 int i;
1478 int num_printed = 0;
1479 u32 tid = _mali_osk_get_tid();
1480
1481 MALI_DEBUG_ASSERT_POINTER(system);
1482
1483 mali_spinlock_reentrant_wait(system->spinlock, tid);
1484
1485
1486 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1487 struct mali_timeline *timeline = system->timelines[i];
1488
1489 MALI_DEBUG_ASSERT_POINTER(timeline);
1490
1491 if (NULL == timeline->tracker_head) continue;
1492
1493 _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n",
1494 timeline_id_to_string((enum mali_timeline_id)i));
1495
1496 mali_timeline_debug_print_timeline(timeline, print_ctx);
1497 num_printed++;
1498 }
1499
1500 if (0 == num_printed) {
1501 _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n");
1502 }
1503
1504 mali_spinlock_reentrant_signal(system->spinlock, tid);
1505}
1506
1507#endif
1508