1
2
3
4
5
6
7
8
9
10
11#include "mali_timeline.h"
12#include "mali_kernel_common.h"
13#include "mali_scheduler.h"
14#include "mali_soft_job.h"
15#include "mali_timeline_fence_wait.h"
16#include "mali_timeline_sync_fence.h"
17#include "mali_executor.h"
18#include "mali_pp_job.h"
19
20#define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
21
22
23
24
25
26
27
28_mali_osk_atomic_t gp_tracker_count;
29_mali_osk_atomic_t phy_pp_tracker_count;
30_mali_osk_atomic_t virt_pp_tracker_count;
31
32static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
33 struct mali_timeline_waiter *waiter);
34
35#if defined(CONFIG_SYNC)
36#include <linux/version.h>
37#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
38#include <linux/list.h>
39#include <linux/workqueue.h>
40#include <linux/spinlock.h>
41
42struct mali_deferred_fence_put_entry {
43 struct hlist_node list;
44 struct sync_fence *fence;
45};
46
47static HLIST_HEAD(mali_timeline_sync_fence_to_free_list);
48static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock);
49
50static void put_sync_fences(struct work_struct *ignore)
51{
52 struct hlist_head list;
53 struct hlist_node *tmp, *pos;
54 unsigned long flags;
55 struct mali_deferred_fence_put_entry *o;
56
57 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
58 hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list);
59 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
60
61 hlist_for_each_entry_safe(o, pos, tmp, &list, list) {
62 sync_fence_put(o->fence);
63 kfree(o);
64 }
65}
66
67static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences);
68#endif
69
70
71static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
72{
73 struct mali_timeline_system *system;
74 struct mali_timeline_waiter *waiter;
75 struct mali_timeline_tracker *tracker;
76 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
77 u32 tid = _mali_osk_get_tid();
78 mali_bool is_aborting = MALI_FALSE;
79#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)
80 int fence_status = sync_fence->status;
81#else
82 int fence_status = atomic_read(&sync_fence->status);
83#endif
84
85 MALI_DEBUG_ASSERT_POINTER(sync_fence);
86 MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
87
88 tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
89 MALI_DEBUG_ASSERT_POINTER(tracker);
90
91 system = tracker->system;
92 MALI_DEBUG_ASSERT_POINTER(system);
93 MALI_DEBUG_ASSERT_POINTER(system->session);
94
95 mali_spinlock_reentrant_wait(system->spinlock, tid);
96
97 is_aborting = system->session->is_aborting;
98 if (!is_aborting && (0 > fence_status)) {
99 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
100 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
101 }
102
103 waiter = tracker->waiter_sync;
104 MALI_DEBUG_ASSERT_POINTER(waiter);
105
106 tracker->sync_fence = NULL;
107 tracker->fence.sync_fd = -1;
108
109 schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
110
111
112 if (is_aborting) {
113 _mali_osk_wait_queue_wake_up(system->wait_queue);
114 }
115
116 mali_spinlock_reentrant_signal(system->spinlock, tid);
117
118
119
120
121
122
123#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
124 {
125 struct mali_deferred_fence_put_entry *obj;
126
127 obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC);
128 if (obj) {
129 unsigned long flags;
130 mali_bool schedule = MALI_FALSE;
131
132 obj->fence = sync_fence;
133
134 spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags);
135 if (hlist_empty(&mali_timeline_sync_fence_to_free_list))
136 schedule = MALI_TRUE;
137 hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list);
138 spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags);
139
140 if (schedule)
141 schedule_delayed_work(&delayed_sync_fence_put, 0);
142 }
143 }
144#else
145 sync_fence_put(sync_fence);
146#endif
147
148 if (!is_aborting) {
149 mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE);
150 }
151}
152#endif
153
154static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
155{
156 MALI_DEBUG_ASSERT_POINTER(tracker);
157 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
158
159 return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
160}
161
162static void mali_timeline_timer_callback(void *data)
163{
164 struct mali_timeline_system *system;
165 struct mali_timeline_tracker *tracker;
166 struct mali_timeline *timeline;
167 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
168 u32 tid = _mali_osk_get_tid();
169
170 timeline = (struct mali_timeline *) data;
171 MALI_DEBUG_ASSERT_POINTER(timeline);
172
173 system = timeline->system;
174 MALI_DEBUG_ASSERT_POINTER(system);
175
176 mali_spinlock_reentrant_wait(system->spinlock, tid);
177
178 if (!system->timer_enabled) {
179 mali_spinlock_reentrant_signal(system->spinlock, tid);
180 return;
181 }
182
183 tracker = timeline->tracker_tail;
184 timeline->timer_active = MALI_FALSE;
185
186 if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
187
188 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
189 mali_spinlock_reentrant_signal(system->spinlock, tid);
190 return;
191 }
192
193 schedule_mask = mali_timeline_tracker_time_out(tracker);
194 tracker->timer_active = MALI_FALSE;
195 } else {
196 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
197 }
198
199 mali_spinlock_reentrant_signal(system->spinlock, tid);
200
201 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
202}
203
204void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
205{
206 u32 i;
207 u32 tid = _mali_osk_get_tid();
208
209 MALI_DEBUG_ASSERT_POINTER(system);
210
211 mali_spinlock_reentrant_wait(system->spinlock, tid);
212 system->timer_enabled = MALI_FALSE;
213 mali_spinlock_reentrant_signal(system->spinlock, tid);
214
215 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
216 struct mali_timeline *timeline = system->timelines[i];
217
218 MALI_DEBUG_ASSERT_POINTER(timeline);
219
220 if (NULL != timeline->delayed_work) {
221 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
222 timeline->timer_active = MALI_FALSE;
223 }
224 }
225}
226
227static void mali_timeline_destroy(struct mali_timeline *timeline)
228{
229 MALI_DEBUG_ASSERT_POINTER(timeline);
230 if (NULL != timeline) {
231
232 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
233 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
234 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
235 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
236 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
237 MALI_DEBUG_ASSERT(NULL != timeline->system);
238 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
239
240#if defined(CONFIG_SYNC)
241 if (NULL != timeline->sync_tl) {
242 sync_timeline_destroy(timeline->sync_tl);
243 }
244#endif
245
246 if (NULL != timeline->delayed_work) {
247 _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
248 _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
249 }
250
251 _mali_osk_free(timeline);
252 }
253}
254
255static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
256{
257 struct mali_timeline *timeline;
258
259 MALI_DEBUG_ASSERT_POINTER(system);
260 MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
261
262 timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
263 if (NULL == timeline) {
264 return NULL;
265 }
266
267
268#if defined(MALI_TIMELINE_DEBUG_START_POINT)
269
270 timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
271#else
272 timeline->point_next = 1;
273#endif
274 timeline->point_oldest = timeline->point_next;
275
276
277
278 timeline->system = system;
279 timeline->id = id;
280
281 timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
282 if (NULL == timeline->delayed_work) {
283 mali_timeline_destroy(timeline);
284 return NULL;
285 }
286
287 timeline->timer_active = MALI_FALSE;
288
289#if defined(CONFIG_SYNC)
290 {
291 char timeline_name[32];
292
293 switch (id) {
294 case MALI_TIMELINE_GP:
295 _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
296 break;
297 case MALI_TIMELINE_PP:
298 _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
299 break;
300 case MALI_TIMELINE_SOFT:
301 _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
302 break;
303 default:
304 MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
305 mali_timeline_destroy(timeline);
306 return NULL;
307 }
308
309 timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name);
310 if (NULL == timeline->sync_tl) {
311 mali_timeline_destroy(timeline);
312 return NULL;
313 }
314 }
315#endif
316
317 return timeline;
318}
319
320static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
321{
322 MALI_DEBUG_ASSERT_POINTER(timeline);
323 MALI_DEBUG_ASSERT_POINTER(tracker);
324
325 if (mali_timeline_is_full(timeline)) {
326
327 tracker->point = MALI_TIMELINE_NO_POINT;
328 return;
329 }
330
331 tracker->timeline = timeline;
332 tracker->point = timeline->point_next;
333
334
335 timeline->point_next++;
336 if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
337 timeline->point_next++;
338 }
339
340 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
341
342 if (MALI_TIMELINE_TRACKER_GP == tracker->type) {
343 _mali_osk_atomic_inc(&gp_tracker_count);
344 } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) {
345 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
346 _mali_osk_atomic_inc(&virt_pp_tracker_count);
347 } else {
348 _mali_osk_atomic_inc(&phy_pp_tracker_count);
349 }
350 }
351
352
353 if (NULL == timeline->tracker_head) {
354
355 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
356
357 timeline->tracker_tail = tracker;
358
359 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
360 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
361 } else {
362 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
363
364 tracker->timeline_prev = timeline->tracker_head;
365 timeline->tracker_head->timeline_next = tracker;
366
367 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
368 }
369 timeline->tracker_head = tracker;
370
371 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
372 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
373}
374
375
376static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
377{
378 struct mali_timeline_waiter *waiter_prev;
379 struct mali_timeline_waiter *waiter_next;
380
381
382
383 MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
384 MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
385
386
387 waiter_prev = timeline->waiter_head;
388 waiter_next = NULL;
389
390
391
392 while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
393 waiter_next = waiter_prev;
394 waiter_prev = waiter_prev->timeline_prev;
395 }
396
397 if (NULL == waiter_prev && NULL == waiter_next) {
398
399 timeline->waiter_head = waiter_new;
400 timeline->waiter_tail = waiter_new;
401 } else if (NULL == waiter_next) {
402
403 waiter_new->timeline_prev = timeline->waiter_head;
404 timeline->waiter_head->timeline_next = waiter_new;
405 timeline->waiter_head = waiter_new;
406 } else if (NULL == waiter_prev) {
407
408 waiter_new->timeline_next = timeline->waiter_tail;
409 timeline->waiter_tail->timeline_prev = waiter_new;
410 timeline->waiter_tail = waiter_new;
411 } else {
412
413 waiter_new->timeline_next = waiter_next;
414 waiter_new->timeline_prev = waiter_prev;
415 waiter_next->timeline_prev = waiter_new;
416 waiter_prev->timeline_next = waiter_new;
417 }
418}
419
420static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
421{
422 struct mali_timeline_system *system;
423 struct mali_timeline_tracker *oldest_tracker;
424
425 MALI_DEBUG_ASSERT_POINTER(timeline);
426 MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
427
428 system = timeline->system;
429 MALI_DEBUG_ASSERT_POINTER(system);
430
431 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
432
433
434 if (!system->timer_enabled) return;
435
436 oldest_tracker = timeline->tracker_tail;
437 if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
438 if (MALI_FALSE == oldest_tracker->timer_active) {
439 if (MALI_TRUE == timeline->timer_active) {
440 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
441 }
442 _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
443 oldest_tracker->timer_active = MALI_TRUE;
444 timeline->timer_active = MALI_TRUE;
445 }
446 } else if (MALI_TRUE == timeline->timer_active) {
447 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
448 timeline->timer_active = MALI_FALSE;
449 }
450}
451
452static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
453{
454 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
455
456 MALI_DEBUG_ASSERT_POINTER(timeline);
457
458 MALI_DEBUG_CODE({
459 struct mali_timeline_system *system = timeline->system;
460 MALI_DEBUG_ASSERT_POINTER(system);
461
462 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
463 });
464
465 if (NULL != timeline->tracker_tail) {
466
467 timeline->point_oldest = timeline->tracker_tail->point;
468 } else {
469
470 timeline->point_oldest = timeline->point_next;
471 }
472
473
474
475
476 while (NULL != timeline->waiter_tail) {
477 u32 waiter_time_relative;
478 u32 time_head_relative;
479 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
480
481 time_head_relative = timeline->point_next - timeline->point_oldest;
482 waiter_time_relative = waiter->point - timeline->point_oldest;
483
484 if (waiter_time_relative < time_head_relative) {
485
486 break;
487 }
488
489
490 if (NULL != waiter->timeline_next) {
491 waiter->timeline_next->timeline_prev = NULL;
492 } else {
493
494 timeline->waiter_head = NULL;
495 }
496 timeline->waiter_tail = waiter->timeline_next;
497
498
499
500 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
501 }
502
503 return schedule_mask;
504}
505
506void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
507 mali_timeline_tracker_type type,
508 struct mali_timeline_fence *fence,
509 void *job)
510{
511 MALI_DEBUG_ASSERT_POINTER(tracker);
512 MALI_DEBUG_ASSERT_POINTER(job);
513
514 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
515
516
517 _mali_osk_memset(tracker, 0, sizeof(*tracker));
518
519 tracker->type = type;
520 tracker->job = job;
521 tracker->trigger_ref_count = 1;
522 tracker->os_tick_create = _mali_osk_time_tickcount();
523 MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
524
525 tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
526
527
528 if (NULL != fence) {
529 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
530 }
531}
532
533mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
534{
535 struct mali_timeline *timeline;
536 struct mali_timeline_system *system;
537 struct mali_timeline_tracker *tracker_next, *tracker_prev;
538 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
539 u32 tid = _mali_osk_get_tid();
540
541
542 MALI_DEBUG_ASSERT_POINTER(tracker);
543 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
544
545
546 MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
547
548
549 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
550 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
551
552 MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
553
554 timeline = tracker->timeline;
555 if (NULL == timeline) {
556
557 return MALI_SCHEDULER_MASK_EMPTY;
558 }
559
560 system = timeline->system;
561 MALI_DEBUG_ASSERT_POINTER(system);
562
563 mali_spinlock_reentrant_wait(system->spinlock, tid);
564
565
566 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
567 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point));
568
569
570 MALI_DEBUG_CODE(tracker->magic = 0);
571
572 tracker_next = tracker->timeline_next;
573 tracker_prev = tracker->timeline_prev;
574 tracker->timeline_next = NULL;
575 tracker->timeline_prev = NULL;
576
577
578 if (NULL == tracker_next) {
579
580 timeline->tracker_head = tracker_prev;
581 } else {
582 tracker_next->timeline_prev = tracker_prev;
583 }
584
585 if (NULL == tracker_prev) {
586
587 timeline->tracker_tail = tracker_next;
588 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
589
590 schedule_mask |= mali_timeline_update_oldest_point(timeline);
591 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
592 } else {
593 tracker_prev->timeline_next = tracker_next;
594 }
595
596 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
597
598
599 if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
600 mali_timeline_update_delayed_work(tracker->timeline);
601 }
602
603 mali_spinlock_reentrant_signal(system->spinlock, tid);
604
605 return schedule_mask;
606}
607
608void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
609 struct mali_timeline_waiter *tail,
610 struct mali_timeline_waiter *head)
611{
612 MALI_DEBUG_ASSERT_POINTER(system);
613 MALI_DEBUG_ASSERT_POINTER(head);
614 MALI_DEBUG_ASSERT_POINTER(tail);
615 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
616
617 head->tracker_next = system->waiter_empty_list;
618 system->waiter_empty_list = tail;
619}
620
621static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
622{
623 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
624 struct mali_timeline_system *system;
625 struct mali_timeline *timeline;
626 u32 tid = _mali_osk_get_tid();
627
628 MALI_DEBUG_ASSERT_POINTER(tracker);
629 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
630
631 system = tracker->system;
632 MALI_DEBUG_ASSERT_POINTER(system);
633 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
634
635 tracker->os_tick_activate = _mali_osk_time_tickcount();
636
637 if (NULL != tracker->waiter_head) {
638 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
639 tracker->waiter_head = NULL;
640 tracker->waiter_tail = NULL;
641 }
642
643 switch (tracker->type) {
644 case MALI_TIMELINE_TRACKER_GP:
645 schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job);
646
647 _mali_osk_atomic_dec(&gp_tracker_count);
648 break;
649 case MALI_TIMELINE_TRACKER_PP:
650 schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job);
651
652 if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) {
653 _mali_osk_atomic_dec(&virt_pp_tracker_count);
654 } else {
655 _mali_osk_atomic_dec(&phy_pp_tracker_count);
656 }
657 break;
658 case MALI_TIMELINE_TRACKER_SOFT:
659 timeline = tracker->timeline;
660 MALI_DEBUG_ASSERT_POINTER(timeline);
661
662 schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
663
664
665 mali_spinlock_reentrant_wait(system->spinlock, tid);
666 mali_timeline_update_delayed_work(timeline);
667 mali_spinlock_reentrant_signal(system->spinlock, tid);
668 break;
669 case MALI_TIMELINE_TRACKER_WAIT:
670 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
671 break;
672 case MALI_TIMELINE_TRACKER_SYNC:
673#if defined(CONFIG_SYNC)
674 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
675#else
676 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
677#endif
678 break;
679 default:
680 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
681 break;
682 }
683
684 return schedule_mask;
685}
686
687void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
688{
689 u32 tid = _mali_osk_get_tid();
690
691 MALI_DEBUG_ASSERT_POINTER(tracker);
692 MALI_DEBUG_ASSERT_POINTER(system);
693
694 mali_spinlock_reentrant_wait(system->spinlock, tid);
695
696 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
697 tracker->trigger_ref_count++;
698
699 mali_spinlock_reentrant_signal(system->spinlock, tid);
700}
701
702mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
703{
704 u32 tid = _mali_osk_get_tid();
705 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
706
707 MALI_DEBUG_ASSERT_POINTER(tracker);
708 MALI_DEBUG_ASSERT_POINTER(system);
709
710 mali_spinlock_reentrant_wait(system->spinlock, tid);
711
712 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
713 tracker->trigger_ref_count--;
714
715 tracker->activation_error |= activation_error;
716
717 if (0 == tracker->trigger_ref_count) {
718 schedule_mask |= mali_timeline_tracker_activate(tracker);
719 tracker = NULL;
720 }
721
722 mali_spinlock_reentrant_signal(system->spinlock, tid);
723
724 return schedule_mask;
725}
726
727void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
728{
729 u32 i;
730
731 MALI_DEBUG_ASSERT_POINTER(fence);
732 MALI_DEBUG_ASSERT_POINTER(uk_fence);
733
734 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
735 fence->points[i] = uk_fence->points[i];
736 }
737
738 fence->sync_fd = uk_fence->sync_fd;
739}
740
741struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
742{
743 u32 i;
744 struct mali_timeline_system *system;
745
746 MALI_DEBUG_ASSERT_POINTER(session);
747 MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
748
749 system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
750 if (NULL == system) {
751 return NULL;
752 }
753
754 system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
755 if (NULL == system->spinlock) {
756 mali_timeline_system_destroy(system);
757 return NULL;
758 }
759
760 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
761 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
762 if (NULL == system->timelines[i]) {
763 mali_timeline_system_destroy(system);
764 return NULL;
765 }
766 }
767
768#if defined(CONFIG_SYNC)
769 system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled");
770 if (NULL == system->signaled_sync_tl) {
771 mali_timeline_system_destroy(system);
772 return NULL;
773 }
774#endif
775
776 system->waiter_empty_list = NULL;
777 system->session = session;
778 system->timer_enabled = MALI_TRUE;
779
780 system->wait_queue = _mali_osk_wait_queue_init();
781 if (NULL == system->wait_queue) {
782 mali_timeline_system_destroy(system);
783 return NULL;
784 }
785
786 return system;
787}
788
789#if defined(CONFIG_SYNC)
790
791
792
793
794
795
796
797
798
799static mali_bool mali_timeline_has_no_trackers(void *data)
800{
801 struct mali_timeline *timeline = (struct mali_timeline *) data;
802
803 MALI_DEBUG_ASSERT_POINTER(timeline);
804
805 return mali_timeline_is_empty(timeline);
806}
807
808
809
810
811
812
813
814
815static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
816{
817 u32 i;
818 u32 tid = _mali_osk_get_tid();
819 struct mali_timeline_tracker *tracker, *tracker_next;
820 _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
821
822 MALI_DEBUG_ASSERT_POINTER(system);
823 MALI_DEBUG_ASSERT_POINTER(system->session);
824 MALI_DEBUG_ASSERT(system->session->is_aborting);
825
826 mali_spinlock_reentrant_wait(system->spinlock, tid);
827
828
829 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
830 struct mali_timeline *timeline = system->timelines[i];
831
832 MALI_DEBUG_ASSERT_POINTER(timeline);
833
834 tracker_next = timeline->tracker_tail;
835 while (NULL != tracker_next) {
836 tracker = tracker_next;
837 tracker_next = tracker->timeline_next;
838
839 if (NULL == tracker->sync_fence) continue;
840
841 MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
842
843
844 if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
845
846 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
847 }
848 }
849 }
850
851 mali_spinlock_reentrant_signal(system->spinlock, tid);
852
853
854 _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
855 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
856 }
857
858
859 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
860 struct mali_timeline *timeline = system->timelines[i];
861
862 MALI_DEBUG_ASSERT_POINTER(timeline);
863
864 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
865 }
866}
867
868#endif
869
870void mali_timeline_system_abort(struct mali_timeline_system *system)
871{
872 MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
873
874 MALI_DEBUG_ASSERT_POINTER(system);
875 MALI_DEBUG_ASSERT_POINTER(system->session);
876 MALI_DEBUG_ASSERT(system->session->is_aborting);
877
878 MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
879
880#if defined(CONFIG_SYNC)
881 mali_timeline_cancel_sync_fence_waiters(system);
882#endif
883
884
885 MALI_DEBUG_CODE({
886 u32 i;
887 mali_spinlock_reentrant_wait(system->spinlock, tid);
888 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
889 {
890 struct mali_timeline *timeline = system->timelines[i];
891 MALI_DEBUG_ASSERT_POINTER(timeline);
892 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
893 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
894 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
895 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
896 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
897 }
898 mali_spinlock_reentrant_signal(system->spinlock, tid);
899 });
900}
901
902void mali_timeline_system_destroy(struct mali_timeline_system *system)
903{
904 u32 i;
905 struct mali_timeline_waiter *waiter, *next;
906
907 MALI_DEBUG_ASSERT_POINTER(system);
908 MALI_DEBUG_ASSERT_POINTER(system->session);
909
910 MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
911
912 if (NULL != system) {
913
914 if (NULL != system->wait_queue) {
915 _mali_osk_wait_queue_term(system->wait_queue);
916 system->wait_queue = NULL;
917 }
918
919
920 waiter = system->waiter_empty_list;
921 while (NULL != waiter) {
922 next = waiter->tracker_next;
923 _mali_osk_free(waiter);
924 waiter = next;
925 }
926
927#if defined(CONFIG_SYNC)
928 if (NULL != system->signaled_sync_tl) {
929 sync_timeline_destroy(system->signaled_sync_tl);
930 }
931#endif
932
933 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
934 if (NULL != system->timelines[i]) {
935 mali_timeline_destroy(system->timelines[i]);
936 }
937 }
938 if (NULL != system->spinlock) {
939 mali_spinlock_reentrant_term(system->spinlock);
940 }
941
942 _mali_osk_free(system);
943 }
944}
945
946
947
948
949
950
951
952static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
953{
954 u32 i, num_waiters = 0;
955
956 MALI_DEBUG_ASSERT_POINTER(fence);
957
958 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
959 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
960 ++num_waiters;
961 }
962 }
963
964#if defined(CONFIG_SYNC)
965 if (-1 != fence->sync_fd) ++num_waiters;
966#endif
967
968 return num_waiters;
969}
970
971static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
972{
973 struct mali_timeline_waiter *waiter;
974
975 MALI_DEBUG_ASSERT_POINTER(system);
976 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
977
978 waiter = system->waiter_empty_list;
979 if (NULL != waiter) {
980
981 system->waiter_empty_list = waiter->tracker_next;
982 _mali_osk_memset(waiter, 0, sizeof(*waiter));
983 }
984
985
986 return waiter;
987}
988
989static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
990 struct mali_timeline_waiter **tail,
991 struct mali_timeline_waiter **head,
992 int max_num_waiters)
993{
994 u32 i, tid = _mali_osk_get_tid();
995 mali_bool do_alloc;
996 struct mali_timeline_waiter *waiter;
997
998 MALI_DEBUG_ASSERT_POINTER(system);
999 MALI_DEBUG_ASSERT_POINTER(tail);
1000 MALI_DEBUG_ASSERT_POINTER(head);
1001
1002 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1003
1004 *head = *tail = NULL;
1005 do_alloc = MALI_FALSE;
1006 i = 0;
1007 while (i < max_num_waiters) {
1008 if (MALI_FALSE == do_alloc) {
1009 waiter = mali_timeline_system_get_zeroed_waiter(system);
1010 if (NULL == waiter) {
1011 do_alloc = MALI_TRUE;
1012 mali_spinlock_reentrant_signal(system->spinlock, tid);
1013 continue;
1014 }
1015 } else {
1016 waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
1017 if (NULL == waiter) break;
1018 }
1019 ++i;
1020 if (NULL == *tail) {
1021 *tail = waiter;
1022 *head = waiter;
1023 } else {
1024 (*head)->tracker_next = waiter;
1025 *head = waiter;
1026 }
1027 }
1028 if (MALI_TRUE == do_alloc) {
1029 mali_spinlock_reentrant_wait(system->spinlock, tid);
1030 }
1031}
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
1044 struct mali_timeline_tracker *tracker,
1045 struct mali_timeline_waiter *waiter_tail,
1046 struct mali_timeline_waiter *waiter_head)
1047{
1048 int i;
1049 u32 tid = _mali_osk_get_tid();
1050 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1051#if defined(CONFIG_SYNC)
1052 struct sync_fence *sync_fence = NULL;
1053#endif
1054
1055 MALI_DEBUG_ASSERT_POINTER(system);
1056 MALI_DEBUG_ASSERT_POINTER(tracker);
1057
1058 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1059
1060 MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
1061 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1062 MALI_DEBUG_ASSERT(NULL != tracker->job);
1063
1064
1065
1066 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1067 mali_timeline_point point;
1068 struct mali_timeline *timeline;
1069 struct mali_timeline_waiter *waiter;
1070
1071
1072 point = tracker->fence.points[i];
1073
1074 if (likely(MALI_TIMELINE_NO_POINT == point)) {
1075
1076 continue;
1077 }
1078
1079 timeline = system->timelines[i];
1080 MALI_DEBUG_ASSERT_POINTER(timeline);
1081
1082 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
1083 MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
1084 point, timeline->point_oldest, timeline->point_next));
1085 continue;
1086 }
1087
1088 if (likely(mali_timeline_is_point_released(timeline, point))) {
1089
1090
1091 continue;
1092 }
1093
1094
1095 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
1096
1097
1098 if (likely(NULL != waiter_tail)) {
1099 waiter = waiter_tail;
1100 waiter_tail = waiter_tail->tracker_next;
1101 } else {
1102 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1103 continue;
1104 }
1105
1106
1107 tracker->trigger_ref_count++;
1108
1109 waiter->point = point;
1110 waiter->tracker = tracker;
1111
1112
1113 if (NULL == tracker->waiter_head) {
1114
1115 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1116 tracker->waiter_tail = waiter;
1117 } else {
1118 tracker->waiter_head->tracker_next = waiter;
1119 }
1120 tracker->waiter_head = waiter;
1121
1122
1123 mali_timeline_insert_waiter(timeline, waiter);
1124 }
1125#if defined(CONFIG_SYNC)
1126 if (-1 != tracker->fence.sync_fd) {
1127 int ret;
1128 struct mali_timeline_waiter *waiter;
1129
1130 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1131 if (unlikely(NULL == sync_fence)) {
1132 MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1133 goto exit;
1134 }
1135
1136
1137 if (unlikely(NULL == waiter_tail)) {
1138 MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1139 goto exit;
1140 }
1141
1142
1143 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1144 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1145 if (1 == ret) {
1146
1147 tracker->fence.sync_fd = -1;
1148 goto exit;
1149 } else if (0 != ret) {
1150 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1151 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1152 goto exit;
1153 }
1154
1155
1156 waiter = waiter_tail;
1157 waiter_tail = waiter_tail->tracker_next;
1158
1159
1160 tracker->trigger_ref_count++;
1161
1162 waiter->point = MALI_TIMELINE_NO_POINT;
1163 waiter->tracker = tracker;
1164
1165
1166 if (NULL == tracker->waiter_head) {
1167
1168 MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1169 tracker->waiter_tail = waiter;
1170 } else {
1171 tracker->waiter_head->tracker_next = waiter;
1172 }
1173 tracker->waiter_head = waiter;
1174
1175
1176 tracker->waiter_sync = waiter;
1177
1178
1179 tracker->sync_fence = sync_fence;
1180
1181 sync_fence = NULL;
1182 }
1183exit:
1184#endif
1185
1186 if (NULL != waiter_tail) {
1187 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1188 }
1189
1190
1191 tracker->trigger_ref_count--;
1192
1193
1194 if (0 == tracker->trigger_ref_count) {
1195 schedule_mask |= mali_timeline_tracker_activate(tracker);
1196 }
1197
1198 mali_spinlock_reentrant_signal(system->spinlock, tid);
1199
1200#if defined(CONFIG_SYNC)
1201 if (NULL != sync_fence) {
1202 sync_fence_put(sync_fence);
1203 }
1204#endif
1205
1206 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1207}
1208
1209mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
1210 struct mali_timeline_tracker *tracker,
1211 enum mali_timeline_id timeline_id)
1212{
1213 int num_waiters = 0;
1214 struct mali_timeline_waiter *waiter_tail, *waiter_head;
1215 u32 tid = _mali_osk_get_tid();
1216 mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1217
1218 MALI_DEBUG_ASSERT_POINTER(system);
1219 MALI_DEBUG_ASSERT_POINTER(system->session);
1220 MALI_DEBUG_ASSERT_POINTER(tracker);
1221
1222 MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1223 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1224 MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1225
1226 MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1227
1228 MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1229 tracker->system = system;
1230
1231 mali_spinlock_reentrant_wait(system->spinlock, tid);
1232
1233 num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1234
1235
1236 mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1237 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1238
1239
1240
1241
1242
1243
1244
1245 MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1246 if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1247 struct mali_timeline *timeline = system->timelines[timeline_id];
1248 mali_timeline_insert_tracker(timeline, tracker);
1249 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1250 }
1251
1252 point = tracker->point;
1253
1254
1255
1256 mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1257 tracker = NULL;
1258
1259
1260
1261
1262
1263
1264
1265 return point;
1266}
1267
1268static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
1269 struct mali_timeline_waiter *waiter)
1270{
1271 struct mali_timeline_tracker *tracker;
1272 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1273
1274 MALI_DEBUG_ASSERT_POINTER(system);
1275 MALI_DEBUG_ASSERT_POINTER(waiter);
1276
1277 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1278
1279 tracker = waiter->tracker;
1280 MALI_DEBUG_ASSERT_POINTER(tracker);
1281
1282
1283
1284
1285
1286 waiter->point = MALI_TIMELINE_NO_POINT;
1287 waiter->tracker = NULL;
1288
1289 tracker->trigger_ref_count--;
1290 if (0 == tracker->trigger_ref_count) {
1291
1292 schedule_mask |= mali_timeline_tracker_activate(tracker);
1293 tracker = NULL;
1294 }
1295
1296 return schedule_mask;
1297}
1298
1299mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
1300 enum mali_timeline_id timeline_id)
1301{
1302 mali_timeline_point point;
1303 struct mali_timeline *timeline;
1304 u32 tid = _mali_osk_get_tid();
1305
1306 MALI_DEBUG_ASSERT_POINTER(system);
1307
1308 if (MALI_TIMELINE_MAX <= timeline_id) {
1309 return MALI_TIMELINE_NO_POINT;
1310 }
1311
1312 mali_spinlock_reentrant_wait(system->spinlock, tid);
1313
1314 timeline = system->timelines[timeline_id];
1315 MALI_DEBUG_ASSERT_POINTER(timeline);
1316
1317 point = MALI_TIMELINE_NO_POINT;
1318 if (timeline->point_oldest != timeline->point_next) {
1319 point = timeline->point_next - 1;
1320 if (MALI_TIMELINE_NO_POINT == point) point--;
1321 }
1322
1323 mali_spinlock_reentrant_signal(system->spinlock, tid);
1324
1325 return point;
1326}
1327
1328void mali_timeline_initialize(void)
1329{
1330 _mali_osk_atomic_init(&gp_tracker_count, 0);
1331 _mali_osk_atomic_init(&phy_pp_tracker_count, 0);
1332 _mali_osk_atomic_init(&virt_pp_tracker_count, 0);
1333}
1334
1335void mali_timeline_terminate(void)
1336{
1337 _mali_osk_atomic_term(&gp_tracker_count);
1338 _mali_osk_atomic_term(&phy_pp_tracker_count);
1339 _mali_osk_atomic_term(&virt_pp_tracker_count);
1340}
1341
1342#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1343
1344static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1345{
1346 struct mali_timeline *timeline;
1347 struct mali_timeline_system *system;
1348
1349 MALI_DEBUG_ASSERT_POINTER(tracker);
1350
1351 MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1352 timeline = tracker->timeline;
1353
1354 MALI_DEBUG_ASSERT_POINTER(timeline->system);
1355 system = timeline->system;
1356
1357 if (MALI_TIMELINE_MAX > id) {
1358 if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) {
1359 return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1360 } else {
1361 return MALI_FALSE;
1362 }
1363 } else {
1364 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1365 return MALI_FALSE;
1366 }
1367}
1368
1369static const char *timeline_id_to_string(enum mali_timeline_id id)
1370{
1371 switch (id) {
1372 case MALI_TIMELINE_GP:
1373 return "GP";
1374 case MALI_TIMELINE_PP:
1375 return "PP";
1376 case MALI_TIMELINE_SOFT:
1377 return "SOFT";
1378 default:
1379 return "NONE";
1380 }
1381}
1382
1383static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1384{
1385 switch (type) {
1386 case MALI_TIMELINE_TRACKER_GP:
1387 return "GP";
1388 case MALI_TIMELINE_TRACKER_PP:
1389 return "PP";
1390 case MALI_TIMELINE_TRACKER_SOFT:
1391 return "SOFT";
1392 case MALI_TIMELINE_TRACKER_WAIT:
1393 return "WAIT";
1394 case MALI_TIMELINE_TRACKER_SYNC:
1395 return "SYNC";
1396 default:
1397 return "INVALID";
1398 }
1399}
1400
1401mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1402{
1403 struct mali_timeline *timeline = NULL;
1404
1405 MALI_DEBUG_ASSERT_POINTER(tracker);
1406 timeline = tracker->timeline;
1407
1408 if (0 != tracker->trigger_ref_count) {
1409 return MALI_TIMELINE_TS_WAITING;
1410 }
1411
1412 if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1413 return MALI_TIMELINE_TS_ACTIVE;
1414 }
1415
1416 if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1417 return MALI_TIMELINE_TS_INIT;
1418 }
1419
1420 return MALI_TIMELINE_TS_FINISH;
1421}
1422
1423void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx)
1424{
1425 const char *tracker_state = "IWAF";
1426 char state_char = 'I';
1427 char tracker_type[32] = {0};
1428
1429 MALI_DEBUG_ASSERT_POINTER(tracker);
1430
1431 state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker));
1432 _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type));
1433
1434#if defined(CONFIG_SYNC)
1435 if (0 != tracker->trigger_ref_count) {
1436 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n",
1437 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1438 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1439 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1440 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1441 tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1442 } else {
1443 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n",
1444 tracker_type, tracker->point, state_char,
1445 tracker->fence.sync_fd, tracker->sync_fence, tracker->job);
1446 }
1447#else
1448 if (0 != tracker->trigger_ref_count) {
1449 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n",
1450 tracker_type, tracker->point, state_char, tracker->trigger_ref_count,
1451 is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0],
1452 is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1],
1453 is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2],
1454 tracker->job);
1455 } else {
1456 _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n",
1457 tracker_type, tracker->point, state_char,
1458 tracker->job);
1459 }
1460#endif
1461}
1462
1463void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx)
1464{
1465 struct mali_timeline_tracker *tracker = NULL;
1466
1467 MALI_DEBUG_ASSERT_POINTER(timeline);
1468
1469 tracker = timeline->tracker_tail;
1470 while (NULL != tracker) {
1471 mali_timeline_debug_print_tracker(tracker, print_ctx);
1472 tracker = tracker->timeline_next;
1473 }
1474}
1475
1476void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx)
1477{
1478 int i;
1479 int num_printed = 0;
1480 u32 tid = _mali_osk_get_tid();
1481
1482 MALI_DEBUG_ASSERT_POINTER(system);
1483
1484 mali_spinlock_reentrant_wait(system->spinlock, tid);
1485
1486
1487 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1488 struct mali_timeline *timeline = system->timelines[i];
1489
1490 MALI_DEBUG_ASSERT_POINTER(timeline);
1491
1492 if (NULL == timeline->tracker_head) continue;
1493
1494 _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n",
1495 timeline_id_to_string((enum mali_timeline_id)i));
1496
1497 mali_timeline_debug_print_timeline(timeline, print_ctx);
1498 num_printed++;
1499 }
1500
1501 if (0 == num_printed) {
1502 _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n");
1503 }
1504
1505 mali_spinlock_reentrant_signal(system->spinlock, tid);
1506}
1507
1508#endif
1509