1
2
3
4#include <linux/delay.h>
5#include <linux/interrupt.h>
6#include <linux/io.h>
7#include <linux/iopoll.h>
8#include <linux/platform_device.h>
9#include <linux/pm_runtime.h>
10#include <linux/dma-resv.h>
11#include <drm/gpu_scheduler.h>
12#include <drm/panfrost_drm.h>
13
14#include "panfrost_device.h"
15#include "panfrost_devfreq.h"
16#include "panfrost_job.h"
17#include "panfrost_features.h"
18#include "panfrost_issues.h"
19#include "panfrost_gem.h"
20#include "panfrost_regs.h"
21#include "panfrost_gpu.h"
22#include "panfrost_mmu.h"
23
24#define JOB_TIMEOUT_MS 500
25
26#define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
27#define job_read(dev, reg) readl(dev->iomem + (reg))
28
29struct panfrost_queue_state {
30 struct drm_gpu_scheduler sched;
31 u64 fence_context;
32 u64 emit_seqno;
33};
34
35struct panfrost_job_slot {
36 struct panfrost_queue_state queue[NUM_JOB_SLOTS];
37 spinlock_t job_lock;
38 int irq;
39};
40
41static struct panfrost_job *
42to_panfrost_job(struct drm_sched_job *sched_job)
43{
44 return container_of(sched_job, struct panfrost_job, base);
45}
46
47struct panfrost_fence {
48 struct dma_fence base;
49 struct drm_device *dev;
50
51 u64 seqno;
52 int queue;
53};
54
55static inline struct panfrost_fence *
56to_panfrost_fence(struct dma_fence *fence)
57{
58 return (struct panfrost_fence *)fence;
59}
60
61static const char *panfrost_fence_get_driver_name(struct dma_fence *fence)
62{
63 return "panfrost";
64}
65
66static const char *panfrost_fence_get_timeline_name(struct dma_fence *fence)
67{
68 struct panfrost_fence *f = to_panfrost_fence(fence);
69
70 switch (f->queue) {
71 case 0:
72 return "panfrost-js-0";
73 case 1:
74 return "panfrost-js-1";
75 case 2:
76 return "panfrost-js-2";
77 default:
78 return NULL;
79 }
80}
81
82static const struct dma_fence_ops panfrost_fence_ops = {
83 .get_driver_name = panfrost_fence_get_driver_name,
84 .get_timeline_name = panfrost_fence_get_timeline_name,
85};
86
87static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)
88{
89 struct panfrost_fence *fence;
90 struct panfrost_job_slot *js = pfdev->js;
91
92 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
93 if (!fence)
94 return ERR_PTR(-ENOMEM);
95
96 fence->dev = pfdev->ddev;
97 fence->queue = js_num;
98 fence->seqno = ++js->queue[js_num].emit_seqno;
99 dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,
100 js->queue[js_num].fence_context, fence->seqno);
101
102 return &fence->base;
103}
104
105static int panfrost_job_get_slot(struct panfrost_job *job)
106{
107
108
109
110
111 if (job->requirements & PANFROST_JD_REQ_FS)
112 return 0;
113
114
115#if 0
116 if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {
117 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&
118 (job->pfdev->features.nr_core_groups == 2))
119 return 2;
120 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
121 return 2;
122 }
123#endif
124 return 1;
125}
126
127static void panfrost_job_write_affinity(struct panfrost_device *pfdev,
128 u32 requirements,
129 int js)
130{
131 u64 affinity;
132
133
134
135
136
137
138 affinity = pfdev->features.shader_present;
139
140 job_write(pfdev, JS_AFFINITY_NEXT_LO(js), affinity & 0xFFFFFFFF);
141 job_write(pfdev, JS_AFFINITY_NEXT_HI(js), affinity >> 32);
142}
143
144static u32
145panfrost_get_job_chain_flag(const struct panfrost_job *job)
146{
147 struct panfrost_fence *f = to_panfrost_fence(job->done_fence);
148
149 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
150 return 0;
151
152 return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0;
153}
154
155static struct panfrost_job *
156panfrost_dequeue_job(struct panfrost_device *pfdev, int slot)
157{
158 struct panfrost_job *job = pfdev->jobs[slot][0];
159
160 WARN_ON(!job);
161 pfdev->jobs[slot][0] = pfdev->jobs[slot][1];
162 pfdev->jobs[slot][1] = NULL;
163
164 return job;
165}
166
167static unsigned int
168panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,
169 struct panfrost_job *job)
170{
171 if (WARN_ON(!job))
172 return 0;
173
174 if (!pfdev->jobs[slot][0]) {
175 pfdev->jobs[slot][0] = job;
176 return 0;
177 }
178
179 WARN_ON(pfdev->jobs[slot][1]);
180 pfdev->jobs[slot][1] = job;
181 WARN_ON(panfrost_get_job_chain_flag(job) ==
182 panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));
183 return 1;
184}
185
186static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
187{
188 struct panfrost_device *pfdev = job->pfdev;
189 unsigned int subslot;
190 u32 cfg;
191 u64 jc_head = job->jc;
192 int ret;
193
194 panfrost_devfreq_record_busy(&pfdev->pfdevfreq);
195
196 ret = pm_runtime_get_sync(pfdev->dev);
197 if (ret < 0)
198 return;
199
200 if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {
201 return;
202 }
203
204 cfg = panfrost_mmu_as_get(pfdev, job->file_priv->mmu);
205
206 job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF);
207 job_write(pfdev, JS_HEAD_NEXT_HI(js), jc_head >> 32);
208
209 panfrost_job_write_affinity(pfdev, job->requirements, js);
210
211
212
213 cfg |= JS_CONFIG_THREAD_PRI(8) |
214 JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |
215 JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE |
216 panfrost_get_job_chain_flag(job);
217
218 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
219 cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
220
221 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))
222 cfg |= JS_CONFIG_START_MMU;
223
224 job_write(pfdev, JS_CONFIG_NEXT(js), cfg);
225
226 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
227 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
228
229
230
231 spin_lock(&pfdev->js->job_lock);
232 subslot = panfrost_enqueue_job(pfdev, js, job);
233
234 if (!atomic_read(&pfdev->reset.pending)) {
235 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
236 dev_dbg(pfdev->dev,
237 "JS: Submitting atom %p to js[%d][%d] with head=0x%llx AS %d",
238 job, js, subslot, jc_head, cfg & 0xf);
239 }
240 spin_unlock(&pfdev->js->job_lock);
241}
242
243static int panfrost_acquire_object_fences(struct drm_gem_object **bos,
244 int bo_count,
245 struct xarray *deps)
246{
247 int i, ret;
248
249 for (i = 0; i < bo_count; i++) {
250
251 ret = drm_gem_fence_array_add_implicit(deps, bos[i], true);
252 if (ret)
253 return ret;
254 }
255
256 return 0;
257}
258
259static void panfrost_attach_object_fences(struct drm_gem_object **bos,
260 int bo_count,
261 struct dma_fence *fence)
262{
263 int i;
264
265 for (i = 0; i < bo_count; i++)
266 dma_resv_add_excl_fence(bos[i]->resv, fence);
267}
268
269int panfrost_job_push(struct panfrost_job *job)
270{
271 struct panfrost_device *pfdev = job->pfdev;
272 int slot = panfrost_job_get_slot(job);
273 struct drm_sched_entity *entity = &job->file_priv->sched_entity[slot];
274 struct ww_acquire_ctx acquire_ctx;
275 int ret = 0;
276
277
278 ret = drm_gem_lock_reservations(job->bos, job->bo_count,
279 &acquire_ctx);
280 if (ret)
281 return ret;
282
283 mutex_lock(&pfdev->sched_lock);
284
285 ret = drm_sched_job_init(&job->base, entity, NULL);
286 if (ret) {
287 mutex_unlock(&pfdev->sched_lock);
288 goto unlock;
289 }
290
291 job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);
292
293 ret = panfrost_acquire_object_fences(job->bos, job->bo_count,
294 &job->deps);
295 if (ret) {
296 mutex_unlock(&pfdev->sched_lock);
297 goto unlock;
298 }
299
300 kref_get(&job->refcount);
301
302 drm_sched_entity_push_job(&job->base, entity);
303
304 mutex_unlock(&pfdev->sched_lock);
305
306 panfrost_attach_object_fences(job->bos, job->bo_count,
307 job->render_done_fence);
308
309unlock:
310 drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);
311
312 return ret;
313}
314
315static void panfrost_job_cleanup(struct kref *ref)
316{
317 struct panfrost_job *job = container_of(ref, struct panfrost_job,
318 refcount);
319 struct dma_fence *fence;
320 unsigned long index;
321 unsigned int i;
322
323 xa_for_each(&job->deps, index, fence) {
324 dma_fence_put(fence);
325 }
326 xa_destroy(&job->deps);
327
328 dma_fence_put(job->done_fence);
329 dma_fence_put(job->render_done_fence);
330
331 if (job->mappings) {
332 for (i = 0; i < job->bo_count; i++) {
333 if (!job->mappings[i])
334 break;
335
336 atomic_dec(&job->mappings[i]->obj->gpu_usecount);
337 panfrost_gem_mapping_put(job->mappings[i]);
338 }
339 kvfree(job->mappings);
340 }
341
342 if (job->bos) {
343 for (i = 0; i < job->bo_count; i++)
344 drm_gem_object_put(job->bos[i]);
345
346 kvfree(job->bos);
347 }
348
349 kfree(job);
350}
351
352void panfrost_job_put(struct panfrost_job *job)
353{
354 kref_put(&job->refcount, panfrost_job_cleanup);
355}
356
357static void panfrost_job_free(struct drm_sched_job *sched_job)
358{
359 struct panfrost_job *job = to_panfrost_job(sched_job);
360
361 drm_sched_job_cleanup(sched_job);
362
363 panfrost_job_put(job);
364}
365
366static struct dma_fence *panfrost_job_dependency(struct drm_sched_job *sched_job,
367 struct drm_sched_entity *s_entity)
368{
369 struct panfrost_job *job = to_panfrost_job(sched_job);
370
371 if (!xa_empty(&job->deps))
372 return xa_erase(&job->deps, job->last_dep++);
373
374 return NULL;
375}
376
377static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job)
378{
379 struct panfrost_job *job = to_panfrost_job(sched_job);
380 struct panfrost_device *pfdev = job->pfdev;
381 int slot = panfrost_job_get_slot(job);
382 struct dma_fence *fence = NULL;
383
384 if (unlikely(job->base.s_fence->finished.error))
385 return NULL;
386
387
388
389
390 if (!job->jc)
391 return NULL;
392
393 fence = panfrost_fence_create(pfdev, slot);
394 if (IS_ERR(fence))
395 return fence;
396
397 if (job->done_fence)
398 dma_fence_put(job->done_fence);
399 job->done_fence = dma_fence_get(fence);
400
401 panfrost_job_hw_submit(job, slot);
402
403 return fence;
404}
405
406void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
407{
408 int j;
409 u32 irq_mask = 0;
410
411 for (j = 0; j < NUM_JOB_SLOTS; j++) {
412 irq_mask |= MK_JS_MASK(j);
413 }
414
415 job_write(pfdev, JOB_INT_CLEAR, irq_mask);
416 job_write(pfdev, JOB_INT_MASK, irq_mask);
417}
418
419static void panfrost_job_handle_err(struct panfrost_device *pfdev,
420 struct panfrost_job *job,
421 unsigned int js)
422{
423 u32 js_status = job_read(pfdev, JS_STATUS(js));
424 const char *exception_name = panfrost_exception_name(js_status);
425 bool signal_fence = true;
426
427 if (!panfrost_exception_is_fault(js_status)) {
428 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",
429 js, exception_name,
430 job_read(pfdev, JS_HEAD_LO(js)),
431 job_read(pfdev, JS_TAIL_LO(js)));
432 } else {
433 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
434 js, exception_name,
435 job_read(pfdev, JS_HEAD_LO(js)),
436 job_read(pfdev, JS_TAIL_LO(js)));
437 }
438
439 if (js_status == DRM_PANFROST_EXCEPTION_STOPPED) {
440
441 job->jc = job_read(pfdev, JS_TAIL_LO(js)) |
442 ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);
443
444
445 signal_fence = false;
446 } else if (js_status == DRM_PANFROST_EXCEPTION_TERMINATED) {
447
448 dma_fence_set_error(job->done_fence, -ECANCELED);
449 job->jc = 0;
450 } else if (panfrost_exception_is_fault(js_status)) {
451
452
453
454
455 dma_fence_set_error(job->done_fence, -EINVAL);
456 job->jc = 0;
457 }
458
459 panfrost_mmu_as_put(pfdev, job->file_priv->mmu);
460 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
461
462 if (signal_fence)
463 dma_fence_signal_locked(job->done_fence);
464
465 pm_runtime_put_autosuspend(pfdev->dev);
466
467 if (panfrost_exception_needs_reset(pfdev, js_status)) {
468 atomic_set(&pfdev->reset.pending, 1);
469 drm_sched_fault(&pfdev->js->queue[js].sched);
470 }
471}
472
473static void panfrost_job_handle_done(struct panfrost_device *pfdev,
474 struct panfrost_job *job)
475{
476
477
478
479 job->jc = 0;
480 panfrost_mmu_as_put(pfdev, job->file_priv->mmu);
481 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
482
483 dma_fence_signal_locked(job->done_fence);
484 pm_runtime_put_autosuspend(pfdev->dev);
485}
486
487static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status)
488{
489 struct panfrost_job *done[NUM_JOB_SLOTS][2] = {};
490 struct panfrost_job *failed[NUM_JOB_SLOTS] = {};
491 u32 js_state = 0, js_events = 0;
492 unsigned int i, j;
493
494
495 while (status) {
496 u32 js_state_mask = 0;
497
498 for (j = 0; j < NUM_JOB_SLOTS; j++) {
499 if (status & MK_JS_MASK(j))
500 js_state_mask |= MK_JS_MASK(j);
501
502 if (status & JOB_INT_MASK_DONE(j)) {
503 if (done[j][0])
504 done[j][1] = panfrost_dequeue_job(pfdev, j);
505 else
506 done[j][0] = panfrost_dequeue_job(pfdev, j);
507 }
508
509 if (status & JOB_INT_MASK_ERR(j)) {
510
511
512
513
514 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
515 failed[j] = panfrost_dequeue_job(pfdev, j);
516 }
517 }
518
519
520
521
522
523
524
525
526
527
528
529
530
531 job_write(pfdev, JOB_INT_CLEAR, status);
532 js_state &= ~js_state_mask;
533 js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;
534 js_events |= status;
535 status = job_read(pfdev, JOB_INT_RAWSTAT);
536 }
537
538
539 for (j = 0; j < NUM_JOB_SLOTS; j++) {
540 if (!(js_events & MK_JS_MASK(j)))
541 continue;
542
543 if (failed[j]) {
544 panfrost_job_handle_err(pfdev, failed[j], j);
545 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {
546
547
548
549
550
551
552
553
554
555 if (WARN_ON(!done[j][0]))
556 done[j][0] = panfrost_dequeue_job(pfdev, j);
557 else
558 done[j][1] = panfrost_dequeue_job(pfdev, j);
559 }
560
561 for (i = 0; i < ARRAY_SIZE(done[0]) && done[j][i]; i++)
562 panfrost_job_handle_done(pfdev, done[j][i]);
563 }
564
565
566
567
568 for (j = 0; j < NUM_JOB_SLOTS; j++) {
569 if (!(js_events & MK_JS_MASK(j)))
570 continue;
571
572 if (!failed[j] || !pfdev->jobs[j][0])
573 continue;
574
575 if (pfdev->jobs[j][0]->jc == 0) {
576
577 struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);
578
579 dma_fence_set_error(canceled->done_fence, -ECANCELED);
580 panfrost_job_handle_done(pfdev, canceled);
581 } else if (!atomic_read(&pfdev->reset.pending)) {
582
583 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);
584 }
585 }
586}
587
588static void panfrost_job_handle_irqs(struct panfrost_device *pfdev)
589{
590 u32 status = job_read(pfdev, JOB_INT_RAWSTAT);
591
592 while (status) {
593 pm_runtime_mark_last_busy(pfdev->dev);
594
595 spin_lock(&pfdev->js->job_lock);
596 panfrost_job_handle_irq(pfdev, status);
597 spin_unlock(&pfdev->js->job_lock);
598 status = job_read(pfdev, JOB_INT_RAWSTAT);
599 }
600}
601
602static u32 panfrost_active_slots(struct panfrost_device *pfdev,
603 u32 *js_state_mask, u32 js_state)
604{
605 u32 rawstat;
606
607 if (!(js_state & *js_state_mask))
608 return 0;
609
610 rawstat = job_read(pfdev, JOB_INT_RAWSTAT);
611 if (rawstat) {
612 unsigned int i;
613
614 for (i = 0; i < NUM_JOB_SLOTS; i++) {
615 if (rawstat & MK_JS_MASK(i))
616 *js_state_mask &= ~MK_JS_MASK(i);
617 }
618 }
619
620 return js_state & *js_state_mask;
621}
622
623static void
624panfrost_reset(struct panfrost_device *pfdev,
625 struct drm_sched_job *bad)
626{
627 u32 js_state, js_state_mask = 0xffffffff;
628 unsigned int i, j;
629 bool cookie;
630 int ret;
631
632 if (!atomic_read(&pfdev->reset.pending))
633 return;
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648 for (i = 0; i < NUM_JOB_SLOTS; i++)
649 drm_sched_stop(&pfdev->js->queue[i].sched, bad);
650
651 cookie = dma_fence_begin_signalling();
652
653 if (bad)
654 drm_sched_increase_karma(bad);
655
656
657
658
659 job_write(pfdev, JOB_INT_MASK, 0);
660 synchronize_irq(pfdev->js->irq);
661
662 for (i = 0; i < NUM_JOB_SLOTS; i++) {
663
664 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
665 job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);
666 }
667
668
669 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,
670 !panfrost_active_slots(pfdev, &js_state_mask, js_state),
671 10, 10000);
672
673 if (ret)
674 dev_err(pfdev->dev, "Soft-stop failed\n");
675
676
677 panfrost_job_handle_irqs(pfdev);
678
679
680
681
682
683
684 spin_lock(&pfdev->js->job_lock);
685 for (i = 0; i < NUM_JOB_SLOTS; i++) {
686 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {
687 pm_runtime_put_noidle(pfdev->dev);
688 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
689 }
690 }
691 memset(pfdev->jobs, 0, sizeof(pfdev->jobs));
692 spin_unlock(&pfdev->js->job_lock);
693
694
695 panfrost_device_reset(pfdev);
696
697
698
699
700 job_write(pfdev, JOB_INT_MASK, 0);
701
702
703 atomic_set(&pfdev->reset.pending, 0);
704
705
706
707
708
709
710
711
712 dma_fence_end_signalling(cookie);
713 for (i = 0; i < NUM_JOB_SLOTS; i++)
714 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
715 cookie = dma_fence_begin_signalling();
716
717
718 for (i = 0; i < NUM_JOB_SLOTS; i++)
719 drm_sched_start(&pfdev->js->queue[i].sched, true);
720
721
722 job_write(pfdev, JOB_INT_MASK,
723 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
724 GENMASK(NUM_JOB_SLOTS - 1, 0));
725
726 dma_fence_end_signalling(cookie);
727}
728
729static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job
730 *sched_job)
731{
732 struct panfrost_job *job = to_panfrost_job(sched_job);
733 struct panfrost_device *pfdev = job->pfdev;
734 int js = panfrost_job_get_slot(job);
735
736
737
738
739
740 if (dma_fence_is_signaled(job->done_fence))
741 return DRM_GPU_SCHED_STAT_NOMINAL;
742
743 dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
744 js,
745 job_read(pfdev, JS_CONFIG(js)),
746 job_read(pfdev, JS_STATUS(js)),
747 job_read(pfdev, JS_HEAD_LO(js)),
748 job_read(pfdev, JS_TAIL_LO(js)),
749 sched_job);
750
751 atomic_set(&pfdev->reset.pending, 1);
752 panfrost_reset(pfdev, sched_job);
753
754 return DRM_GPU_SCHED_STAT_NOMINAL;
755}
756
757static void panfrost_reset_work(struct work_struct *work)
758{
759 struct panfrost_device *pfdev;
760
761 pfdev = container_of(work, struct panfrost_device, reset.work);
762 panfrost_reset(pfdev, NULL);
763}
764
765static const struct drm_sched_backend_ops panfrost_sched_ops = {
766 .dependency = panfrost_job_dependency,
767 .run_job = panfrost_job_run,
768 .timedout_job = panfrost_job_timedout,
769 .free_job = panfrost_job_free
770};
771
772static irqreturn_t panfrost_job_irq_handler_thread(int irq, void *data)
773{
774 struct panfrost_device *pfdev = data;
775
776 panfrost_job_handle_irqs(pfdev);
777 job_write(pfdev, JOB_INT_MASK,
778 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
779 GENMASK(NUM_JOB_SLOTS - 1, 0));
780 return IRQ_HANDLED;
781}
782
783static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
784{
785 struct panfrost_device *pfdev = data;
786 u32 status = job_read(pfdev, JOB_INT_STAT);
787
788 if (!status)
789 return IRQ_NONE;
790
791 job_write(pfdev, JOB_INT_MASK, 0);
792 return IRQ_WAKE_THREAD;
793}
794
795int panfrost_job_init(struct panfrost_device *pfdev)
796{
797 struct panfrost_job_slot *js;
798 unsigned int nentries = 2;
799 int ret, j;
800
801
802
803
804
805 if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
806 nentries = 1;
807
808 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
809 if (!js)
810 return -ENOMEM;
811
812 INIT_WORK(&pfdev->reset.work, panfrost_reset_work);
813 spin_lock_init(&js->job_lock);
814
815 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
816 if (js->irq <= 0)
817 return -ENODEV;
818
819 ret = devm_request_threaded_irq(pfdev->dev, js->irq,
820 panfrost_job_irq_handler,
821 panfrost_job_irq_handler_thread,
822 IRQF_SHARED, KBUILD_MODNAME "-job",
823 pfdev);
824 if (ret) {
825 dev_err(pfdev->dev, "failed to request job irq");
826 return ret;
827 }
828
829 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);
830 if (!pfdev->reset.wq)
831 return -ENOMEM;
832
833 for (j = 0; j < NUM_JOB_SLOTS; j++) {
834 js->queue[j].fence_context = dma_fence_context_alloc(1);
835
836 ret = drm_sched_init(&js->queue[j].sched,
837 &panfrost_sched_ops,
838 nentries, 0,
839 msecs_to_jiffies(JOB_TIMEOUT_MS),
840 pfdev->reset.wq,
841 NULL, "pan_js");
842 if (ret) {
843 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
844 goto err_sched;
845 }
846 }
847
848 panfrost_job_enable_interrupts(pfdev);
849
850 return 0;
851
852err_sched:
853 for (j--; j >= 0; j--)
854 drm_sched_fini(&js->queue[j].sched);
855
856 destroy_workqueue(pfdev->reset.wq);
857 return ret;
858}
859
860void panfrost_job_fini(struct panfrost_device *pfdev)
861{
862 struct panfrost_job_slot *js = pfdev->js;
863 int j;
864
865 job_write(pfdev, JOB_INT_MASK, 0);
866
867 for (j = 0; j < NUM_JOB_SLOTS; j++) {
868 drm_sched_fini(&js->queue[j].sched);
869 }
870
871 cancel_work_sync(&pfdev->reset.work);
872 destroy_workqueue(pfdev->reset.wq);
873}
874
875int panfrost_job_open(struct panfrost_file_priv *panfrost_priv)
876{
877 struct panfrost_device *pfdev = panfrost_priv->pfdev;
878 struct panfrost_job_slot *js = pfdev->js;
879 struct drm_gpu_scheduler *sched;
880 int ret, i;
881
882 for (i = 0; i < NUM_JOB_SLOTS; i++) {
883 sched = &js->queue[i].sched;
884 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i],
885 DRM_SCHED_PRIORITY_NORMAL, &sched,
886 1, NULL);
887 if (WARN_ON(ret))
888 return ret;
889 }
890 return 0;
891}
892
893void panfrost_job_close(struct panfrost_file_priv *panfrost_priv)
894{
895 struct panfrost_device *pfdev = panfrost_priv->pfdev;
896 int i;
897
898 for (i = 0; i < NUM_JOB_SLOTS; i++)
899 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);
900
901
902 spin_lock(&pfdev->js->job_lock);
903 for (i = 0; i < NUM_JOB_SLOTS; i++) {
904 struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i];
905 int j;
906
907 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {
908 struct panfrost_job *job = pfdev->jobs[i][j];
909 u32 cmd;
910
911 if (!job || job->base.entity != entity)
912 continue;
913
914 if (j == 1) {
915
916 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
917
918
919
920 job->jc = 0;
921 }
922
923 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
924 cmd = panfrost_get_job_chain_flag(job) ?
925 JS_COMMAND_HARD_STOP_1 :
926 JS_COMMAND_HARD_STOP_0;
927 } else {
928 cmd = JS_COMMAND_HARD_STOP;
929 }
930
931 job_write(pfdev, JS_COMMAND(i), cmd);
932 }
933 }
934 spin_unlock(&pfdev->js->job_lock);
935}
936
937int panfrost_job_is_idle(struct panfrost_device *pfdev)
938{
939 struct panfrost_job_slot *js = pfdev->js;
940 int i;
941
942 for (i = 0; i < NUM_JOB_SLOTS; i++) {
943
944 if (atomic_read(&js->queue[i].sched.hw_rq_count))
945 return false;
946 }
947
948 return true;
949}
950