1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/seq_file.h>
30#include <linux/slab.h>
31#include <drm/drmP.h>
32#include <drm/radeon_drm.h>
33#include "radeon_reg.h"
34#include "radeon.h"
35#include "atom.h"
36
37
38
39
40
41
42
43
44
45
46static int radeon_debugfs_sa_init(struct radeon_device *rdev);
47
48
49
50
51
52
53
54
55
56
57
58
59
60int radeon_ib_get(struct radeon_device *rdev, int ring,
61 struct radeon_ib *ib, struct radeon_vm *vm,
62 unsigned size)
63{
64 int i, r;
65
66 r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true);
67 if (r) {
68 dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
69 return r;
70 }
71
72 r = radeon_semaphore_create(rdev, &ib->semaphore);
73 if (r) {
74 return r;
75 }
76
77 ib->ring = ring;
78 ib->fence = NULL;
79 ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo);
80 ib->vm = vm;
81 if (vm) {
82
83
84
85 ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET;
86 } else {
87 ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
88 }
89 ib->is_const_ib = false;
90 for (i = 0; i < RADEON_NUM_RINGS; ++i)
91 ib->sync_to[i] = NULL;
92
93 return 0;
94}
95
96
97
98
99
100
101
102
103
104void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib)
105{
106 radeon_semaphore_free(rdev, &ib->semaphore, ib->fence);
107 radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence);
108 radeon_fence_unref(&ib->fence);
109}
110
111
112
113
114
115
116
117
118
119void radeon_ib_sync_to(struct radeon_ib *ib, struct radeon_fence *fence)
120{
121 struct radeon_fence *other;
122
123 if (!fence)
124 return;
125
126 other = ib->sync_to[fence->ring];
127 ib->sync_to[fence->ring] = radeon_fence_later(fence, other);
128}
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
151 struct radeon_ib *const_ib)
152{
153 struct radeon_ring *ring = &rdev->ring[ib->ring];
154 bool need_sync = false;
155 int i, r = 0;
156
157 if (!ib->length_dw || !ring->ready) {
158
159 dev_err(rdev->dev, "couldn't schedule ib\n");
160 return -EINVAL;
161 }
162
163
164 r = radeon_ring_lock(rdev, ring, 64 + RADEON_NUM_RINGS * 8);
165 if (r) {
166 dev_err(rdev->dev, "scheduling IB failed (%d).\n", r);
167 return r;
168 }
169 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
170 struct radeon_fence *fence = ib->sync_to[i];
171 if (radeon_fence_need_sync(fence, ib->ring)) {
172 need_sync = true;
173 radeon_semaphore_sync_rings(rdev, ib->semaphore,
174 fence->ring, ib->ring);
175 radeon_fence_note_sync(fence, ib->ring);
176 }
177 }
178
179 if (!need_sync) {
180 radeon_semaphore_free(rdev, &ib->semaphore, NULL);
181 }
182
183
184 if (ib->vm ) {
185 radeon_ring_vm_flush(rdev, ib->ring, ib->vm);
186 }
187 if (const_ib) {
188 radeon_ring_ib_execute(rdev, const_ib->ring, const_ib);
189 radeon_semaphore_free(rdev, &const_ib->semaphore, NULL);
190 }
191 radeon_ring_ib_execute(rdev, ib->ring, ib);
192 r = radeon_fence_emit(rdev, &ib->fence, ib->ring);
193 if (r) {
194 dev_err(rdev->dev, "failed to emit fence for new IB (%d)\n", r);
195 radeon_ring_unlock_undo(rdev, ring);
196 return r;
197 }
198 if (const_ib) {
199 const_ib->fence = radeon_fence_ref(ib->fence);
200 }
201
202 if (ib->vm && !ib->vm->last_flush) {
203 ib->vm->last_flush = radeon_fence_ref(ib->fence);
204 }
205 radeon_ring_unlock_commit(rdev, ring);
206 return 0;
207}
208
209
210
211
212
213
214
215
216
217
218int radeon_ib_pool_init(struct radeon_device *rdev)
219{
220 int r;
221
222 if (rdev->ib_pool_ready) {
223 return 0;
224 }
225 r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo,
226 RADEON_IB_POOL_SIZE*64*1024,
227 RADEON_GPU_PAGE_SIZE,
228 RADEON_GEM_DOMAIN_GTT);
229 if (r) {
230 return r;
231 }
232
233 r = radeon_sa_bo_manager_start(rdev, &rdev->ring_tmp_bo);
234 if (r) {
235 return r;
236 }
237
238 rdev->ib_pool_ready = true;
239 if (radeon_debugfs_sa_init(rdev)) {
240 dev_err(rdev->dev, "failed to register debugfs file for SA\n");
241 }
242 return 0;
243}
244
245
246
247
248
249
250
251
252
253void radeon_ib_pool_fini(struct radeon_device *rdev)
254{
255 if (rdev->ib_pool_ready) {
256 radeon_sa_bo_manager_suspend(rdev, &rdev->ring_tmp_bo);
257 radeon_sa_bo_manager_fini(rdev, &rdev->ring_tmp_bo);
258 rdev->ib_pool_ready = false;
259 }
260}
261
262
263
264
265
266
267
268
269
270
271
272int radeon_ib_ring_tests(struct radeon_device *rdev)
273{
274 unsigned i;
275 int r;
276
277 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
278 struct radeon_ring *ring = &rdev->ring[i];
279
280 if (!ring->ready)
281 continue;
282
283 r = radeon_ib_test(rdev, i, ring);
284 if (r) {
285 ring->ready = false;
286
287 if (i == RADEON_RING_TYPE_GFX_INDEX) {
288
289 DRM_ERROR("radeon: failed testing IB on GFX ring (%d).\n", r);
290 rdev->accel_working = false;
291 return r;
292
293 } else {
294
295 DRM_ERROR("radeon: failed testing IB on ring %d (%d).\n", i, r);
296 }
297 }
298 }
299 return 0;
300}
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
316
317
318
319
320
321
322
323
324
325void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
326{
327#if DRM_DEBUG_CODE
328 if (ring->count_dw <= 0) {
329 DRM_ERROR("radeon: writing more dwords to the ring than expected!\n");
330 }
331#endif
332 ring->ring[ring->wptr++] = v;
333 ring->wptr &= ring->ptr_mask;
334 ring->count_dw--;
335 ring->ring_free_dw--;
336}
337
338
339
340
341
342
343
344
345
346
347
348bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
349 struct radeon_ring *ring)
350{
351 switch (ring->idx) {
352 case RADEON_RING_TYPE_GFX_INDEX:
353 case CAYMAN_RING_TYPE_CP1_INDEX:
354 case CAYMAN_RING_TYPE_CP2_INDEX:
355 return true;
356 default:
357 return false;
358 }
359}
360
361u32 radeon_ring_generic_get_rptr(struct radeon_device *rdev,
362 struct radeon_ring *ring)
363{
364 u32 rptr;
365
366 if (rdev->wb.enabled)
367 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
368 else
369 rptr = RREG32(ring->rptr_reg);
370
371 return rptr;
372}
373
374u32 radeon_ring_generic_get_wptr(struct radeon_device *rdev,
375 struct radeon_ring *ring)
376{
377 u32 wptr;
378
379 wptr = RREG32(ring->wptr_reg);
380
381 return wptr;
382}
383
384void radeon_ring_generic_set_wptr(struct radeon_device *rdev,
385 struct radeon_ring *ring)
386{
387 WREG32(ring->wptr_reg, ring->wptr);
388 (void)RREG32(ring->wptr_reg);
389}
390
391
392
393
394
395
396
397
398
399void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
400{
401 ring->rptr = radeon_ring_get_rptr(rdev, ring);
402
403 ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
404 ring->ring_free_dw -= ring->wptr;
405 ring->ring_free_dw &= ring->ptr_mask;
406 if (!ring->ring_free_dw) {
407 ring->ring_free_dw = ring->ring_size / 4;
408 }
409}
410
411
412
413
414
415
416
417
418
419
420
421int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
422{
423 int r;
424
425
426 if (ndw > (ring->ring_size / 4))
427 return -ENOMEM;
428
429
430 radeon_ring_free_size(rdev, ring);
431 if (ring->ring_free_dw == (ring->ring_size / 4)) {
432
433
434
435 radeon_ring_lockup_update(ring);
436 }
437 ndw = (ndw + ring->align_mask) & ~ring->align_mask;
438 while (ndw > (ring->ring_free_dw - 1)) {
439 radeon_ring_free_size(rdev, ring);
440 if (ndw < ring->ring_free_dw) {
441 break;
442 }
443 r = radeon_fence_wait_next_locked(rdev, ring->idx);
444 if (r)
445 return r;
446 }
447 ring->count_dw = ndw;
448 ring->wptr_old = ring->wptr;
449 return 0;
450}
451
452
453
454
455
456
457
458
459
460
461
462
463int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
464{
465 int r;
466
467 mutex_lock(&rdev->ring_lock);
468 r = radeon_ring_alloc(rdev, ring, ndw);
469 if (r) {
470 mutex_unlock(&rdev->ring_lock);
471 return r;
472 }
473 return 0;
474}
475
476
477
478
479
480
481
482
483
484
485
486void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring)
487{
488
489 while (ring->wptr & ring->align_mask) {
490 radeon_ring_write(ring, ring->nop);
491 }
492 DRM_MEMORYBARRIER();
493 radeon_ring_set_wptr(rdev, ring);
494}
495
496
497
498
499
500
501
502
503
504
505void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring)
506{
507 radeon_ring_commit(rdev, ring);
508 mutex_unlock(&rdev->ring_lock);
509}
510
511
512
513
514
515
516
517
518void radeon_ring_undo(struct radeon_ring *ring)
519{
520 ring->wptr = ring->wptr_old;
521}
522
523
524
525
526
527
528
529
530void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
531{
532 radeon_ring_undo(ring);
533 mutex_unlock(&rdev->ring_lock);
534}
535
536
537
538
539
540
541
542
543
544
545void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring)
546{
547 int r;
548
549 radeon_ring_free_size(rdev, ring);
550 if (ring->rptr == ring->wptr) {
551 r = radeon_ring_alloc(rdev, ring, 1);
552 if (!r) {
553 radeon_ring_write(ring, ring->nop);
554 radeon_ring_commit(rdev, ring);
555 }
556 }
557}
558
559
560
561
562
563
564
565
566void radeon_ring_lockup_update(struct radeon_ring *ring)
567{
568 ring->last_rptr = ring->rptr;
569 ring->last_activity = jiffies;
570}
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
593{
594 unsigned long cjiffies, elapsed;
595
596 cjiffies = jiffies;
597 if (!time_after(cjiffies, ring->last_activity)) {
598
599 radeon_ring_lockup_update(ring);
600 return false;
601 }
602 ring->rptr = radeon_ring_get_rptr(rdev, ring);
603 if (ring->rptr != ring->last_rptr) {
604
605 radeon_ring_lockup_update(ring);
606 return false;
607 }
608 elapsed = jiffies_to_msecs(cjiffies - ring->last_activity);
609 if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) {
610 dev_err(rdev->dev, "GPU lockup CP stall for more than %lumsec\n", elapsed);
611 return true;
612 }
613
614 return false;
615}
616
617
618
619
620
621
622
623
624
625unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
626 uint32_t **data)
627{
628 unsigned size, ptr, i;
629
630
631 mutex_lock(&rdev->ring_lock);
632 *data = NULL;
633
634 if (ring->ring_obj == NULL) {
635 mutex_unlock(&rdev->ring_lock);
636 return 0;
637 }
638
639
640 if (!radeon_fence_count_emitted(rdev, ring->idx)) {
641 mutex_unlock(&rdev->ring_lock);
642 return 0;
643 }
644
645
646 if (ring->rptr_save_reg)
647 ptr = RREG32(ring->rptr_save_reg);
648 else if (rdev->wb.enabled)
649 ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
650 else {
651
652 mutex_unlock(&rdev->ring_lock);
653 return 0;
654 }
655
656 size = ring->wptr + (ring->ring_size / 4);
657 size -= ptr;
658 size &= ring->ptr_mask;
659 if (size == 0) {
660 mutex_unlock(&rdev->ring_lock);
661 return 0;
662 }
663
664
665 *data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
666 if (!*data) {
667 mutex_unlock(&rdev->ring_lock);
668 return 0;
669 }
670 for (i = 0; i < size; ++i) {
671 (*data)[i] = ring->ring[ptr++];
672 ptr &= ring->ptr_mask;
673 }
674
675 mutex_unlock(&rdev->ring_lock);
676 return size;
677}
678
679
680
681
682
683
684
685
686
687
688
689int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
690 unsigned size, uint32_t *data)
691{
692 int i, r;
693
694 if (!size || !data)
695 return 0;
696
697
698 r = radeon_ring_lock(rdev, ring, size);
699 if (r)
700 return r;
701
702 for (i = 0; i < size; ++i) {
703 radeon_ring_write(ring, data[i]);
704 }
705
706 radeon_ring_unlock_commit(rdev, ring);
707 kfree(data);
708 return 0;
709}
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
726 unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg, u32 nop)
727{
728 int r;
729
730 ring->ring_size = ring_size;
731 ring->rptr_offs = rptr_offs;
732 ring->rptr_reg = rptr_reg;
733 ring->wptr_reg = wptr_reg;
734 ring->nop = nop;
735
736 if (ring->ring_obj == NULL) {
737 r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
738 RADEON_GEM_DOMAIN_GTT,
739 NULL, &ring->ring_obj);
740 if (r) {
741 dev_err(rdev->dev, "(%d) ring create failed\n", r);
742 return r;
743 }
744 r = radeon_bo_reserve(ring->ring_obj, false);
745 if (unlikely(r != 0))
746 return r;
747 r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
748 &ring->gpu_addr);
749 if (r) {
750 radeon_bo_unreserve(ring->ring_obj);
751 dev_err(rdev->dev, "(%d) ring pin failed\n", r);
752 return r;
753 }
754 r = radeon_bo_kmap(ring->ring_obj,
755 (void **)&ring->ring);
756 radeon_bo_unreserve(ring->ring_obj);
757 if (r) {
758 dev_err(rdev->dev, "(%d) ring map failed\n", r);
759 return r;
760 }
761 }
762 ring->ptr_mask = (ring->ring_size / 4) - 1;
763 ring->ring_free_dw = ring->ring_size / 4;
764 if (rdev->wb.enabled) {
765 u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);
766 ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;
767 ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];
768 }
769 if (radeon_debugfs_ring_init(rdev, ring)) {
770 DRM_ERROR("Failed to register debugfs file for rings !\n");
771 }
772 radeon_ring_lockup_update(ring);
773 return 0;
774}
775
776
777
778
779
780
781
782
783
784void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
785{
786 int r;
787 struct radeon_bo *ring_obj;
788
789 mutex_lock(&rdev->ring_lock);
790 ring_obj = ring->ring_obj;
791 ring->ready = false;
792 ring->ring = NULL;
793 ring->ring_obj = NULL;
794 mutex_unlock(&rdev->ring_lock);
795
796 if (ring_obj) {
797 r = radeon_bo_reserve(ring_obj, false);
798 if (likely(r == 0)) {
799 radeon_bo_kunmap(ring_obj);
800 radeon_bo_unpin(ring_obj);
801 radeon_bo_unreserve(ring_obj);
802 }
803 radeon_bo_unref(&ring_obj);
804 }
805}
806
807
808
809
810#if defined(CONFIG_DEBUG_FS)
811
812static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
813{
814 struct drm_info_node *node = (struct drm_info_node *) m->private;
815 struct drm_device *dev = node->minor->dev;
816 struct radeon_device *rdev = dev->dev_private;
817 int ridx = *(int*)node->info_ent->data;
818 struct radeon_ring *ring = &rdev->ring[ridx];
819 unsigned count, i, j;
820 u32 tmp;
821
822 radeon_ring_free_size(rdev, ring);
823 count = (ring->ring_size / 4) - ring->ring_free_dw;
824 tmp = radeon_ring_get_wptr(rdev, ring);
825 seq_printf(m, "wptr(0x%04x): 0x%08x [%5d]\n", ring->wptr_reg, tmp, tmp);
826 tmp = radeon_ring_get_rptr(rdev, ring);
827 seq_printf(m, "rptr(0x%04x): 0x%08x [%5d]\n", ring->rptr_reg, tmp, tmp);
828 if (ring->rptr_save_reg) {
829 seq_printf(m, "rptr next(0x%04x): 0x%08x\n", ring->rptr_save_reg,
830 RREG32(ring->rptr_save_reg));
831 }
832 seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n", ring->wptr, ring->wptr);
833 seq_printf(m, "driver's copy of the rptr: 0x%08x [%5d]\n", ring->rptr, ring->rptr);
834 seq_printf(m, "last semaphore signal addr : 0x%016llx\n", ring->last_semaphore_signal_addr);
835 seq_printf(m, "last semaphore wait addr : 0x%016llx\n", ring->last_semaphore_wait_addr);
836 seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
837 seq_printf(m, "%u dwords in ring\n", count);
838
839
840
841 i = (ring->rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
842 if (ring->ready) {
843 for (j = 0; j <= (count + 32); j++) {
844 seq_printf(m, "r[%5d]=0x%08x\n", i, ring->ring[i]);
845 i = (i + 1) & ring->ptr_mask;
846 }
847 }
848 return 0;
849}
850
851static int radeon_gfx_index = RADEON_RING_TYPE_GFX_INDEX;
852static int cayman_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX;
853static int cayman_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;
854static int radeon_dma1_index = R600_RING_TYPE_DMA_INDEX;
855static int radeon_dma2_index = CAYMAN_RING_TYPE_DMA1_INDEX;
856static int r600_uvd_index = R600_RING_TYPE_UVD_INDEX;
857
858static struct drm_info_list radeon_debugfs_ring_info_list[] = {
859 {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_gfx_index},
860 {"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_cp1_index},
861 {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_cp2_index},
862 {"radeon_ring_dma1", radeon_debugfs_ring_info, 0, &radeon_dma1_index},
863 {"radeon_ring_dma2", radeon_debugfs_ring_info, 0, &radeon_dma2_index},
864 {"radeon_ring_uvd", radeon_debugfs_ring_info, 0, &r600_uvd_index},
865};
866
867static int radeon_debugfs_sa_info(struct seq_file *m, void *data)
868{
869 struct drm_info_node *node = (struct drm_info_node *) m->private;
870 struct drm_device *dev = node->minor->dev;
871 struct radeon_device *rdev = dev->dev_private;
872
873 radeon_sa_bo_dump_debug_info(&rdev->ring_tmp_bo, m);
874
875 return 0;
876
877}
878
879static struct drm_info_list radeon_debugfs_sa_list[] = {
880 {"radeon_sa_info", &radeon_debugfs_sa_info, 0, NULL},
881};
882
883#endif
884
885static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
886{
887#if defined(CONFIG_DEBUG_FS)
888 unsigned i;
889 for (i = 0; i < ARRAY_SIZE(radeon_debugfs_ring_info_list); ++i) {
890 struct drm_info_list *info = &radeon_debugfs_ring_info_list[i];
891 int ridx = *(int*)radeon_debugfs_ring_info_list[i].data;
892 unsigned r;
893
894 if (&rdev->ring[ridx] != ring)
895 continue;
896
897 r = radeon_debugfs_add_files(rdev, info, 1);
898 if (r)
899 return r;
900 }
901#endif
902 return 0;
903}
904
905static int radeon_debugfs_sa_init(struct radeon_device *rdev)
906{
907#if defined(CONFIG_DEBUG_FS)
908 return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1);
909#else
910 return 0;
911#endif
912}
913