1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/seq_file.h>
30#include <linux/slab.h>
31#include <drm/drmP.h>
32#include <drm/radeon_drm.h>
33#include "radeon_reg.h"
34#include "radeon.h"
35#include "atom.h"
36
37
38
39
40
41
42
43
44
45
46static int radeon_debugfs_sa_init(struct radeon_device *rdev);
47
48
49
50
51
52
53
54
55
56
57
58
59
60int radeon_ib_get(struct radeon_device *rdev, int ring,
61 struct radeon_ib *ib, struct radeon_vm *vm,
62 unsigned size)
63{
64 int i, r;
65
66 r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true);
67 if (r) {
68 dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
69 return r;
70 }
71
72 r = radeon_semaphore_create(rdev, &ib->semaphore);
73 if (r) {
74 return r;
75 }
76
77 ib->ring = ring;
78 ib->fence = NULL;
79 ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo);
80 ib->vm = vm;
81 if (vm) {
82
83
84
85 ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET;
86 } else {
87 ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
88 }
89 ib->is_const_ib = false;
90 for (i = 0; i < RADEON_NUM_RINGS; ++i)
91 ib->sync_to[i] = NULL;
92
93 return 0;
94}
95
96
97
98
99
100
101
102
103
104void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib)
105{
106 radeon_semaphore_free(rdev, &ib->semaphore, ib->fence);
107 radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence);
108 radeon_fence_unref(&ib->fence);
109}
110
111
112
113
114
115
116
117
118
119void radeon_ib_sync_to(struct radeon_ib *ib, struct radeon_fence *fence)
120{
121 struct radeon_fence *other;
122
123 if (!fence)
124 return;
125
126 other = ib->sync_to[fence->ring];
127 ib->sync_to[fence->ring] = radeon_fence_later(fence, other);
128}
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
151 struct radeon_ib *const_ib)
152{
153 struct radeon_ring *ring = &rdev->ring[ib->ring];
154 bool need_sync = false;
155 int i, r = 0;
156
157 if (!ib->length_dw || !ring->ready) {
158
159 dev_err(rdev->dev, "couldn't schedule ib\n");
160 return -EINVAL;
161 }
162
163
164 r = radeon_ring_lock(rdev, ring, 64 + RADEON_NUM_RINGS * 8);
165 if (r) {
166 dev_err(rdev->dev, "scheduling IB failed (%d).\n", r);
167 return r;
168 }
169 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
170 struct radeon_fence *fence = ib->sync_to[i];
171 if (radeon_fence_need_sync(fence, ib->ring)) {
172 need_sync = true;
173 radeon_semaphore_sync_rings(rdev, ib->semaphore,
174 fence->ring, ib->ring);
175 radeon_fence_note_sync(fence, ib->ring);
176 }
177 }
178
179 if (!need_sync) {
180 radeon_semaphore_free(rdev, &ib->semaphore, NULL);
181 }
182
183
184 if (ib->vm ) {
185 radeon_ring_vm_flush(rdev, ib->ring, ib->vm);
186 }
187 if (const_ib) {
188 radeon_ring_ib_execute(rdev, const_ib->ring, const_ib);
189 radeon_semaphore_free(rdev, &const_ib->semaphore, NULL);
190 }
191 radeon_ring_ib_execute(rdev, ib->ring, ib);
192 r = radeon_fence_emit(rdev, &ib->fence, ib->ring);
193 if (r) {
194 dev_err(rdev->dev, "failed to emit fence for new IB (%d)\n", r);
195 radeon_ring_unlock_undo(rdev, ring);
196 return r;
197 }
198 if (const_ib) {
199 const_ib->fence = radeon_fence_ref(ib->fence);
200 }
201
202 if (ib->vm && !ib->vm->last_flush) {
203 ib->vm->last_flush = radeon_fence_ref(ib->fence);
204 }
205 radeon_ring_unlock_commit(rdev, ring);
206 return 0;
207}
208
209
210
211
212
213
214
215
216
217
218int radeon_ib_pool_init(struct radeon_device *rdev)
219{
220 int r;
221
222 if (rdev->ib_pool_ready) {
223 return 0;
224 }
225 r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo,
226 RADEON_IB_POOL_SIZE*64*1024,
227 RADEON_GPU_PAGE_SIZE,
228 RADEON_GEM_DOMAIN_GTT);
229 if (r) {
230 return r;
231 }
232
233 r = radeon_sa_bo_manager_start(rdev, &rdev->ring_tmp_bo);
234 if (r) {
235 return r;
236 }
237
238 rdev->ib_pool_ready = true;
239 if (radeon_debugfs_sa_init(rdev)) {
240 dev_err(rdev->dev, "failed to register debugfs file for SA\n");
241 }
242 return 0;
243}
244
245
246
247
248
249
250
251
252
253void radeon_ib_pool_fini(struct radeon_device *rdev)
254{
255 if (rdev->ib_pool_ready) {
256 radeon_sa_bo_manager_suspend(rdev, &rdev->ring_tmp_bo);
257 radeon_sa_bo_manager_fini(rdev, &rdev->ring_tmp_bo);
258 rdev->ib_pool_ready = false;
259 }
260}
261
262
263
264
265
266
267
268
269
270
271
272int radeon_ib_ring_tests(struct radeon_device *rdev)
273{
274 unsigned i;
275 int r;
276
277 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
278 struct radeon_ring *ring = &rdev->ring[i];
279
280 if (!ring->ready)
281 continue;
282
283 r = radeon_ib_test(rdev, i, ring);
284 if (r) {
285 ring->ready = false;
286
287 if (i == RADEON_RING_TYPE_GFX_INDEX) {
288
289 DRM_ERROR("radeon: failed testing IB on GFX ring (%d).\n", r);
290 rdev->accel_working = false;
291 return r;
292
293 } else {
294
295 DRM_ERROR("radeon: failed testing IB on ring %d (%d).\n", i, r);
296 }
297 }
298 }
299 return 0;
300}
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
316
317
318
319
320
321
322
323
324
325void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
326{
327#if DRM_DEBUG_CODE
328 if (ring->count_dw <= 0) {
329 DRM_ERROR("radeon: writing more dwords to the ring than expected!\n");
330 }
331#endif
332 ring->ring[ring->wptr++] = v;
333 ring->wptr &= ring->ptr_mask;
334 ring->count_dw--;
335 ring->ring_free_dw--;
336}
337
338
339
340
341
342
343
344
345
346
347
348bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
349 struct radeon_ring *ring)
350{
351 switch (ring->idx) {
352 case RADEON_RING_TYPE_GFX_INDEX:
353 case CAYMAN_RING_TYPE_CP1_INDEX:
354 case CAYMAN_RING_TYPE_CP2_INDEX:
355 return true;
356 default:
357 return false;
358 }
359}
360
361u32 radeon_ring_generic_get_rptr(struct radeon_device *rdev,
362 struct radeon_ring *ring)
363{
364 u32 rptr;
365
366 if (rdev->wb.enabled && ring != &rdev->ring[R600_RING_TYPE_UVD_INDEX])
367 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
368 else
369 rptr = RREG32(ring->rptr_reg);
370 rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
371
372 return rptr;
373}
374
375u32 radeon_ring_generic_get_wptr(struct radeon_device *rdev,
376 struct radeon_ring *ring)
377{
378 u32 wptr;
379
380 wptr = RREG32(ring->wptr_reg);
381 wptr = (wptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
382
383 return wptr;
384}
385
386void radeon_ring_generic_set_wptr(struct radeon_device *rdev,
387 struct radeon_ring *ring)
388{
389 WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask);
390 (void)RREG32(ring->wptr_reg);
391}
392
393
394
395
396
397
398
399
400
401void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
402{
403 ring->rptr = radeon_ring_get_rptr(rdev, ring);
404
405 ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
406 ring->ring_free_dw -= ring->wptr;
407 ring->ring_free_dw &= ring->ptr_mask;
408 if (!ring->ring_free_dw) {
409 ring->ring_free_dw = ring->ring_size / 4;
410 }
411}
412
413
414
415
416
417
418
419
420
421
422
423int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
424{
425 int r;
426
427
428 if (ndw > (ring->ring_size / 4))
429 return -ENOMEM;
430
431
432 radeon_ring_free_size(rdev, ring);
433 if (ring->ring_free_dw == (ring->ring_size / 4)) {
434
435
436
437 radeon_ring_lockup_update(ring);
438 }
439 ndw = (ndw + ring->align_mask) & ~ring->align_mask;
440 while (ndw > (ring->ring_free_dw - 1)) {
441 radeon_ring_free_size(rdev, ring);
442 if (ndw < ring->ring_free_dw) {
443 break;
444 }
445 r = radeon_fence_wait_next_locked(rdev, ring->idx);
446 if (r)
447 return r;
448 }
449 ring->count_dw = ndw;
450 ring->wptr_old = ring->wptr;
451 return 0;
452}
453
454
455
456
457
458
459
460
461
462
463
464
465int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
466{
467 int r;
468
469 mutex_lock(&rdev->ring_lock);
470 r = radeon_ring_alloc(rdev, ring, ndw);
471 if (r) {
472 mutex_unlock(&rdev->ring_lock);
473 return r;
474 }
475 return 0;
476}
477
478
479
480
481
482
483
484
485
486
487
488void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring)
489{
490
491 while (ring->wptr & ring->align_mask) {
492 radeon_ring_write(ring, ring->nop);
493 }
494 DRM_MEMORYBARRIER();
495 radeon_ring_set_wptr(rdev, ring);
496}
497
498
499
500
501
502
503
504
505
506
507void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring)
508{
509 radeon_ring_commit(rdev, ring);
510 mutex_unlock(&rdev->ring_lock);
511}
512
513
514
515
516
517
518
519
520void radeon_ring_undo(struct radeon_ring *ring)
521{
522 ring->wptr = ring->wptr_old;
523}
524
525
526
527
528
529
530
531
532void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
533{
534 radeon_ring_undo(ring);
535 mutex_unlock(&rdev->ring_lock);
536}
537
538
539
540
541
542
543
544
545
546
547void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring)
548{
549 int r;
550
551 radeon_ring_free_size(rdev, ring);
552 if (ring->rptr == ring->wptr) {
553 r = radeon_ring_alloc(rdev, ring, 1);
554 if (!r) {
555 radeon_ring_write(ring, ring->nop);
556 radeon_ring_commit(rdev, ring);
557 }
558 }
559}
560
561
562
563
564
565
566
567
568void radeon_ring_lockup_update(struct radeon_ring *ring)
569{
570 ring->last_rptr = ring->rptr;
571 ring->last_activity = jiffies;
572}
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
595{
596 unsigned long cjiffies, elapsed;
597
598 cjiffies = jiffies;
599 if (!time_after(cjiffies, ring->last_activity)) {
600
601 radeon_ring_lockup_update(ring);
602 return false;
603 }
604 ring->rptr = radeon_ring_get_rptr(rdev, ring);
605 if (ring->rptr != ring->last_rptr) {
606
607 radeon_ring_lockup_update(ring);
608 return false;
609 }
610 elapsed = jiffies_to_msecs(cjiffies - ring->last_activity);
611 if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) {
612 dev_err(rdev->dev, "GPU lockup CP stall for more than %lumsec\n", elapsed);
613 return true;
614 }
615
616 return false;
617}
618
619
620
621
622
623
624
625
626
627unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
628 uint32_t **data)
629{
630 unsigned size, ptr, i;
631
632
633 mutex_lock(&rdev->ring_lock);
634 *data = NULL;
635
636 if (ring->ring_obj == NULL) {
637 mutex_unlock(&rdev->ring_lock);
638 return 0;
639 }
640
641
642 if (!radeon_fence_count_emitted(rdev, ring->idx)) {
643 mutex_unlock(&rdev->ring_lock);
644 return 0;
645 }
646
647
648 if (ring->rptr_save_reg)
649 ptr = RREG32(ring->rptr_save_reg);
650 else if (rdev->wb.enabled)
651 ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
652 else {
653
654 mutex_unlock(&rdev->ring_lock);
655 return 0;
656 }
657
658 size = ring->wptr + (ring->ring_size / 4);
659 size -= ptr;
660 size &= ring->ptr_mask;
661 if (size == 0) {
662 mutex_unlock(&rdev->ring_lock);
663 return 0;
664 }
665
666
667 *data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
668 if (!*data) {
669 mutex_unlock(&rdev->ring_lock);
670 return 0;
671 }
672 for (i = 0; i < size; ++i) {
673 (*data)[i] = ring->ring[ptr++];
674 ptr &= ring->ptr_mask;
675 }
676
677 mutex_unlock(&rdev->ring_lock);
678 return size;
679}
680
681
682
683
684
685
686
687
688
689
690
691int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
692 unsigned size, uint32_t *data)
693{
694 int i, r;
695
696 if (!size || !data)
697 return 0;
698
699
700 r = radeon_ring_lock(rdev, ring, size);
701 if (r)
702 return r;
703
704 for (i = 0; i < size; ++i) {
705 radeon_ring_write(ring, data[i]);
706 }
707
708 radeon_ring_unlock_commit(rdev, ring);
709 kfree(data);
710 return 0;
711}
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
730 unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg,
731 u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop)
732{
733 int r;
734
735 ring->ring_size = ring_size;
736 ring->rptr_offs = rptr_offs;
737 ring->rptr_reg = rptr_reg;
738 ring->wptr_reg = wptr_reg;
739 ring->ptr_reg_shift = ptr_reg_shift;
740 ring->ptr_reg_mask = ptr_reg_mask;
741 ring->nop = nop;
742
743 if (ring->ring_obj == NULL) {
744 r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
745 RADEON_GEM_DOMAIN_GTT,
746 NULL, &ring->ring_obj);
747 if (r) {
748 dev_err(rdev->dev, "(%d) ring create failed\n", r);
749 return r;
750 }
751 r = radeon_bo_reserve(ring->ring_obj, false);
752 if (unlikely(r != 0))
753 return r;
754 r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
755 &ring->gpu_addr);
756 if (r) {
757 radeon_bo_unreserve(ring->ring_obj);
758 dev_err(rdev->dev, "(%d) ring pin failed\n", r);
759 return r;
760 }
761 r = radeon_bo_kmap(ring->ring_obj,
762 (void **)&ring->ring);
763 radeon_bo_unreserve(ring->ring_obj);
764 if (r) {
765 dev_err(rdev->dev, "(%d) ring map failed\n", r);
766 return r;
767 }
768 }
769 ring->ptr_mask = (ring->ring_size / 4) - 1;
770 ring->ring_free_dw = ring->ring_size / 4;
771 if (rdev->wb.enabled) {
772 u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);
773 ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;
774 ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];
775 }
776 if (radeon_debugfs_ring_init(rdev, ring)) {
777 DRM_ERROR("Failed to register debugfs file for rings !\n");
778 }
779 radeon_ring_lockup_update(ring);
780 return 0;
781}
782
783
784
785
786
787
788
789
790
791void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
792{
793 int r;
794 struct radeon_bo *ring_obj;
795
796 mutex_lock(&rdev->ring_lock);
797 ring_obj = ring->ring_obj;
798 ring->ready = false;
799 ring->ring = NULL;
800 ring->ring_obj = NULL;
801 mutex_unlock(&rdev->ring_lock);
802
803 if (ring_obj) {
804 r = radeon_bo_reserve(ring_obj, false);
805 if (likely(r == 0)) {
806 radeon_bo_kunmap(ring_obj);
807 radeon_bo_unpin(ring_obj);
808 radeon_bo_unreserve(ring_obj);
809 }
810 radeon_bo_unref(&ring_obj);
811 }
812}
813
814
815
816
817#if defined(CONFIG_DEBUG_FS)
818
819static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
820{
821 struct drm_info_node *node = (struct drm_info_node *) m->private;
822 struct drm_device *dev = node->minor->dev;
823 struct radeon_device *rdev = dev->dev_private;
824 int ridx = *(int*)node->info_ent->data;
825 struct radeon_ring *ring = &rdev->ring[ridx];
826 unsigned count, i, j;
827 u32 tmp;
828
829 radeon_ring_free_size(rdev, ring);
830 count = (ring->ring_size / 4) - ring->ring_free_dw;
831 tmp = radeon_ring_get_wptr(rdev, ring);
832 seq_printf(m, "wptr(0x%04x): 0x%08x [%5d]\n", ring->wptr_reg, tmp, tmp);
833 tmp = radeon_ring_get_rptr(rdev, ring);
834 seq_printf(m, "rptr(0x%04x): 0x%08x [%5d]\n", ring->rptr_reg, tmp, tmp);
835 if (ring->rptr_save_reg) {
836 seq_printf(m, "rptr next(0x%04x): 0x%08x\n", ring->rptr_save_reg,
837 RREG32(ring->rptr_save_reg));
838 }
839 seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n", ring->wptr, ring->wptr);
840 seq_printf(m, "driver's copy of the rptr: 0x%08x [%5d]\n", ring->rptr, ring->rptr);
841 seq_printf(m, "last semaphore signal addr : 0x%016llx\n", ring->last_semaphore_signal_addr);
842 seq_printf(m, "last semaphore wait addr : 0x%016llx\n", ring->last_semaphore_wait_addr);
843 seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
844 seq_printf(m, "%u dwords in ring\n", count);
845
846
847
848 i = (ring->rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
849 for (j = 0; j <= (count + 32); j++) {
850 seq_printf(m, "r[%5d]=0x%08x\n", i, ring->ring[i]);
851 i = (i + 1) & ring->ptr_mask;
852 }
853 return 0;
854}
855
856static int radeon_gfx_index = RADEON_RING_TYPE_GFX_INDEX;
857static int cayman_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX;
858static int cayman_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;
859static int radeon_dma1_index = R600_RING_TYPE_DMA_INDEX;
860static int radeon_dma2_index = CAYMAN_RING_TYPE_DMA1_INDEX;
861static int r600_uvd_index = R600_RING_TYPE_UVD_INDEX;
862
863static struct drm_info_list radeon_debugfs_ring_info_list[] = {
864 {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_gfx_index},
865 {"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_cp1_index},
866 {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_cp2_index},
867 {"radeon_ring_dma1", radeon_debugfs_ring_info, 0, &radeon_dma1_index},
868 {"radeon_ring_dma2", radeon_debugfs_ring_info, 0, &radeon_dma2_index},
869 {"radeon_ring_uvd", radeon_debugfs_ring_info, 0, &r600_uvd_index},
870};
871
872static int radeon_debugfs_sa_info(struct seq_file *m, void *data)
873{
874 struct drm_info_node *node = (struct drm_info_node *) m->private;
875 struct drm_device *dev = node->minor->dev;
876 struct radeon_device *rdev = dev->dev_private;
877
878 radeon_sa_bo_dump_debug_info(&rdev->ring_tmp_bo, m);
879
880 return 0;
881
882}
883
884static struct drm_info_list radeon_debugfs_sa_list[] = {
885 {"radeon_sa_info", &radeon_debugfs_sa_info, 0, NULL},
886};
887
888#endif
889
890static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
891{
892#if defined(CONFIG_DEBUG_FS)
893 unsigned i;
894 for (i = 0; i < ARRAY_SIZE(radeon_debugfs_ring_info_list); ++i) {
895 struct drm_info_list *info = &radeon_debugfs_ring_info_list[i];
896 int ridx = *(int*)radeon_debugfs_ring_info_list[i].data;
897 unsigned r;
898
899 if (&rdev->ring[ridx] != ring)
900 continue;
901
902 r = radeon_debugfs_add_files(rdev, info, 1);
903 if (r)
904 return r;
905 }
906#endif
907 return 0;
908}
909
910static int radeon_debugfs_sa_init(struct radeon_device *rdev)
911{
912#if defined(CONFIG_DEBUG_FS)
913 return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1);
914#else
915 return 0;
916#endif
917}
918