1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <drm/drmP.h>
25#include "amdgpu.h"
26#include "amdgpu_ih.h"
27#include "amdgpu_amdkfd.h"
28
29
30
31
32
33
34
35
36
37static int amdgpu_ih_ring_alloc(struct amdgpu_device *adev)
38{
39 int r;
40
41
42 if (adev->irq.ih.ring_obj == NULL) {
43 r = amdgpu_bo_create(adev, adev->irq.ih.ring_size,
44 PAGE_SIZE, true,
45 AMDGPU_GEM_DOMAIN_GTT, 0,
46 NULL, NULL, &adev->irq.ih.ring_obj);
47 if (r) {
48 DRM_ERROR("amdgpu: failed to create ih ring buffer (%d).\n", r);
49 return r;
50 }
51 r = amdgpu_bo_reserve(adev->irq.ih.ring_obj, false);
52 if (unlikely(r != 0))
53 return r;
54 r = amdgpu_bo_pin(adev->irq.ih.ring_obj,
55 AMDGPU_GEM_DOMAIN_GTT,
56 &adev->irq.ih.gpu_addr);
57 if (r) {
58 amdgpu_bo_unreserve(adev->irq.ih.ring_obj);
59 DRM_ERROR("amdgpu: failed to pin ih ring buffer (%d).\n", r);
60 return r;
61 }
62 r = amdgpu_bo_kmap(adev->irq.ih.ring_obj,
63 (void **)&adev->irq.ih.ring);
64 amdgpu_bo_unreserve(adev->irq.ih.ring_obj);
65 if (r) {
66 DRM_ERROR("amdgpu: failed to map ih ring buffer (%d).\n", r);
67 return r;
68 }
69 }
70 return 0;
71}
72
73
74
75
76
77
78
79
80
81
82int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
83 bool use_bus_addr)
84{
85 u32 rb_bufsz;
86 int r;
87
88
89 rb_bufsz = order_base_2(ring_size / 4);
90 ring_size = (1 << rb_bufsz) * 4;
91 adev->irq.ih.ring_size = ring_size;
92 adev->irq.ih.ptr_mask = adev->irq.ih.ring_size - 1;
93 adev->irq.ih.rptr = 0;
94 adev->irq.ih.use_bus_addr = use_bus_addr;
95
96 if (adev->irq.ih.use_bus_addr) {
97 if (!adev->irq.ih.ring) {
98
99
100
101 adev->irq.ih.ring = pci_alloc_consistent(adev->pdev,
102 adev->irq.ih.ring_size + 8,
103 &adev->irq.ih.rb_dma_addr);
104 if (adev->irq.ih.ring == NULL)
105 return -ENOMEM;
106 memset((void *)adev->irq.ih.ring, 0, adev->irq.ih.ring_size + 8);
107 adev->irq.ih.wptr_offs = (adev->irq.ih.ring_size / 4) + 0;
108 adev->irq.ih.rptr_offs = (adev->irq.ih.ring_size / 4) + 1;
109 }
110 return 0;
111 } else {
112 r = amdgpu_wb_get(adev, &adev->irq.ih.wptr_offs);
113 if (r) {
114 dev_err(adev->dev, "(%d) ih wptr_offs wb alloc failed\n", r);
115 return r;
116 }
117
118 r = amdgpu_wb_get(adev, &adev->irq.ih.rptr_offs);
119 if (r) {
120 amdgpu_wb_free(adev, adev->irq.ih.wptr_offs);
121 dev_err(adev->dev, "(%d) ih rptr_offs wb alloc failed\n", r);
122 return r;
123 }
124
125 return amdgpu_ih_ring_alloc(adev);
126 }
127}
128
129
130
131
132
133
134
135
136
137void amdgpu_ih_ring_fini(struct amdgpu_device *adev)
138{
139 int r;
140
141 if (adev->irq.ih.use_bus_addr) {
142 if (adev->irq.ih.ring) {
143
144
145
146 pci_free_consistent(adev->pdev, adev->irq.ih.ring_size + 8,
147 (void *)adev->irq.ih.ring,
148 adev->irq.ih.rb_dma_addr);
149 adev->irq.ih.ring = NULL;
150 }
151 } else {
152 if (adev->irq.ih.ring_obj) {
153 r = amdgpu_bo_reserve(adev->irq.ih.ring_obj, false);
154 if (likely(r == 0)) {
155 amdgpu_bo_kunmap(adev->irq.ih.ring_obj);
156 amdgpu_bo_unpin(adev->irq.ih.ring_obj);
157 amdgpu_bo_unreserve(adev->irq.ih.ring_obj);
158 }
159 amdgpu_bo_unref(&adev->irq.ih.ring_obj);
160 adev->irq.ih.ring = NULL;
161 adev->irq.ih.ring_obj = NULL;
162 }
163 amdgpu_wb_free(adev, adev->irq.ih.wptr_offs);
164 amdgpu_wb_free(adev, adev->irq.ih.rptr_offs);
165 }
166}
167
168
169
170
171
172
173
174
175
176int amdgpu_ih_process(struct amdgpu_device *adev)
177{
178 struct amdgpu_iv_entry entry;
179 u32 wptr;
180
181 if (!adev->irq.ih.enabled || adev->shutdown)
182 return IRQ_NONE;
183
184 wptr = amdgpu_ih_get_wptr(adev);
185
186restart_ih:
187
188 if (atomic_xchg(&adev->irq.ih.lock, 1))
189 return IRQ_NONE;
190
191 DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, adev->irq.ih.rptr, wptr);
192
193
194 rmb();
195
196 while (adev->irq.ih.rptr != wptr) {
197 u32 ring_index = adev->irq.ih.rptr >> 2;
198
199
200 amdgpu_amdkfd_interrupt(adev,
201 (const void *) &adev->irq.ih.ring[ring_index]);
202
203 entry.iv_entry = (const uint32_t *)
204 &adev->irq.ih.ring[ring_index];
205 amdgpu_ih_decode_iv(adev, &entry);
206 adev->irq.ih.rptr &= adev->irq.ih.ptr_mask;
207
208 amdgpu_irq_dispatch(adev, &entry);
209 }
210 amdgpu_ih_set_rptr(adev);
211 atomic_set(&adev->irq.ih.lock, 0);
212
213
214 wptr = amdgpu_ih_get_wptr(adev);
215 if (wptr != adev->irq.ih.rptr)
216 goto restart_ih;
217
218 return IRQ_HANDLED;
219}
220