1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/dma-mapping.h>
25
26#include "amdgpu.h"
27#include "amdgpu_ih.h"
28
29
30
31
32
33
34
35
36
37
38
39
40
41int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
42 unsigned ring_size, bool use_bus_addr)
43{
44 u32 rb_bufsz;
45 int r;
46
47
48 rb_bufsz = order_base_2(ring_size / 4);
49 ring_size = (1 << rb_bufsz) * 4;
50 ih->ring_size = ring_size;
51 ih->ptr_mask = ih->ring_size - 1;
52 ih->rptr = 0;
53 ih->use_bus_addr = use_bus_addr;
54
55 if (use_bus_addr) {
56 dma_addr_t dma_addr;
57
58 if (ih->ring)
59 return 0;
60
61
62
63
64 ih->ring = dma_alloc_coherent(adev->dev, ih->ring_size + 8,
65 &dma_addr, GFP_KERNEL);
66 if (ih->ring == NULL)
67 return -ENOMEM;
68
69 ih->gpu_addr = dma_addr;
70 ih->wptr_addr = dma_addr + ih->ring_size;
71 ih->wptr_cpu = &ih->ring[ih->ring_size / 4];
72 ih->rptr_addr = dma_addr + ih->ring_size + 4;
73 ih->rptr_cpu = &ih->ring[(ih->ring_size / 4) + 1];
74 } else {
75 unsigned wptr_offs, rptr_offs;
76
77 r = amdgpu_device_wb_get(adev, &wptr_offs);
78 if (r)
79 return r;
80
81 r = amdgpu_device_wb_get(adev, &rptr_offs);
82 if (r) {
83 amdgpu_device_wb_free(adev, wptr_offs);
84 return r;
85 }
86
87 r = amdgpu_bo_create_kernel(adev, ih->ring_size, PAGE_SIZE,
88 AMDGPU_GEM_DOMAIN_GTT,
89 &ih->ring_obj, &ih->gpu_addr,
90 (void **)&ih->ring);
91 if (r) {
92 amdgpu_device_wb_free(adev, rptr_offs);
93 amdgpu_device_wb_free(adev, wptr_offs);
94 return r;
95 }
96
97 ih->wptr_addr = adev->wb.gpu_addr + wptr_offs * 4;
98 ih->wptr_cpu = &adev->wb.wb[wptr_offs];
99 ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4;
100 ih->rptr_cpu = &adev->wb.wb[rptr_offs];
101 }
102
103 init_waitqueue_head(&ih->wait_process);
104 return 0;
105}
106
107
108
109
110
111
112
113
114
115
116void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
117{
118
119 if (!ih->ring)
120 return;
121
122 if (ih->use_bus_addr) {
123
124
125
126
127 dma_free_coherent(adev->dev, ih->ring_size + 8,
128 (void *)ih->ring, ih->gpu_addr);
129 ih->ring = NULL;
130 } else {
131 amdgpu_bo_free_kernel(&ih->ring_obj, &ih->gpu_addr,
132 (void **)&ih->ring);
133 amdgpu_device_wb_free(adev, (ih->wptr_addr - ih->gpu_addr) / 4);
134 amdgpu_device_wb_free(adev, (ih->rptr_addr - ih->gpu_addr) / 4);
135 }
136}
137
138
139
140
141
142
143
144
145
146
147
148void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv,
149 unsigned int num_dw)
150{
151 uint32_t wptr = le32_to_cpu(*ih->wptr_cpu) >> 2;
152 unsigned int i;
153
154 for (i = 0; i < num_dw; ++i)
155 ih->ring[wptr++] = cpu_to_le32(iv[i]);
156
157 wptr <<= 2;
158 wptr &= ih->ptr_mask;
159
160
161 if (wptr != READ_ONCE(ih->rptr)) {
162 wmb();
163 WRITE_ONCE(*ih->wptr_cpu, cpu_to_le32(wptr));
164 }
165}
166
167
168static bool amdgpu_ih_has_checkpoint_processed(struct amdgpu_device *adev,
169 struct amdgpu_ih_ring *ih,
170 uint32_t checkpoint_wptr,
171 uint32_t *prev_rptr)
172{
173 uint32_t cur_rptr = ih->rptr | (*prev_rptr & ~ih->ptr_mask);
174
175
176 if (cur_rptr < *prev_rptr)
177 cur_rptr += ih->ptr_mask + 1;
178 *prev_rptr = cur_rptr;
179
180
181 return cur_rptr >= checkpoint_wptr ||
182 (cur_rptr & ih->ptr_mask) == amdgpu_ih_get_wptr(adev, ih);
183}
184
185
186
187
188
189
190
191
192
193int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
194 struct amdgpu_ih_ring *ih)
195{
196 uint32_t checkpoint_wptr, rptr;
197
198 if (!ih->enabled || adev->shutdown)
199 return -ENODEV;
200
201 checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);
202
203 rmb();
204 rptr = READ_ONCE(ih->rptr);
205
206
207 if (rptr > checkpoint_wptr)
208 checkpoint_wptr += ih->ptr_mask + 1;
209
210 return wait_event_interruptible(ih->wait_process,
211 amdgpu_ih_has_checkpoint_processed(adev, ih,
212 checkpoint_wptr, &rptr));
213}
214
215
216
217
218
219
220
221
222
223
224int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
225{
226 unsigned int count = AMDGPU_IH_MAX_NUM_IVS;
227 u32 wptr;
228
229 if (!ih->enabled || adev->shutdown)
230 return IRQ_NONE;
231
232 wptr = amdgpu_ih_get_wptr(adev, ih);
233
234restart_ih:
235 DRM_DEBUG("%s: rptr %d, wptr %d\n", __func__, ih->rptr, wptr);
236
237
238 rmb();
239
240 while (ih->rptr != wptr && --count) {
241 amdgpu_irq_dispatch(adev, ih);
242 ih->rptr &= ih->ptr_mask;
243 }
244
245 amdgpu_ih_set_rptr(adev, ih);
246 wake_up_all(&ih->wait_process);
247
248
249 wptr = amdgpu_ih_get_wptr(adev, ih);
250 if (wptr != ih->rptr)
251 goto restart_ih;
252
253 return IRQ_HANDLED;
254}
255
256
257
258
259
260
261
262
263
264
265
266
267void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev,
268 struct amdgpu_ih_ring *ih,
269 struct amdgpu_iv_entry *entry)
270{
271
272 u32 ring_index = ih->rptr >> 2;
273 uint32_t dw[8];
274
275 dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
276 dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
277 dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
278 dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
279 dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
280 dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
281 dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
282 dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
283
284 entry->client_id = dw[0] & 0xff;
285 entry->src_id = (dw[0] >> 8) & 0xff;
286 entry->ring_id = (dw[0] >> 16) & 0xff;
287 entry->vmid = (dw[0] >> 24) & 0xf;
288 entry->vmid_src = (dw[0] >> 31);
289 entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
290 entry->timestamp_src = dw[2] >> 31;
291 entry->pasid = dw[3] & 0xffff;
292 entry->pasid_src = dw[3] >> 31;
293 entry->src_data[0] = dw[4];
294 entry->src_data[1] = dw[5];
295 entry->src_data[2] = dw[6];
296 entry->src_data[3] = dw[7];
297
298
299 ih->rptr += 32;
300}
301