1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "msm_gem.h"
15#include "msm_mmu.h"
16#include "a5xx_gpu.h"
17
18extern bool hang_debug;
19static void a5xx_dump(struct msm_gpu *gpu);
20
21static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
22 struct msm_file_private *ctx)
23{
24 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
25 struct msm_drm_private *priv = gpu->dev->dev_private;
26 struct msm_ringbuffer *ring = gpu->rb;
27 unsigned int i, ibs = 0;
28
29 for (i = 0; i < submit->nr_cmds; i++) {
30 switch (submit->cmd[i].type) {
31 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
32 break;
33 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
34 if (priv->lastctx == ctx)
35 break;
36 case MSM_SUBMIT_CMD_BUF:
37 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
38 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
39 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
40 OUT_RING(ring, submit->cmd[i].size);
41 ibs++;
42 break;
43 }
44 }
45
46 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
47 OUT_RING(ring, submit->fence->seqno);
48
49 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
50 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
51 OUT_RING(ring, lower_32_bits(rbmemptr(adreno_gpu, fence)));
52 OUT_RING(ring, upper_32_bits(rbmemptr(adreno_gpu, fence)));
53 OUT_RING(ring, submit->fence->seqno);
54
55 gpu->funcs->flush(gpu);
56}
57
58struct a5xx_hwcg {
59 u32 offset;
60 u32 value;
61};
62
63static const struct a5xx_hwcg a530_hwcg[] = {
64 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
65 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
66 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
67 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
68 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
69 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
70 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
71 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
72 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
73 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
74 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
75 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
76 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
77 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
78 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
79 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
80 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
81 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
82 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
83 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
84 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
85 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
86 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
87 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
88 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
89 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
90 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
91 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
92 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
93 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
94 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
95 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
96 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
97 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
98 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
99 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
100 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
101 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
102 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
103 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
104 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
105 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
106 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
107 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
108 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
109 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
110 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
111 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
112 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
113 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
114 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
115 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
116 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
117 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
118 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
119 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
120 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
121 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
122 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
123 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
124 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
125 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
126 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
127 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
128 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
129 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
130 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
131 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
132 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
133 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
134 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
135 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
136 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
137 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
138 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
139 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
140 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
141 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
142 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
143 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
144 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
145 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
146 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
147 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
148 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
149 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
150 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
151 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
152 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
153 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
154 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
155 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
156};
157
158static const struct {
159 int (*test)(struct adreno_gpu *gpu);
160 const struct a5xx_hwcg *regs;
161 unsigned int count;
162} a5xx_hwcg_regs[] = {
163 { adreno_is_a530, a530_hwcg, ARRAY_SIZE(a530_hwcg), },
164};
165
166static void _a5xx_enable_hwcg(struct msm_gpu *gpu,
167 const struct a5xx_hwcg *regs, unsigned int count)
168{
169 unsigned int i;
170
171 for (i = 0; i < count; i++)
172 gpu_write(gpu, regs[i].offset, regs[i].value);
173
174 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xAAA8AA00);
175 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, 0x182);
176}
177
178static void a5xx_enable_hwcg(struct msm_gpu *gpu)
179{
180 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
181 unsigned int i;
182
183 for (i = 0; i < ARRAY_SIZE(a5xx_hwcg_regs); i++) {
184 if (a5xx_hwcg_regs[i].test(adreno_gpu)) {
185 _a5xx_enable_hwcg(gpu, a5xx_hwcg_regs[i].regs,
186 a5xx_hwcg_regs[i].count);
187 return;
188 }
189 }
190}
191
192static int a5xx_me_init(struct msm_gpu *gpu)
193{
194 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
195 struct msm_ringbuffer *ring = gpu->rb;
196
197 OUT_PKT7(ring, CP_ME_INIT, 8);
198
199 OUT_RING(ring, 0x0000002F);
200
201
202 OUT_RING(ring, 0x00000003);
203
204
205 OUT_RING(ring, 0x20000000);
206
207
208 OUT_RING(ring, 0x00000000);
209 OUT_RING(ring, 0x00000000);
210
211
212 if (adreno_is_a530(adreno_gpu)) {
213
214
215
216
217 OUT_RING(ring, 0x0000000B);
218 } else {
219
220 OUT_RING(ring, 0x00000000);
221 }
222
223 OUT_RING(ring, 0x00000000);
224 OUT_RING(ring, 0x00000000);
225
226 gpu->funcs->flush(gpu);
227
228 return gpu->funcs->idle(gpu) ? 0 : -EINVAL;
229}
230
231static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
232 const struct firmware *fw, u64 *iova)
233{
234 struct drm_device *drm = gpu->dev;
235 struct drm_gem_object *bo;
236 void *ptr;
237
238 mutex_lock(&drm->struct_mutex);
239 bo = msm_gem_new(drm, fw->size - 4, MSM_BO_UNCACHED);
240 mutex_unlock(&drm->struct_mutex);
241
242 if (IS_ERR(bo))
243 return bo;
244
245 ptr = msm_gem_get_vaddr(bo);
246 if (!ptr) {
247 drm_gem_object_unreference_unlocked(bo);
248 return ERR_PTR(-ENOMEM);
249 }
250
251 if (iova) {
252 int ret = msm_gem_get_iova(bo, gpu->id, iova);
253
254 if (ret) {
255 drm_gem_object_unreference_unlocked(bo);
256 return ERR_PTR(ret);
257 }
258 }
259
260 memcpy(ptr, &fw->data[4], fw->size - 4);
261
262 msm_gem_put_vaddr(bo);
263 return bo;
264}
265
266static int a5xx_ucode_init(struct msm_gpu *gpu)
267{
268 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
269 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
270 int ret;
271
272 if (!a5xx_gpu->pm4_bo) {
273 a5xx_gpu->pm4_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pm4,
274 &a5xx_gpu->pm4_iova);
275
276 if (IS_ERR(a5xx_gpu->pm4_bo)) {
277 ret = PTR_ERR(a5xx_gpu->pm4_bo);
278 a5xx_gpu->pm4_bo = NULL;
279 dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
280 ret);
281 return ret;
282 }
283 }
284
285 if (!a5xx_gpu->pfp_bo) {
286 a5xx_gpu->pfp_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pfp,
287 &a5xx_gpu->pfp_iova);
288
289 if (IS_ERR(a5xx_gpu->pfp_bo)) {
290 ret = PTR_ERR(a5xx_gpu->pfp_bo);
291 a5xx_gpu->pfp_bo = NULL;
292 dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
293 ret);
294 return ret;
295 }
296 }
297
298 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
299 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
300
301 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
302 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
303
304 return 0;
305}
306
307#define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
308 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
309 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
310 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
311 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
312 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
313 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
314 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
315 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
316 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
317
318static int a5xx_hw_init(struct msm_gpu *gpu)
319{
320 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
321 int ret;
322
323 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
324
325
326 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
327
328
329 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
330
331 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
332
333
334
335
336
337 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
338 0xF0000000);
339 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
340 0xFFFFFFFF);
341 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
342 0xFFFFFFFF);
343 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
344 0xFFFFFFFF);
345 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
346 0xFFFFFFFF);
347 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
348 0xFFFFFFFF);
349 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
350 0xFFFFFFFF);
351 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
352 0xFFFFFFFF);
353 }
354
355
356 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
357 (1 << 30) | 0xFFFF);
358
359
360 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
361
362
363 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
364
365
366 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
367 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
368 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
369 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
370
371
372 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
373 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
374 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
375 0x00100000 + adreno_gpu->gmem - 1);
376 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
377
378 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
379 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
380 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
381 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
382
383 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
384
385 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
386 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
387
388 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
389
390
391 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
392
393
394 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
395
396
397 a5xx_enable_hwcg(gpu);
398
399 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
400
401
402 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
403 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
404
405
406 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
407
408
409 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
410 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
411 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
412 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
413 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
414 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
415
416
417 gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
418 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
419 16));
420 gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
421 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
422
423
424 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
425 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
426 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
427 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
428
429
430 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
431 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
432
433
434 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
435 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
436
437
438 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
439
440 if (adreno_is_a530(adreno_gpu))
441 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
442 ADRENO_PROTECT_RW(0x10000, 0x8000));
443
444 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
445
446
447
448
449
450 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
451 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
452 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
453
454
455 a5xx_gpmu_ucode_init(gpu);
456
457 ret = adreno_hw_init(gpu);
458 if (ret)
459 return ret;
460
461 ret = a5xx_ucode_init(gpu);
462 if (ret)
463 return ret;
464
465
466 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
467
468
469 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
470 ret = a5xx_me_init(gpu);
471 if (ret)
472 return ret;
473
474 ret = a5xx_power_init(gpu);
475 if (ret)
476 return ret;
477
478
479
480
481
482 if (adreno_is_a530(adreno_gpu)) {
483 OUT_PKT7(gpu->rb, CP_EVENT_WRITE, 1);
484 OUT_RING(gpu->rb, 0x0F);
485
486 gpu->funcs->flush(gpu);
487 if (!gpu->funcs->idle(gpu))
488 return -EINVAL;
489 }
490
491
492 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
493
494 return 0;
495}
496
497static void a5xx_recover(struct msm_gpu *gpu)
498{
499 int i;
500
501 adreno_dump_info(gpu);
502
503 for (i = 0; i < 8; i++) {
504 printk("CP_SCRATCH_REG%d: %u\n", i,
505 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
506 }
507
508 if (hang_debug)
509 a5xx_dump(gpu);
510
511 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
512 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
513 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
514 adreno_recover(gpu);
515}
516
517static void a5xx_destroy(struct msm_gpu *gpu)
518{
519 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
520 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
521
522 DBG("%s", gpu->name);
523
524 if (a5xx_gpu->pm4_bo) {
525 if (a5xx_gpu->pm4_iova)
526 msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->id);
527 drm_gem_object_unreference_unlocked(a5xx_gpu->pm4_bo);
528 }
529
530 if (a5xx_gpu->pfp_bo) {
531 if (a5xx_gpu->pfp_iova)
532 msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->id);
533 drm_gem_object_unreference_unlocked(a5xx_gpu->pfp_bo);
534 }
535
536 if (a5xx_gpu->gpmu_bo) {
537 if (a5xx_gpu->gpmu_iova)
538 msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->id);
539 drm_gem_object_unreference_unlocked(a5xx_gpu->gpmu_bo);
540 }
541
542 adreno_gpu_cleanup(adreno_gpu);
543 kfree(a5xx_gpu);
544}
545
546static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
547{
548 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
549 return false;
550
551
552
553
554
555 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
556 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
557}
558
559static bool a5xx_idle(struct msm_gpu *gpu)
560{
561
562 if (!adreno_idle(gpu))
563 return false;
564
565 if (spin_until(_a5xx_check_idle(gpu))) {
566 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X\n",
567 gpu->name, __builtin_return_address(0),
568 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
569 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS));
570
571 return false;
572 }
573
574 return true;
575}
576
577static int a5xx_fault_handler(void *arg, unsigned long iova, int flags)
578{
579 struct msm_gpu *gpu = arg;
580 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
581 iova, flags,
582 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
583 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
584 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
585 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
586
587 return -EFAULT;
588}
589
590static void a5xx_cp_err_irq(struct msm_gpu *gpu)
591{
592 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
593
594 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
595 u32 val;
596
597 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
598
599
600
601
602
603
604 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
605 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
606
607 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
608 val);
609 }
610
611 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
612 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
613 gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
614
615 if (status & A5XX_CP_INT_CP_DMA_ERROR)
616 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
617
618 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
619 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
620
621 dev_err_ratelimited(gpu->dev->dev,
622 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
623 val & (1 << 24) ? "WRITE" : "READ",
624 (val & 0xFFFFF) >> 2, val);
625 }
626
627 if (status & A5XX_CP_INT_CP_AHB_ERROR) {
628 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
629 const char *access[16] = { "reserved", "reserved",
630 "timestamp lo", "timestamp hi", "pfp read", "pfp write",
631 "", "", "me read", "me write", "", "", "crashdump read",
632 "crashdump write" };
633
634 dev_err_ratelimited(gpu->dev->dev,
635 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
636 status & 0xFFFFF, access[(status >> 24) & 0xF],
637 (status & (1 << 31)), status);
638 }
639}
640
641static void a5xx_rbbm_err_irq(struct msm_gpu *gpu)
642{
643 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
644
645 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
646 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
647
648 dev_err_ratelimited(gpu->dev->dev,
649 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
650 val & (1 << 28) ? "WRITE" : "READ",
651 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
652 (val >> 24) & 0xF);
653
654
655 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
656 }
657
658 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
659 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
660
661 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
662 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
663 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
664
665 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
666 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
667 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
668
669 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
670 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
671 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
672
673 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
674 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
675
676 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
677 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
678}
679
680static void a5xx_uche_err_irq(struct msm_gpu *gpu)
681{
682 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
683
684 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
685
686 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
687 addr);
688}
689
690static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
691{
692 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
693}
694
695#define RBBM_ERROR_MASK \
696 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
697 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
698 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
699 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
700 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
701 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
702
703static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
704{
705 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
706
707 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, status);
708
709 if (status & RBBM_ERROR_MASK)
710 a5xx_rbbm_err_irq(gpu);
711
712 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
713 a5xx_cp_err_irq(gpu);
714
715 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
716 a5xx_uche_err_irq(gpu);
717
718 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
719 a5xx_gpmu_err_irq(gpu);
720
721 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
722 msm_gpu_retire(gpu);
723
724 return IRQ_HANDLED;
725}
726
727static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
728 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
729 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
730 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
731 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
732 REG_A5XX_CP_RB_RPTR_ADDR_HI),
733 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
734 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
735 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
736};
737
738static const u32 a5xx_registers[] = {
739 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
740 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
741 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
742 0x04E0, 0x0533, 0x0540, 0x0555, 0xF400, 0xF400, 0xF800, 0xF807,
743 0x0800, 0x081A, 0x081F, 0x0841, 0x0860, 0x0860, 0x0880, 0x08A0,
744 0x0B00, 0x0B12, 0x0B15, 0x0B28, 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD,
745 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 0x0C60, 0x0C61, 0x0C80, 0x0C82,
746 0x0C84, 0x0C85, 0x0C90, 0x0C98, 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2,
747 0x2180, 0x2185, 0x2580, 0x2585, 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7,
748 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8,
749 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 0x2100, 0x211E, 0x2140, 0x2145,
750 0x2500, 0x251E, 0x2540, 0x2545, 0x0D10, 0x0D17, 0x0D20, 0x0D23,
751 0x0D30, 0x0D30, 0x20C0, 0x20C0, 0x24C0, 0x24C0, 0x0E40, 0x0E43,
752 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 0x0E60, 0x0E7C, 0x0E80, 0x0E8E,
753 0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 0x0EB0, 0x0EB2, 0xE140, 0xE147,
754 0xE150, 0xE187, 0xE1A0, 0xE1A9, 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7,
755 0xE1D0, 0xE1D1, 0xE200, 0xE201, 0xE210, 0xE21C, 0xE240, 0xE268,
756 0xE000, 0xE006, 0xE010, 0xE09A, 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB,
757 0xE100, 0xE105, 0xE380, 0xE38F, 0xE3B0, 0xE3B0, 0xE400, 0xE405,
758 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 0xE280, 0xE280, 0xE282, 0xE2A3,
759 0xE2A5, 0xE2C2, 0xE940, 0xE947, 0xE950, 0xE987, 0xE9A0, 0xE9A9,
760 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 0xE9D0, 0xE9D1, 0xEA00, 0xEA01,
761 0xEA10, 0xEA1C, 0xEA40, 0xEA68, 0xE800, 0xE806, 0xE810, 0xE89A,
762 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 0xE900, 0xE905, 0xEB80, 0xEB8F,
763 0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 0xEC08, 0xECE9, 0xECF0, 0xECF0,
764 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 0xEAA5, 0xEAC2, 0xA800, 0xA8FF,
765 0xAC60, 0xAC60, 0xB000, 0xB97F, 0xB9A0, 0xB9BF,
766 ~0
767};
768
769static void a5xx_dump(struct msm_gpu *gpu)
770{
771 dev_info(gpu->dev->dev, "status: %08x\n",
772 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
773 adreno_dump(gpu);
774}
775
776static int a5xx_pm_resume(struct msm_gpu *gpu)
777{
778 int ret;
779
780
781 ret = msm_gpu_pm_resume(gpu);
782 if (ret)
783 return ret;
784
785
786 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
787
788
789 udelay(3);
790
791 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
792 (1 << 20), (1 << 20));
793 if (ret) {
794 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
795 gpu->name,
796 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
797 return ret;
798 }
799
800
801 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
802 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
803 (1 << 20), (1 << 20));
804 if (ret)
805 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
806 gpu->name);
807
808 return ret;
809}
810
811static int a5xx_pm_suspend(struct msm_gpu *gpu)
812{
813
814 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
815 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
816
817 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
818
819
820
821
822
823 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
824 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
825
826 return msm_gpu_pm_suspend(gpu);
827}
828
829static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
830{
831 *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
832 REG_A5XX_RBBM_PERFCTR_CP_0_HI);
833
834 return 0;
835}
836
837#ifdef CONFIG_DEBUG_FS
838static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
839{
840 gpu->funcs->pm_resume(gpu);
841
842 seq_printf(m, "status: %08x\n",
843 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
844 gpu->funcs->pm_suspend(gpu);
845
846 adreno_show(gpu, m);
847}
848#endif
849
850static const struct adreno_gpu_funcs funcs = {
851 .base = {
852 .get_param = adreno_get_param,
853 .hw_init = a5xx_hw_init,
854 .pm_suspend = a5xx_pm_suspend,
855 .pm_resume = a5xx_pm_resume,
856 .recover = a5xx_recover,
857 .last_fence = adreno_last_fence,
858 .submit = a5xx_submit,
859 .flush = adreno_flush,
860 .idle = a5xx_idle,
861 .irq = a5xx_irq,
862 .destroy = a5xx_destroy,
863#ifdef CONFIG_DEBUG_FS
864 .show = a5xx_show,
865#endif
866 },
867 .get_timestamp = a5xx_get_timestamp,
868};
869
870struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
871{
872 struct msm_drm_private *priv = dev->dev_private;
873 struct platform_device *pdev = priv->gpu_pdev;
874 struct a5xx_gpu *a5xx_gpu = NULL;
875 struct adreno_gpu *adreno_gpu;
876 struct msm_gpu *gpu;
877 int ret;
878
879 if (!pdev) {
880 dev_err(dev->dev, "No A5XX device is defined\n");
881 return ERR_PTR(-ENXIO);
882 }
883
884 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
885 if (!a5xx_gpu)
886 return ERR_PTR(-ENOMEM);
887
888 adreno_gpu = &a5xx_gpu->base;
889 gpu = &adreno_gpu->base;
890
891 a5xx_gpu->pdev = pdev;
892 adreno_gpu->registers = a5xx_registers;
893 adreno_gpu->reg_offsets = a5xx_register_offsets;
894
895 a5xx_gpu->lm_leakage = 0x4E001A;
896
897 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs);
898 if (ret) {
899 a5xx_destroy(&(a5xx_gpu->base.base));
900 return ERR_PTR(ret);
901 }
902
903 if (gpu->aspace)
904 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);
905
906 return gpu;
907}
908
