1
2
3
4
5#include "msm_gem.h"
6#include "msm_mmu.h"
7#include "msm_gpu_trace.h"
8#include "a6xx_gpu.h"
9#include "a6xx_gmu.xml.h"
10
11#include <linux/bitfield.h>
12#include <linux/devfreq.h>
13#include <linux/nvmem-consumer.h>
14#include <linux/soc/qcom/llcc-qcom.h>
15
16#define GPU_PAS_ID 13
17
18static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
19{
20 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
21 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
22
23
24 if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))
25 return false;
26
27
28 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
29 ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
30 return false;
31
32 return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
33 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
34}
35
36static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
37{
38
39 if (!adreno_idle(gpu, ring))
40 return false;
41
42 if (spin_until(_a6xx_check_idle(gpu))) {
43 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
44 gpu->name, __builtin_return_address(0),
45 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
46 gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
47 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
48 gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
49 return false;
50 }
51
52 return true;
53}
54
55static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
56{
57 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
58 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
59
60
61 if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) {
62 OUT_PKT7(ring, CP_WHERE_AM_I, 2);
63 OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring)));
64 OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring)));
65 }
66}
67
68static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
69{
70 uint32_t wptr;
71 unsigned long flags;
72
73 update_shadow_rptr(gpu, ring);
74
75 spin_lock_irqsave(&ring->preempt_lock, flags);
76
77
78 ring->cur = ring->next;
79
80
81 wptr = get_wptr(ring);
82
83 spin_unlock_irqrestore(&ring->preempt_lock, flags);
84
85
86 mb();
87
88 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
89}
90
91static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter,
92 u64 iova)
93{
94 OUT_PKT7(ring, CP_REG_TO_MEM, 3);
95 OUT_RING(ring, CP_REG_TO_MEM_0_REG(counter) |
96 CP_REG_TO_MEM_0_CNT(2) |
97 CP_REG_TO_MEM_0_64B);
98 OUT_RING(ring, lower_32_bits(iova));
99 OUT_RING(ring, upper_32_bits(iova));
100}
101
102static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
103 struct msm_ringbuffer *ring, struct msm_file_private *ctx)
104{
105 phys_addr_t ttbr;
106 u32 asid;
107 u64 memptr = rbmemptr(ring, ttbr0);
108
109 if (ctx->seqno == a6xx_gpu->cur_ctx_seqno)
110 return;
111
112 if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid))
113 return;
114
115
116 OUT_PKT7(ring, CP_SMMU_TABLE_UPDATE, 4);
117 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(lower_32_bits(ttbr)));
118
119 OUT_RING(ring,
120 CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(upper_32_bits(ttbr)) |
121 CP_SMMU_TABLE_UPDATE_1_ASID(asid));
122 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(0));
123 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(0));
124
125
126
127
128 OUT_PKT7(ring, CP_MEM_WRITE, 4);
129 OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr)));
130 OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr)));
131 OUT_RING(ring, lower_32_bits(ttbr));
132 OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr));
133
134
135
136
137
138
139 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
140 OUT_RING(ring, 0x31);
141
142 a6xx_gpu->cur_ctx_seqno = ctx->seqno;
143}
144
145static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
146{
147 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
148 struct msm_drm_private *priv = gpu->dev->dev_private;
149 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
150 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
151 struct msm_ringbuffer *ring = submit->ring;
152 unsigned int i, ibs = 0;
153
154 a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
155
156 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
157 rbmemptr_stats(ring, index, cpcycles_start));
158
159
160
161
162
163
164 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
165 rbmemptr_stats(ring, index, alwayson_start));
166
167
168 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
169 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_DEPTH));
170
171 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
172 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_COLOR));
173
174
175 for (i = 0; i < submit->nr_cmds; i++) {
176 switch (submit->cmd[i].type) {
177 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
178 break;
179 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
180 if (priv->lastctx == submit->queue->ctx)
181 break;
182 fallthrough;
183 case MSM_SUBMIT_CMD_BUF:
184 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
185 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
186 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
187 OUT_RING(ring, submit->cmd[i].size);
188 ibs++;
189 break;
190 }
191
192
193
194
195
196
197
198
199 if ((ibs % 32) == 0)
200 update_shadow_rptr(gpu, ring);
201 }
202
203 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
204 rbmemptr_stats(ring, index, cpcycles_end));
205 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
206 rbmemptr_stats(ring, index, alwayson_end));
207
208
209 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
210 OUT_RING(ring, submit->seqno);
211
212
213
214
215
216 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
217 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
218 CP_EVENT_WRITE_0_IRQ);
219 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
220 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
221 OUT_RING(ring, submit->seqno);
222
223 trace_msm_gpu_submit_flush(submit,
224 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
225 REG_A6XX_CP_ALWAYS_ON_COUNTER_HI));
226
227 a6xx_flush(gpu, ring);
228}
229
230const struct adreno_reglist a630_hwcg[] = {
231 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
232 {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
233 {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
234 {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
235 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
236 {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
237 {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
238 {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
239 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
240 {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
241 {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
242 {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
243 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
244 {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
245 {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
246 {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
247 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
248 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
249 {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
250 {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
251 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
252 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
253 {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
254 {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
255 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
256 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
257 {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
258 {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
259 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
260 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
261 {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
262 {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
263 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
264 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
265 {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
266 {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
267 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
268 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
269 {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
270 {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
271 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
272 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
273 {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
274 {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
275 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
276 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
277 {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
278 {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
279 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
280 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
281 {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
282 {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
283 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
284 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
285 {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
286 {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
287 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
288 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
289 {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
290 {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
291 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
292 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
293 {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
294 {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
295 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
296 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
297 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
298 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
299 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
300 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
301 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
302 {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
303 {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
304 {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
305 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
306 {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
307 {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
308 {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
309 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
310 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
311 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
312 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
313 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
314 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
315 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
316 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
317 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
318 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
319 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
320 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
321 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
322 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
323 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
324 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
325 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
326 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
327 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
328 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
329 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
330 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
331 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
332 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
333 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
334 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
335 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
336 {},
337};
338
339const struct adreno_reglist a640_hwcg[] = {
340 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
341 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
342 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
343 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
344 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
345 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
346 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
347 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
348 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
349 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
350 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
351 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
352 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
353 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
354 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
355 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
356 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
357 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
358 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
359 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
360 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022},
361 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
362 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
363 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
364 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
365 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
366 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
367 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
368 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
369 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
370 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
371 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
372 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
373 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
374 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
375 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
376 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
377 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
378 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
379 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
380 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
381 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
382 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
383 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
384 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
385 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
386 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
387 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
388 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
389 {},
390};
391
392const struct adreno_reglist a650_hwcg[] = {
393 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
394 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
395 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
396 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
397 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
398 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
399 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
400 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
401 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
402 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
403 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
404 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
405 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
406 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
407 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
408 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
409 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
410 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
411 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
412 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
413 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
414 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
415 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
416 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
417 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
418 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
419 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
420 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
421 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
422 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
423 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
424 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
425 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
426 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
427 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
428 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
429 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
430 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
431 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
432 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777},
433 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
434 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
435 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
436 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
437 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
438 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
439 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
440 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
441 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
442 {},
443};
444
445const struct adreno_reglist a660_hwcg[] = {
446 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
447 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
448 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
449 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
450 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
451 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
452 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
453 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
454 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
455 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
456 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
457 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
458 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
459 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
460 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
461 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
462 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
463 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
464 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
465 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
466 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
467 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
468 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
469 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
470 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
471 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
472 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
473 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
474 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
475 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
476 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
477 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
478 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
479 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
480 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
481 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
482 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
483 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
484 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
485 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
486 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
487 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
488 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
489 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
490 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
491 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
492 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
493 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
494 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
495 {},
496};
497
498static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
499{
500 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
501 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
502 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
503 const struct adreno_reglist *reg;
504 unsigned int i;
505 u32 val, clock_cntl_on;
506
507 if (!adreno_gpu->info->hwcg)
508 return;
509
510 if (adreno_is_a630(adreno_gpu))
511 clock_cntl_on = 0x8aa8aa02;
512 else
513 clock_cntl_on = 0x8aa8aa82;
514
515 val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
516
517
518 if ((!state && !val) || (state && (val == clock_cntl_on)))
519 return;
520
521
522 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
523
524 for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++)
525 gpu_write(gpu, reg->offset, state ? reg->value : 0);
526
527
528 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
529
530 gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? clock_cntl_on : 0);
531}
532
533
534static const u32 a6xx_protect[] = {
535 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
536 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
537 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
538 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
539 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
540 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
541 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
542 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
543 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
544 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
545 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
546 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
547 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
548 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
549 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
550 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
551 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
552 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
553 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
554 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
555 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
556 A6XX_PROTECT_NORDWR(0x09e70, 0x0001),
557 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
558 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
559 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
560 A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
561 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
562 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
563 A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
564 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
565 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
566 A6XX_PROTECT_NORDWR(0x11c00, 0x0000),
567};
568
569
570static const u32 a650_protect[] = {
571 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
572 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
573 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
574 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
575 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
576 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
577 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
578 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
579 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
580 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
581 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
582 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
583 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
584 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
585 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
586 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
587 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
588 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
589 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
590 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
591 A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
592 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
593 A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
594 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
595 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
596 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
597 A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
598 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
599 A6XX_PROTECT_NORDWR(0x0b608, 0x0007),
600 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
601 A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
602 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
603 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
604 A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
605 A6XX_PROTECT_NORDWR(0x1a800, 0x1fff),
606 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
607 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
608 A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
609 A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000),
610};
611
612
613static const u32 a660_protect[] = {
614 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
615 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
616 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
617 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
618 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
619 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
620 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
621 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
622 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
623 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
624 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
625 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
626 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
627 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
628 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
629 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
630 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
631 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
632 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
633 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
634 A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
635 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
636 A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
637 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
638 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
639 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
640 A6XX_PROTECT_NORDWR(0x0ae50, 0x012f),
641 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
642 A6XX_PROTECT_NORDWR(0x0b608, 0x0006),
643 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
644 A6XX_PROTECT_NORDWR(0x0be20, 0x015f),
645 A6XX_PROTECT_NORDWR(0x0d000, 0x05ff),
646 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
647 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
648 A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
649 A6XX_PROTECT_NORDWR(0x1a400, 0x1fff),
650 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
651 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
652 A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
653 A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
654 A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000),
655};
656
657static void a6xx_set_cp_protect(struct msm_gpu *gpu)
658{
659 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
660 const u32 *regs = a6xx_protect;
661 unsigned i, count = ARRAY_SIZE(a6xx_protect), count_max = 32;
662
663 BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
664 BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
665
666 if (adreno_is_a650(adreno_gpu)) {
667 regs = a650_protect;
668 count = ARRAY_SIZE(a650_protect);
669 count_max = 48;
670 } else if (adreno_is_a660_family(adreno_gpu)) {
671 regs = a660_protect;
672 count = ARRAY_SIZE(a660_protect);
673 count_max = 48;
674 }
675
676
677
678
679
680
681 gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, BIT(0) | BIT(1) | BIT(3));
682
683 for (i = 0; i < count - 1; i++)
684 gpu_write(gpu, REG_A6XX_CP_PROTECT(i), regs[i]);
685
686 gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]);
687}
688
689static void a6xx_set_ubwc_config(struct msm_gpu *gpu)
690{
691 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
692 u32 lower_bit = 2;
693 u32 amsbc = 0;
694 u32 rgb565_predicator = 0;
695 u32 uavflagprd_inv = 0;
696
697
698 if (adreno_is_a618(adreno_gpu))
699 return;
700
701 if (adreno_is_a640_family(adreno_gpu))
702 amsbc = 1;
703
704 if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu)) {
705
706 lower_bit = 3;
707 amsbc = 1;
708 rgb565_predicator = 1;
709 uavflagprd_inv = 2;
710 }
711
712 if (adreno_is_7c3(adreno_gpu)) {
713 lower_bit = 1;
714 amsbc = 1;
715 rgb565_predicator = 1;
716 uavflagprd_inv = 2;
717 }
718
719 gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL,
720 rgb565_predicator << 11 | amsbc << 4 | lower_bit << 1);
721 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, lower_bit << 1);
722 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL,
723 uavflagprd_inv << 4 | lower_bit << 1);
724 gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, lower_bit << 21);
725}
726
727static int a6xx_cp_init(struct msm_gpu *gpu)
728{
729 struct msm_ringbuffer *ring = gpu->rb[0];
730
731 OUT_PKT7(ring, CP_ME_INIT, 8);
732
733 OUT_RING(ring, 0x0000002f);
734
735
736 OUT_RING(ring, 0x00000003);
737
738
739 OUT_RING(ring, 0x20000000);
740
741
742 OUT_RING(ring, 0x00000000);
743 OUT_RING(ring, 0x00000000);
744
745
746 OUT_RING(ring, 0x00000000);
747
748
749 OUT_RING(ring, 0x00000000);
750 OUT_RING(ring, 0x00000000);
751
752 a6xx_flush(gpu, ring);
753 return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
754}
755
756
757
758
759
760static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu,
761 struct drm_gem_object *obj)
762{
763 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
764 struct msm_gpu *gpu = &adreno_gpu->base;
765 const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE];
766 u32 *buf = msm_gem_get_vaddr(obj);
767 bool ret = false;
768
769 if (IS_ERR(buf))
770 return false;
771
772
773
774
775
776
777
778
779
780
781
782 if (!strcmp(sqe_name, "a630_sqe.fw")) {
783
784
785
786
787
788
789
790
791
792
793 if ((((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) ||
794 (buf[0] & 0xfff) >= 0x190) {
795 a6xx_gpu->has_whereami = true;
796 ret = true;
797 goto out;
798 }
799
800 DRM_DEV_ERROR(&gpu->pdev->dev,
801 "a630 SQE ucode is too old. Have version %x need at least %x\n",
802 buf[0] & 0xfff, 0x190);
803 } else if (!strcmp(sqe_name, "a650_sqe.fw")) {
804 if ((buf[0] & 0xfff) >= 0x095) {
805 ret = true;
806 goto out;
807 }
808
809 DRM_DEV_ERROR(&gpu->pdev->dev,
810 "a650 SQE ucode is too old. Have version %x need at least %x\n",
811 buf[0] & 0xfff, 0x095);
812 } else if (!strcmp(sqe_name, "a660_sqe.fw")) {
813 ret = true;
814 } else {
815 DRM_DEV_ERROR(&gpu->pdev->dev,
816 "unknown GPU, add it to a6xx_ucode_check_version()!!\n");
817 }
818out:
819 msm_gem_put_vaddr(obj);
820 return ret;
821}
822
823static int a6xx_ucode_init(struct msm_gpu *gpu)
824{
825 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
826 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
827
828 if (!a6xx_gpu->sqe_bo) {
829 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
830 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
831
832 if (IS_ERR(a6xx_gpu->sqe_bo)) {
833 int ret = PTR_ERR(a6xx_gpu->sqe_bo);
834
835 a6xx_gpu->sqe_bo = NULL;
836 DRM_DEV_ERROR(&gpu->pdev->dev,
837 "Could not allocate SQE ucode: %d\n", ret);
838
839 return ret;
840 }
841
842 msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw");
843 if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) {
844 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
845 drm_gem_object_put(a6xx_gpu->sqe_bo);
846
847 a6xx_gpu->sqe_bo = NULL;
848 return -EPERM;
849 }
850 }
851
852 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE,
853 REG_A6XX_CP_SQE_INSTR_BASE+1, a6xx_gpu->sqe_iova);
854
855 return 0;
856}
857
858static int a6xx_zap_shader_init(struct msm_gpu *gpu)
859{
860 static bool loaded;
861 int ret;
862
863 if (loaded)
864 return 0;
865
866 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
867
868 loaded = !ret;
869 return ret;
870}
871
872#define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
873 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
874 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
875 A6XX_RBBM_INT_0_MASK_CP_IB2 | \
876 A6XX_RBBM_INT_0_MASK_CP_IB1 | \
877 A6XX_RBBM_INT_0_MASK_CP_RB | \
878 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
879 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
880 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
881 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
882 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
883
884static int hw_init(struct msm_gpu *gpu)
885{
886 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
887 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
888 int ret;
889
890
891 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
892
893 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
894
895
896
897
898
899
900 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
901 REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
902 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
903
904
905 gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1);
906 gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1);
907 gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1);
908 gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1);
909 gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1);
910 gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1);
911 gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1);
912 gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1);
913 gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1);
914 gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1);
915 gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1);
916 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
917
918
919 a6xx_set_hwcg(gpu, true);
920
921
922 if (adreno_is_a640_family(adreno_gpu) ||
923 adreno_is_a650_family(adreno_gpu)) {
924 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620);
925 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620);
926 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620);
927 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
928 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
929 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3);
930 } else {
931 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
932 }
933
934 if (adreno_is_a630(adreno_gpu))
935 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
936
937
938 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
939
940
941 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0);
942 gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff);
943 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000);
944 gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
945 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
946 gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
947
948 if (!adreno_is_a650_family(adreno_gpu)) {
949
950 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
951 REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
952
953 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
954 REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
955 0x00100000 + adreno_gpu->gmem - 1);
956 }
957
958 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
959 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
960
961 if (adreno_is_a640_family(adreno_gpu) ||
962 adreno_is_a650_family(adreno_gpu))
963 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
964 else
965 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
966 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
967
968 if (adreno_is_a660_family(adreno_gpu))
969 gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020);
970
971
972 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
973
974
975
976
977 if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
978 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
979 else if (adreno_is_a640_family(adreno_gpu) || adreno_is_7c3(adreno_gpu))
980 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200);
981 else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
982 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
983 else
984 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000);
985
986
987 gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
988
989
990 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
991
992
993 gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL(0), PERF_CP_ALWAYS_COUNT);
994
995 a6xx_set_ubwc_config(gpu);
996
997
998 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL,
999 (1 << 30) | 0x1fffff);
1000
1001 gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);
1002
1003
1004 if (adreno_is_a650_family(adreno_gpu)) {
1005 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0);
1006 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1,
1007 0x3fe05ff4);
1008 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2,
1009 0x3fa0ebee);
1010 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3,
1011 0x3f5193ed);
1012 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4,
1013 0x3f0243f0);
1014 }
1015
1016
1017 a6xx_set_cp_protect(gpu);
1018
1019 if (adreno_is_a660_family(adreno_gpu)) {
1020 gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1);
1021 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0);
1022 }
1023
1024
1025 if (adreno_is_a660(adreno_gpu))
1026 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906);
1027
1028
1029 if (gpu->hw_apriv) {
1030 gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1031 (1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1));
1032 }
1033
1034
1035 gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);
1036
1037 ret = adreno_hw_init(gpu);
1038 if (ret)
1039 goto out;
1040
1041 ret = a6xx_ucode_init(gpu);
1042 if (ret)
1043 goto out;
1044
1045
1046 gpu_write64(gpu, REG_A6XX_CP_RB_BASE, REG_A6XX_CP_RB_BASE_HI,
1047 gpu->rb[0]->iova);
1048
1049
1050
1051
1052
1053 if (adreno_gpu->base.hw_apriv)
1054 gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT);
1055 else
1056 gpu_write(gpu, REG_A6XX_CP_RB_CNTL,
1057 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
1058
1059
1060
1061
1062
1063
1064 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) {
1065 if (!a6xx_gpu->shadow_bo) {
1066 a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
1067 sizeof(u32) * gpu->nr_rings,
1068 MSM_BO_WC | MSM_BO_MAP_PRIV,
1069 gpu->aspace, &a6xx_gpu->shadow_bo,
1070 &a6xx_gpu->shadow_iova);
1071
1072 if (IS_ERR(a6xx_gpu->shadow))
1073 return PTR_ERR(a6xx_gpu->shadow);
1074 }
1075
1076 gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR_LO,
1077 REG_A6XX_CP_RB_RPTR_ADDR_HI,
1078 shadowptr(a6xx_gpu, gpu->rb[0]));
1079 }
1080
1081
1082 a6xx_gpu->cur_ring = gpu->rb[0];
1083
1084 a6xx_gpu->cur_ctx_seqno = 0;
1085
1086
1087 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
1088
1089 ret = a6xx_cp_init(gpu);
1090 if (ret)
1091 goto out;
1092
1093
1094
1095
1096
1097
1098
1099
1100 ret = a6xx_zap_shader_init(gpu);
1101 if (!ret) {
1102 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
1103 OUT_RING(gpu->rb[0], 0x00000000);
1104
1105 a6xx_flush(gpu, gpu->rb[0]);
1106 if (!a6xx_idle(gpu, gpu->rb[0]))
1107 return -EINVAL;
1108 } else if (ret == -ENODEV) {
1109
1110
1111
1112
1113
1114
1115 dev_warn_once(gpu->dev->dev,
1116 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
1117 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
1118 ret = 0;
1119 } else {
1120 return ret;
1121 }
1122
1123out:
1124
1125
1126
1127
1128 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
1129
1130 if (a6xx_gpu->gmu.legacy) {
1131
1132 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
1133 }
1134
1135 return ret;
1136}
1137
1138static int a6xx_hw_init(struct msm_gpu *gpu)
1139{
1140 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1141 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1142 int ret;
1143
1144 mutex_lock(&a6xx_gpu->gmu.lock);
1145 ret = hw_init(gpu);
1146 mutex_unlock(&a6xx_gpu->gmu.lock);
1147
1148 return ret;
1149}
1150
1151static void a6xx_dump(struct msm_gpu *gpu)
1152{
1153 DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
1154 gpu_read(gpu, REG_A6XX_RBBM_STATUS));
1155 adreno_dump(gpu);
1156}
1157
1158#define VBIF_RESET_ACK_TIMEOUT 100
1159#define VBIF_RESET_ACK_MASK 0x00f0
1160
1161static void a6xx_recover(struct msm_gpu *gpu)
1162{
1163 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1164 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1165 int i;
1166
1167 adreno_dump_info(gpu);
1168
1169 for (i = 0; i < 8; i++)
1170 DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
1171 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
1172
1173 if (hang_debug)
1174 a6xx_dump(gpu);
1175
1176
1177
1178
1179
1180 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
1181
1182 gpu->funcs->pm_suspend(gpu);
1183 gpu->funcs->pm_resume(gpu);
1184
1185 msm_gpu_hw_init(gpu);
1186}
1187
1188static const char *a6xx_uche_fault_block(struct msm_gpu *gpu, u32 mid)
1189{
1190 static const char *uche_clients[7] = {
1191 "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ",
1192 };
1193 u32 val;
1194
1195 if (mid < 1 || mid > 3)
1196 return "UNKNOWN";
1197
1198
1199
1200
1201
1202 val = gpu_read(gpu, REG_A6XX_UCHE_CLIENT_PF);
1203
1204
1205 if (mid == 3)
1206 return uche_clients[val & 7];
1207
1208
1209 if (mid == 2)
1210 return ((val & 7) == 0) ? "TP" : "TP|VFD";
1211
1212
1213 return "UCHE";
1214}
1215
1216static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id)
1217{
1218 if (id == 0)
1219 return "CP";
1220 else if (id == 4)
1221 return "CCU";
1222 else if (id == 6)
1223 return "CDP Prefetch";
1224
1225 return a6xx_uche_fault_block(gpu, id);
1226}
1227
1228#define ARM_SMMU_FSR_TF BIT(1)
1229#define ARM_SMMU_FSR_PF BIT(3)
1230#define ARM_SMMU_FSR_EF BIT(4)
1231
1232static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
1233{
1234 struct msm_gpu *gpu = arg;
1235 struct adreno_smmu_fault_info *info = data;
1236 const char *type = "UNKNOWN";
1237 const char *block;
1238 bool do_devcoredump = info && !READ_ONCE(gpu->crashstate);
1239
1240
1241
1242
1243
1244 if (!do_devcoredump) {
1245 gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu);
1246 }
1247
1248
1249
1250
1251
1252 if (!info) {
1253 pr_warn_ratelimited("*** gpu fault: iova=%.16lx flags=%d (%u,%u,%u,%u)\n",
1254 iova, flags,
1255 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
1256 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
1257 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
1258 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
1259
1260 return 0;
1261 }
1262
1263 if (info->fsr & ARM_SMMU_FSR_TF)
1264 type = "TRANSLATION";
1265 else if (info->fsr & ARM_SMMU_FSR_PF)
1266 type = "PERMISSION";
1267 else if (info->fsr & ARM_SMMU_FSR_EF)
1268 type = "EXTERNAL";
1269
1270 block = a6xx_fault_block(gpu, info->fsynr1 & 0xff);
1271
1272 pr_warn_ratelimited("*** gpu fault: ttbr0=%.16llx iova=%.16lx dir=%s type=%s source=%s (%u,%u,%u,%u)\n",
1273 info->ttbr0, iova,
1274 flags & IOMMU_FAULT_WRITE ? "WRITE" : "READ",
1275 type, block,
1276 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
1277 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
1278 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
1279 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)));
1280
1281 if (do_devcoredump) {
1282
1283 del_timer(&gpu->hangcheck_timer);
1284
1285 gpu->fault_info.ttbr0 = info->ttbr0;
1286 gpu->fault_info.iova = iova;
1287 gpu->fault_info.flags = flags;
1288 gpu->fault_info.type = type;
1289 gpu->fault_info.block = block;
1290
1291 kthread_queue_work(gpu->worker, &gpu->fault_work);
1292 }
1293
1294 return 0;
1295}
1296
1297static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
1298{
1299 u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
1300
1301 if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
1302 u32 val;
1303
1304 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
1305 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
1306 dev_err_ratelimited(&gpu->pdev->dev,
1307 "CP | opcode error | possible opcode=0x%8.8X\n",
1308 val);
1309 }
1310
1311 if (status & A6XX_CP_INT_CP_UCODE_ERROR)
1312 dev_err_ratelimited(&gpu->pdev->dev,
1313 "CP ucode error interrupt\n");
1314
1315 if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
1316 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
1317 gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
1318
1319 if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
1320 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
1321
1322 dev_err_ratelimited(&gpu->pdev->dev,
1323 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
1324 val & (1 << 20) ? "READ" : "WRITE",
1325 (val & 0x3ffff), val);
1326 }
1327
1328 if (status & A6XX_CP_INT_CP_AHB_ERROR)
1329 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
1330
1331 if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
1332 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
1333
1334 if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
1335 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
1336
1337}
1338
1339static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
1340{
1341 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1342 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1343 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1344
1345
1346
1347
1348
1349
1350
1351 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS3) & A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT)
1352 return;
1353
1354
1355
1356
1357
1358 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
1359
1360 DRM_DEV_ERROR(&gpu->pdev->dev,
1361 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1362 ring ? ring->id : -1, ring ? ring->seqno : 0,
1363 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
1364 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
1365 gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
1366 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI),
1367 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
1368 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI),
1369 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
1370
1371
1372 del_timer(&gpu->hangcheck_timer);
1373
1374 kthread_queue_work(gpu->worker, &gpu->recover_work);
1375}
1376
1377static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
1378{
1379 u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
1380
1381 gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
1382
1383 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
1384 a6xx_fault_detect_irq(gpu);
1385
1386 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
1387 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
1388
1389 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1390 a6xx_cp_hw_err_irq(gpu);
1391
1392 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
1393 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
1394
1395 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1396 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
1397
1398 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1399 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
1400
1401 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
1402 msm_gpu_retire(gpu);
1403
1404 return IRQ_HANDLED;
1405}
1406
1407static void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or)
1408{
1409 return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or);
1410}
1411
1412static void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value)
1413{
1414 return msm_writel(value, a6xx_gpu->llc_mmio + (reg << 2));
1415}
1416
1417static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu)
1418{
1419 llcc_slice_deactivate(a6xx_gpu->llc_slice);
1420 llcc_slice_deactivate(a6xx_gpu->htw_llc_slice);
1421}
1422
1423static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
1424{
1425 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1426 struct msm_gpu *gpu = &adreno_gpu->base;
1427 u32 gpu_scid, cntl1_regval = 0;
1428
1429 if (IS_ERR(a6xx_gpu->llc_mmio))
1430 return;
1431
1432 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
1433 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
1434
1435 gpu_scid &= 0x1f;
1436 cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) |
1437 (gpu_scid << 15) | (gpu_scid << 20);
1438 }
1439
1440
1441
1442
1443
1444 if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) {
1445 if (!a6xx_gpu->have_mmu500) {
1446 u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice);
1447
1448 gpuhtw_scid &= 0x1f;
1449 cntl1_regval |= FIELD_PREP(GENMASK(29, 25), gpuhtw_scid);
1450 }
1451 }
1452
1453 if (!cntl1_regval)
1454 return;
1455
1456
1457
1458
1459
1460 if (!a6xx_gpu->have_mmu500) {
1461 a6xx_llc_write(a6xx_gpu,
1462 REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval);
1463
1464
1465
1466
1467
1468 a6xx_llc_rmw(a6xx_gpu,
1469 REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03);
1470 return;
1471 }
1472
1473 gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval);
1474
1475
1476
1477
1478 if (adreno_is_a660_family(adreno_gpu))
1479 gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
1480 (1 << 8), (gpu_scid << 10) | (1 << 8));
1481}
1482
1483static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)
1484{
1485 llcc_slice_putd(a6xx_gpu->llc_slice);
1486 llcc_slice_putd(a6xx_gpu->htw_llc_slice);
1487}
1488
1489static void a6xx_llc_slices_init(struct platform_device *pdev,
1490 struct a6xx_gpu *a6xx_gpu)
1491{
1492 struct device_node *phandle;
1493
1494
1495
1496
1497
1498 phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0);
1499 a6xx_gpu->have_mmu500 = (phandle &&
1500 of_device_is_compatible(phandle, "arm,mmu-500"));
1501 of_node_put(phandle);
1502
1503 if (a6xx_gpu->have_mmu500)
1504 a6xx_gpu->llc_mmio = NULL;
1505 else
1506 a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem", "gpu_cx");
1507
1508 a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU);
1509 a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW);
1510
1511 if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))
1512 a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL);
1513}
1514
1515static int a6xx_pm_resume(struct msm_gpu *gpu)
1516{
1517 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1518 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1519 int ret;
1520
1521 gpu->needs_hw_init = true;
1522
1523 trace_msm_gpu_resume(0);
1524
1525 mutex_lock(&a6xx_gpu->gmu.lock);
1526 ret = a6xx_gmu_resume(a6xx_gpu);
1527 mutex_unlock(&a6xx_gpu->gmu.lock);
1528 if (ret)
1529 return ret;
1530
1531 msm_devfreq_resume(gpu);
1532
1533 a6xx_llc_activate(a6xx_gpu);
1534
1535 return 0;
1536}
1537
1538static int a6xx_pm_suspend(struct msm_gpu *gpu)
1539{
1540 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1541 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1542 int i, ret;
1543
1544 trace_msm_gpu_suspend(0);
1545
1546 a6xx_llc_deactivate(a6xx_gpu);
1547
1548 msm_devfreq_suspend(gpu);
1549
1550 mutex_lock(&a6xx_gpu->gmu.lock);
1551 ret = a6xx_gmu_stop(a6xx_gpu);
1552 mutex_unlock(&a6xx_gpu->gmu.lock);
1553 if (ret)
1554 return ret;
1555
1556 if (a6xx_gpu->shadow_bo)
1557 for (i = 0; i < gpu->nr_rings; i++)
1558 a6xx_gpu->shadow[i] = 0;
1559
1560 return 0;
1561}
1562
1563static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1564{
1565 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1566 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1567
1568 mutex_lock(&a6xx_gpu->gmu.lock);
1569
1570
1571 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
1572
1573 *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
1574 REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
1575
1576 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
1577
1578 mutex_unlock(&a6xx_gpu->gmu.lock);
1579
1580 return 0;
1581}
1582
1583static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
1584{
1585 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1586 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1587
1588 return a6xx_gpu->cur_ring;
1589}
1590
1591static void a6xx_destroy(struct msm_gpu *gpu)
1592{
1593 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1594 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1595
1596 if (a6xx_gpu->sqe_bo) {
1597 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
1598 drm_gem_object_put(a6xx_gpu->sqe_bo);
1599 }
1600
1601 if (a6xx_gpu->shadow_bo) {
1602 msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace);
1603 drm_gem_object_put(a6xx_gpu->shadow_bo);
1604 }
1605
1606 a6xx_llc_slices_destroy(a6xx_gpu);
1607
1608 a6xx_gmu_remove(a6xx_gpu);
1609
1610 adreno_gpu_cleanup(adreno_gpu);
1611
1612 kfree(a6xx_gpu);
1613}
1614
1615static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
1616{
1617 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1618 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1619 u64 busy_cycles, busy_time;
1620
1621
1622
1623 if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
1624 return 0;
1625
1626 busy_cycles = gmu_read64(&a6xx_gpu->gmu,
1627 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
1628 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
1629
1630 busy_time = (busy_cycles - gpu->devfreq.busy_cycles) * 10;
1631 do_div(busy_time, 192);
1632
1633 gpu->devfreq.busy_cycles = busy_cycles;
1634
1635 pm_runtime_put(a6xx_gpu->gmu.dev);
1636
1637 if (WARN_ON(busy_time > ~0LU))
1638 return ~0LU;
1639
1640 return (unsigned long)busy_time;
1641}
1642
1643void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
1644{
1645 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1646 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1647
1648 mutex_lock(&a6xx_gpu->gmu.lock);
1649 a6xx_gmu_set_freq(gpu, opp);
1650 mutex_unlock(&a6xx_gpu->gmu.lock);
1651}
1652
1653static struct msm_gem_address_space *
1654a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
1655{
1656 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1657 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1658 struct iommu_domain *iommu;
1659 struct msm_mmu *mmu;
1660 struct msm_gem_address_space *aspace;
1661 u64 start, size;
1662
1663 iommu = iommu_domain_alloc(&platform_bus_type);
1664 if (!iommu)
1665 return NULL;
1666
1667
1668
1669
1670
1671 if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))
1672 adreno_set_llc_attributes(iommu);
1673
1674 mmu = msm_iommu_new(&pdev->dev, iommu);
1675 if (IS_ERR(mmu)) {
1676 iommu_domain_free(iommu);
1677 return ERR_CAST(mmu);
1678 }
1679
1680
1681
1682
1683
1684
1685 start = max_t(u64, SZ_16M, iommu->geometry.aperture_start);
1686 size = iommu->geometry.aperture_end - start + 1;
1687
1688 aspace = msm_gem_address_space_create(mmu, "gpu",
1689 start & GENMASK_ULL(48, 0), size);
1690
1691 if (IS_ERR(aspace) && !IS_ERR(mmu))
1692 mmu->funcs->destroy(mmu);
1693
1694 return aspace;
1695}
1696
1697static struct msm_gem_address_space *
1698a6xx_create_private_address_space(struct msm_gpu *gpu)
1699{
1700 struct msm_mmu *mmu;
1701
1702 mmu = msm_iommu_pagetable_create(gpu->aspace->mmu);
1703
1704 if (IS_ERR(mmu))
1705 return ERR_CAST(mmu);
1706
1707 return msm_gem_address_space_create(mmu,
1708 "gpu", 0x100000000ULL, 0x1ffffffffULL);
1709}
1710
1711static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1712{
1713 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1714 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1715
1716 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami)
1717 return a6xx_gpu->shadow[ring->id];
1718
1719 return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR);
1720}
1721
1722static u32 a618_get_speed_bin(u32 fuse)
1723{
1724 if (fuse == 0)
1725 return 0;
1726 else if (fuse == 169)
1727 return 1;
1728 else if (fuse == 174)
1729 return 2;
1730
1731 return UINT_MAX;
1732}
1733
1734static u32 fuse_to_supp_hw(struct device *dev, struct adreno_rev rev, u32 fuse)
1735{
1736 u32 val = UINT_MAX;
1737
1738 if (adreno_cmp_rev(ADRENO_REV(6, 1, 8, ANY_ID), rev))
1739 val = a618_get_speed_bin(fuse);
1740
1741 if (val == UINT_MAX) {
1742 DRM_DEV_ERROR(dev,
1743 "missing support for speed-bin: %u. Some OPPs may not be supported by hardware",
1744 fuse);
1745 return UINT_MAX;
1746 }
1747
1748 return (1 << val);
1749}
1750
1751static int a6xx_set_supported_hw(struct device *dev, struct adreno_rev rev)
1752{
1753 u32 supp_hw = UINT_MAX;
1754 u32 speedbin;
1755 int ret;
1756
1757 ret = nvmem_cell_read_variable_le_u32(dev, "speed_bin", &speedbin);
1758
1759
1760
1761
1762 if (ret == -ENOENT) {
1763 return 0;
1764 } else if (ret) {
1765 DRM_DEV_ERROR(dev,
1766 "failed to read speed-bin (%d). Some OPPs may not be supported by hardware",
1767 ret);
1768 goto done;
1769 }
1770
1771 supp_hw = fuse_to_supp_hw(dev, rev, speedbin);
1772
1773done:
1774 ret = devm_pm_opp_set_supported_hw(dev, &supp_hw, 1);
1775 if (ret)
1776 return ret;
1777
1778 return 0;
1779}
1780
1781static const struct adreno_gpu_funcs funcs = {
1782 .base = {
1783 .get_param = adreno_get_param,
1784 .hw_init = a6xx_hw_init,
1785 .pm_suspend = a6xx_pm_suspend,
1786 .pm_resume = a6xx_pm_resume,
1787 .recover = a6xx_recover,
1788 .submit = a6xx_submit,
1789 .active_ring = a6xx_active_ring,
1790 .irq = a6xx_irq,
1791 .destroy = a6xx_destroy,
1792#if defined(CONFIG_DRM_MSM_GPU_STATE)
1793 .show = a6xx_show,
1794#endif
1795 .gpu_busy = a6xx_gpu_busy,
1796 .gpu_get_freq = a6xx_gmu_get_freq,
1797 .gpu_set_freq = a6xx_gpu_set_freq,
1798#if defined(CONFIG_DRM_MSM_GPU_STATE)
1799 .gpu_state_get = a6xx_gpu_state_get,
1800 .gpu_state_put = a6xx_gpu_state_put,
1801#endif
1802 .create_address_space = a6xx_create_address_space,
1803 .create_private_address_space = a6xx_create_private_address_space,
1804 .get_rptr = a6xx_get_rptr,
1805 },
1806 .get_timestamp = a6xx_get_timestamp,
1807};
1808
1809struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
1810{
1811 struct msm_drm_private *priv = dev->dev_private;
1812 struct platform_device *pdev = priv->gpu_pdev;
1813 struct adreno_platform_config *config = pdev->dev.platform_data;
1814 const struct adreno_info *info;
1815 struct device_node *node;
1816 struct a6xx_gpu *a6xx_gpu;
1817 struct adreno_gpu *adreno_gpu;
1818 struct msm_gpu *gpu;
1819 int ret;
1820
1821 a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
1822 if (!a6xx_gpu)
1823 return ERR_PTR(-ENOMEM);
1824
1825 adreno_gpu = &a6xx_gpu->base;
1826 gpu = &adreno_gpu->base;
1827
1828 adreno_gpu->registers = NULL;
1829
1830
1831
1832
1833
1834
1835 info = adreno_info(config->rev);
1836
1837 if (info && (info->revn == 650 || info->revn == 660 ||
1838 adreno_cmp_rev(ADRENO_REV(6, 3, 5, ANY_ID), info->rev)))
1839 adreno_gpu->base.hw_apriv = true;
1840
1841
1842
1843
1844
1845 if (info && (info->revn == 618))
1846 gpu->clamp_to_idle = true;
1847
1848 a6xx_llc_slices_init(pdev, a6xx_gpu);
1849
1850 ret = a6xx_set_supported_hw(&pdev->dev, config->rev);
1851 if (ret) {
1852 a6xx_destroy(&(a6xx_gpu->base.base));
1853 return ERR_PTR(ret);
1854 }
1855
1856 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
1857 if (ret) {
1858 a6xx_destroy(&(a6xx_gpu->base.base));
1859 return ERR_PTR(ret);
1860 }
1861
1862
1863 node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
1864
1865
1866 BUG_ON(!node);
1867
1868 ret = a6xx_gmu_init(a6xx_gpu, node);
1869 if (ret) {
1870 a6xx_destroy(&(a6xx_gpu->base.base));
1871 return ERR_PTR(ret);
1872 }
1873
1874 if (gpu->aspace)
1875 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
1876 a6xx_fault_handler);
1877
1878 return gpu;
1879}
1880
