1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/firmware.h>
26#include <linux/pci.h>
27#include <linux/slab.h>
28
29#include <drm/drm_vblank.h>
30#include <drm/radeon_drm.h>
31#include <drm/drm_fourcc.h>
32
33#include "atom.h"
34#include "avivod.h"
35#include "cik.h"
36#include "ni.h"
37#include "rv770.h"
38#include "evergreen.h"
39#include "evergreen_blit_shaders.h"
40#include "evergreen_reg.h"
41#include "evergreend.h"
42#include "radeon.h"
43#include "radeon_asic.h"
44#include "radeon_audio.h"
45#include "radeon_ucode.h"
46#include "si.h"
47
48#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
49#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
50#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
51
52
53
54
55u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
56{
57 unsigned long flags;
58 u32 r;
59
60 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
61 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
62 r = RREG32(EVERGREEN_CG_IND_DATA);
63 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
64 return r;
65}
66
67void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
68{
69 unsigned long flags;
70
71 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
72 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
73 WREG32(EVERGREEN_CG_IND_DATA, (v));
74 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
75}
76
77u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
78{
79 unsigned long flags;
80 u32 r;
81
82 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
83 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
84 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
85 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
86 return r;
87}
88
89void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
90{
91 unsigned long flags;
92
93 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
94 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
95 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
96 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
97}
98
99u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
100{
101 unsigned long flags;
102 u32 r;
103
104 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
105 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
106 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
107 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
108 return r;
109}
110
111void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
112{
113 unsigned long flags;
114
115 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
116 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
117 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
118 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
119}
120
121static const u32 crtc_offsets[6] =
122{
123 EVERGREEN_CRTC0_REGISTER_OFFSET,
124 EVERGREEN_CRTC1_REGISTER_OFFSET,
125 EVERGREEN_CRTC2_REGISTER_OFFSET,
126 EVERGREEN_CRTC3_REGISTER_OFFSET,
127 EVERGREEN_CRTC4_REGISTER_OFFSET,
128 EVERGREEN_CRTC5_REGISTER_OFFSET
129};
130
131#include "clearstate_evergreen.h"
132
133static const u32 sumo_rlc_save_restore_register_list[] =
134{
135 0x98fc,
136 0x9830,
137 0x9834,
138 0x9838,
139 0x9870,
140 0x9874,
141 0x8a14,
142 0x8b24,
143 0x8bcc,
144 0x8b10,
145 0x8d00,
146 0x8d04,
147 0x8c00,
148 0x8c04,
149 0x8c08,
150 0x8c0c,
151 0x8d8c,
152 0x8c20,
153 0x8c24,
154 0x8c28,
155 0x8c18,
156 0x8c1c,
157 0x8cf0,
158 0x8e2c,
159 0x8e38,
160 0x8c30,
161 0x9508,
162 0x9688,
163 0x9608,
164 0x960c,
165 0x9610,
166 0x9614,
167 0x88c4,
168 0x88d4,
169 0xa008,
170 0x900c,
171 0x9100,
172 0x913c,
173 0x98f8,
174 0x98f4,
175 0x9b7c,
176 0x3f8c,
177 0x8950,
178 0x8954,
179 0x8a18,
180 0x8b28,
181 0x9144,
182 0x9148,
183 0x914c,
184 0x3f90,
185 0x3f94,
186 0x915c,
187 0x9160,
188 0x9178,
189 0x917c,
190 0x9180,
191 0x918c,
192 0x9190,
193 0x9194,
194 0x9198,
195 0x919c,
196 0x91a8,
197 0x91ac,
198 0x91b0,
199 0x91b4,
200 0x91b8,
201 0x91c4,
202 0x91c8,
203 0x91cc,
204 0x91d0,
205 0x91d4,
206 0x91e0,
207 0x91e4,
208 0x91ec,
209 0x91f0,
210 0x91f4,
211 0x9200,
212 0x9204,
213 0x929c,
214 0x9150,
215 0x802c,
216};
217
218static void evergreen_gpu_init(struct radeon_device *rdev);
219void evergreen_fini(struct radeon_device *rdev);
220void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
221void evergreen_program_aspm(struct radeon_device *rdev);
222
223static const u32 evergreen_golden_registers[] =
224{
225 0x3f90, 0xffff0000, 0xff000000,
226 0x9148, 0xffff0000, 0xff000000,
227 0x3f94, 0xffff0000, 0xff000000,
228 0x914c, 0xffff0000, 0xff000000,
229 0x9b7c, 0xffffffff, 0x00000000,
230 0x8a14, 0xffffffff, 0x00000007,
231 0x8b10, 0xffffffff, 0x00000000,
232 0x960c, 0xffffffff, 0x54763210,
233 0x88c4, 0xffffffff, 0x000000c2,
234 0x88d4, 0xffffffff, 0x00000010,
235 0x8974, 0xffffffff, 0x00000000,
236 0xc78, 0x00000080, 0x00000080,
237 0x5eb4, 0xffffffff, 0x00000002,
238 0x5e78, 0xffffffff, 0x001000f0,
239 0x6104, 0x01000300, 0x00000000,
240 0x5bc0, 0x00300000, 0x00000000,
241 0x7030, 0xffffffff, 0x00000011,
242 0x7c30, 0xffffffff, 0x00000011,
243 0x10830, 0xffffffff, 0x00000011,
244 0x11430, 0xffffffff, 0x00000011,
245 0x12030, 0xffffffff, 0x00000011,
246 0x12c30, 0xffffffff, 0x00000011,
247 0xd02c, 0xffffffff, 0x08421000,
248 0x240c, 0xffffffff, 0x00000380,
249 0x8b24, 0xffffffff, 0x00ff0fff,
250 0x28a4c, 0x06000000, 0x06000000,
251 0x10c, 0x00000001, 0x00000001,
252 0x8d00, 0xffffffff, 0x100e4848,
253 0x8d04, 0xffffffff, 0x00164745,
254 0x8c00, 0xffffffff, 0xe4000003,
255 0x8c04, 0xffffffff, 0x40600060,
256 0x8c08, 0xffffffff, 0x001c001c,
257 0x8cf0, 0xffffffff, 0x08e00620,
258 0x8c20, 0xffffffff, 0x00800080,
259 0x8c24, 0xffffffff, 0x00800080,
260 0x8c18, 0xffffffff, 0x20202078,
261 0x8c1c, 0xffffffff, 0x00001010,
262 0x28350, 0xffffffff, 0x00000000,
263 0xa008, 0xffffffff, 0x00010000,
264 0x5c4, 0xffffffff, 0x00000001,
265 0x9508, 0xffffffff, 0x00000002,
266 0x913c, 0x0000000f, 0x0000000a
267};
268
269static const u32 evergreen_golden_registers2[] =
270{
271 0x2f4c, 0xffffffff, 0x00000000,
272 0x54f4, 0xffffffff, 0x00000000,
273 0x54f0, 0xffffffff, 0x00000000,
274 0x5498, 0xffffffff, 0x00000000,
275 0x549c, 0xffffffff, 0x00000000,
276 0x5494, 0xffffffff, 0x00000000,
277 0x53cc, 0xffffffff, 0x00000000,
278 0x53c8, 0xffffffff, 0x00000000,
279 0x53c4, 0xffffffff, 0x00000000,
280 0x53c0, 0xffffffff, 0x00000000,
281 0x53bc, 0xffffffff, 0x00000000,
282 0x53b8, 0xffffffff, 0x00000000,
283 0x53b4, 0xffffffff, 0x00000000,
284 0x53b0, 0xffffffff, 0x00000000
285};
286
287static const u32 cypress_mgcg_init[] =
288{
289 0x802c, 0xffffffff, 0xc0000000,
290 0x5448, 0xffffffff, 0x00000100,
291 0x55e4, 0xffffffff, 0x00000100,
292 0x160c, 0xffffffff, 0x00000100,
293 0x5644, 0xffffffff, 0x00000100,
294 0xc164, 0xffffffff, 0x00000100,
295 0x8a18, 0xffffffff, 0x00000100,
296 0x897c, 0xffffffff, 0x06000100,
297 0x8b28, 0xffffffff, 0x00000100,
298 0x9144, 0xffffffff, 0x00000100,
299 0x9a60, 0xffffffff, 0x00000100,
300 0x9868, 0xffffffff, 0x00000100,
301 0x8d58, 0xffffffff, 0x00000100,
302 0x9510, 0xffffffff, 0x00000100,
303 0x949c, 0xffffffff, 0x00000100,
304 0x9654, 0xffffffff, 0x00000100,
305 0x9030, 0xffffffff, 0x00000100,
306 0x9034, 0xffffffff, 0x00000100,
307 0x9038, 0xffffffff, 0x00000100,
308 0x903c, 0xffffffff, 0x00000100,
309 0x9040, 0xffffffff, 0x00000100,
310 0xa200, 0xffffffff, 0x00000100,
311 0xa204, 0xffffffff, 0x00000100,
312 0xa208, 0xffffffff, 0x00000100,
313 0xa20c, 0xffffffff, 0x00000100,
314 0x971c, 0xffffffff, 0x00000100,
315 0x977c, 0xffffffff, 0x00000100,
316 0x3f80, 0xffffffff, 0x00000100,
317 0xa210, 0xffffffff, 0x00000100,
318 0xa214, 0xffffffff, 0x00000100,
319 0x4d8, 0xffffffff, 0x00000100,
320 0x9784, 0xffffffff, 0x00000100,
321 0x9698, 0xffffffff, 0x00000100,
322 0x4d4, 0xffffffff, 0x00000200,
323 0x30cc, 0xffffffff, 0x00000100,
324 0xd0c0, 0xffffffff, 0xff000100,
325 0x802c, 0xffffffff, 0x40000000,
326 0x915c, 0xffffffff, 0x00010000,
327 0x9160, 0xffffffff, 0x00030002,
328 0x9178, 0xffffffff, 0x00070000,
329 0x917c, 0xffffffff, 0x00030002,
330 0x9180, 0xffffffff, 0x00050004,
331 0x918c, 0xffffffff, 0x00010006,
332 0x9190, 0xffffffff, 0x00090008,
333 0x9194, 0xffffffff, 0x00070000,
334 0x9198, 0xffffffff, 0x00030002,
335 0x919c, 0xffffffff, 0x00050004,
336 0x91a8, 0xffffffff, 0x00010006,
337 0x91ac, 0xffffffff, 0x00090008,
338 0x91b0, 0xffffffff, 0x00070000,
339 0x91b4, 0xffffffff, 0x00030002,
340 0x91b8, 0xffffffff, 0x00050004,
341 0x91c4, 0xffffffff, 0x00010006,
342 0x91c8, 0xffffffff, 0x00090008,
343 0x91cc, 0xffffffff, 0x00070000,
344 0x91d0, 0xffffffff, 0x00030002,
345 0x91d4, 0xffffffff, 0x00050004,
346 0x91e0, 0xffffffff, 0x00010006,
347 0x91e4, 0xffffffff, 0x00090008,
348 0x91e8, 0xffffffff, 0x00000000,
349 0x91ec, 0xffffffff, 0x00070000,
350 0x91f0, 0xffffffff, 0x00030002,
351 0x91f4, 0xffffffff, 0x00050004,
352 0x9200, 0xffffffff, 0x00010006,
353 0x9204, 0xffffffff, 0x00090008,
354 0x9208, 0xffffffff, 0x00070000,
355 0x920c, 0xffffffff, 0x00030002,
356 0x9210, 0xffffffff, 0x00050004,
357 0x921c, 0xffffffff, 0x00010006,
358 0x9220, 0xffffffff, 0x00090008,
359 0x9224, 0xffffffff, 0x00070000,
360 0x9228, 0xffffffff, 0x00030002,
361 0x922c, 0xffffffff, 0x00050004,
362 0x9238, 0xffffffff, 0x00010006,
363 0x923c, 0xffffffff, 0x00090008,
364 0x9240, 0xffffffff, 0x00070000,
365 0x9244, 0xffffffff, 0x00030002,
366 0x9248, 0xffffffff, 0x00050004,
367 0x9254, 0xffffffff, 0x00010006,
368 0x9258, 0xffffffff, 0x00090008,
369 0x925c, 0xffffffff, 0x00070000,
370 0x9260, 0xffffffff, 0x00030002,
371 0x9264, 0xffffffff, 0x00050004,
372 0x9270, 0xffffffff, 0x00010006,
373 0x9274, 0xffffffff, 0x00090008,
374 0x9278, 0xffffffff, 0x00070000,
375 0x927c, 0xffffffff, 0x00030002,
376 0x9280, 0xffffffff, 0x00050004,
377 0x928c, 0xffffffff, 0x00010006,
378 0x9290, 0xffffffff, 0x00090008,
379 0x9294, 0xffffffff, 0x00000000,
380 0x929c, 0xffffffff, 0x00000001,
381 0x802c, 0xffffffff, 0x40010000,
382 0x915c, 0xffffffff, 0x00010000,
383 0x9160, 0xffffffff, 0x00030002,
384 0x9178, 0xffffffff, 0x00070000,
385 0x917c, 0xffffffff, 0x00030002,
386 0x9180, 0xffffffff, 0x00050004,
387 0x918c, 0xffffffff, 0x00010006,
388 0x9190, 0xffffffff, 0x00090008,
389 0x9194, 0xffffffff, 0x00070000,
390 0x9198, 0xffffffff, 0x00030002,
391 0x919c, 0xffffffff, 0x00050004,
392 0x91a8, 0xffffffff, 0x00010006,
393 0x91ac, 0xffffffff, 0x00090008,
394 0x91b0, 0xffffffff, 0x00070000,
395 0x91b4, 0xffffffff, 0x00030002,
396 0x91b8, 0xffffffff, 0x00050004,
397 0x91c4, 0xffffffff, 0x00010006,
398 0x91c8, 0xffffffff, 0x00090008,
399 0x91cc, 0xffffffff, 0x00070000,
400 0x91d0, 0xffffffff, 0x00030002,
401 0x91d4, 0xffffffff, 0x00050004,
402 0x91e0, 0xffffffff, 0x00010006,
403 0x91e4, 0xffffffff, 0x00090008,
404 0x91e8, 0xffffffff, 0x00000000,
405 0x91ec, 0xffffffff, 0x00070000,
406 0x91f0, 0xffffffff, 0x00030002,
407 0x91f4, 0xffffffff, 0x00050004,
408 0x9200, 0xffffffff, 0x00010006,
409 0x9204, 0xffffffff, 0x00090008,
410 0x9208, 0xffffffff, 0x00070000,
411 0x920c, 0xffffffff, 0x00030002,
412 0x9210, 0xffffffff, 0x00050004,
413 0x921c, 0xffffffff, 0x00010006,
414 0x9220, 0xffffffff, 0x00090008,
415 0x9224, 0xffffffff, 0x00070000,
416 0x9228, 0xffffffff, 0x00030002,
417 0x922c, 0xffffffff, 0x00050004,
418 0x9238, 0xffffffff, 0x00010006,
419 0x923c, 0xffffffff, 0x00090008,
420 0x9240, 0xffffffff, 0x00070000,
421 0x9244, 0xffffffff, 0x00030002,
422 0x9248, 0xffffffff, 0x00050004,
423 0x9254, 0xffffffff, 0x00010006,
424 0x9258, 0xffffffff, 0x00090008,
425 0x925c, 0xffffffff, 0x00070000,
426 0x9260, 0xffffffff, 0x00030002,
427 0x9264, 0xffffffff, 0x00050004,
428 0x9270, 0xffffffff, 0x00010006,
429 0x9274, 0xffffffff, 0x00090008,
430 0x9278, 0xffffffff, 0x00070000,
431 0x927c, 0xffffffff, 0x00030002,
432 0x9280, 0xffffffff, 0x00050004,
433 0x928c, 0xffffffff, 0x00010006,
434 0x9290, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
438};
439
440static const u32 redwood_mgcg_init[] =
441{
442 0x802c, 0xffffffff, 0xc0000000,
443 0x5448, 0xffffffff, 0x00000100,
444 0x55e4, 0xffffffff, 0x00000100,
445 0x160c, 0xffffffff, 0x00000100,
446 0x5644, 0xffffffff, 0x00000100,
447 0xc164, 0xffffffff, 0x00000100,
448 0x8a18, 0xffffffff, 0x00000100,
449 0x897c, 0xffffffff, 0x06000100,
450 0x8b28, 0xffffffff, 0x00000100,
451 0x9144, 0xffffffff, 0x00000100,
452 0x9a60, 0xffffffff, 0x00000100,
453 0x9868, 0xffffffff, 0x00000100,
454 0x8d58, 0xffffffff, 0x00000100,
455 0x9510, 0xffffffff, 0x00000100,
456 0x949c, 0xffffffff, 0x00000100,
457 0x9654, 0xffffffff, 0x00000100,
458 0x9030, 0xffffffff, 0x00000100,
459 0x9034, 0xffffffff, 0x00000100,
460 0x9038, 0xffffffff, 0x00000100,
461 0x903c, 0xffffffff, 0x00000100,
462 0x9040, 0xffffffff, 0x00000100,
463 0xa200, 0xffffffff, 0x00000100,
464 0xa204, 0xffffffff, 0x00000100,
465 0xa208, 0xffffffff, 0x00000100,
466 0xa20c, 0xffffffff, 0x00000100,
467 0x971c, 0xffffffff, 0x00000100,
468 0x977c, 0xffffffff, 0x00000100,
469 0x3f80, 0xffffffff, 0x00000100,
470 0xa210, 0xffffffff, 0x00000100,
471 0xa214, 0xffffffff, 0x00000100,
472 0x4d8, 0xffffffff, 0x00000100,
473 0x9784, 0xffffffff, 0x00000100,
474 0x9698, 0xffffffff, 0x00000100,
475 0x4d4, 0xffffffff, 0x00000200,
476 0x30cc, 0xffffffff, 0x00000100,
477 0xd0c0, 0xffffffff, 0xff000100,
478 0x802c, 0xffffffff, 0x40000000,
479 0x915c, 0xffffffff, 0x00010000,
480 0x9160, 0xffffffff, 0x00030002,
481 0x9178, 0xffffffff, 0x00070000,
482 0x917c, 0xffffffff, 0x00030002,
483 0x9180, 0xffffffff, 0x00050004,
484 0x918c, 0xffffffff, 0x00010006,
485 0x9190, 0xffffffff, 0x00090008,
486 0x9194, 0xffffffff, 0x00070000,
487 0x9198, 0xffffffff, 0x00030002,
488 0x919c, 0xffffffff, 0x00050004,
489 0x91a8, 0xffffffff, 0x00010006,
490 0x91ac, 0xffffffff, 0x00090008,
491 0x91b0, 0xffffffff, 0x00070000,
492 0x91b4, 0xffffffff, 0x00030002,
493 0x91b8, 0xffffffff, 0x00050004,
494 0x91c4, 0xffffffff, 0x00010006,
495 0x91c8, 0xffffffff, 0x00090008,
496 0x91cc, 0xffffffff, 0x00070000,
497 0x91d0, 0xffffffff, 0x00030002,
498 0x91d4, 0xffffffff, 0x00050004,
499 0x91e0, 0xffffffff, 0x00010006,
500 0x91e4, 0xffffffff, 0x00090008,
501 0x91e8, 0xffffffff, 0x00000000,
502 0x91ec, 0xffffffff, 0x00070000,
503 0x91f0, 0xffffffff, 0x00030002,
504 0x91f4, 0xffffffff, 0x00050004,
505 0x9200, 0xffffffff, 0x00010006,
506 0x9204, 0xffffffff, 0x00090008,
507 0x9294, 0xffffffff, 0x00000000,
508 0x929c, 0xffffffff, 0x00000001,
509 0x802c, 0xffffffff, 0xc0000000
510};
511
512static const u32 cedar_golden_registers[] =
513{
514 0x3f90, 0xffff0000, 0xff000000,
515 0x9148, 0xffff0000, 0xff000000,
516 0x3f94, 0xffff0000, 0xff000000,
517 0x914c, 0xffff0000, 0xff000000,
518 0x9b7c, 0xffffffff, 0x00000000,
519 0x8a14, 0xffffffff, 0x00000007,
520 0x8b10, 0xffffffff, 0x00000000,
521 0x960c, 0xffffffff, 0x54763210,
522 0x88c4, 0xffffffff, 0x000000c2,
523 0x88d4, 0xffffffff, 0x00000000,
524 0x8974, 0xffffffff, 0x00000000,
525 0xc78, 0x00000080, 0x00000080,
526 0x5eb4, 0xffffffff, 0x00000002,
527 0x5e78, 0xffffffff, 0x001000f0,
528 0x6104, 0x01000300, 0x00000000,
529 0x5bc0, 0x00300000, 0x00000000,
530 0x7030, 0xffffffff, 0x00000011,
531 0x7c30, 0xffffffff, 0x00000011,
532 0x10830, 0xffffffff, 0x00000011,
533 0x11430, 0xffffffff, 0x00000011,
534 0xd02c, 0xffffffff, 0x08421000,
535 0x240c, 0xffffffff, 0x00000380,
536 0x8b24, 0xffffffff, 0x00ff0fff,
537 0x28a4c, 0x06000000, 0x06000000,
538 0x10c, 0x00000001, 0x00000001,
539 0x8d00, 0xffffffff, 0x100e4848,
540 0x8d04, 0xffffffff, 0x00164745,
541 0x8c00, 0xffffffff, 0xe4000003,
542 0x8c04, 0xffffffff, 0x40600060,
543 0x8c08, 0xffffffff, 0x001c001c,
544 0x8cf0, 0xffffffff, 0x08e00410,
545 0x8c20, 0xffffffff, 0x00800080,
546 0x8c24, 0xffffffff, 0x00800080,
547 0x8c18, 0xffffffff, 0x20202078,
548 0x8c1c, 0xffffffff, 0x00001010,
549 0x28350, 0xffffffff, 0x00000000,
550 0xa008, 0xffffffff, 0x00010000,
551 0x5c4, 0xffffffff, 0x00000001,
552 0x9508, 0xffffffff, 0x00000002
553};
554
555static const u32 cedar_mgcg_init[] =
556{
557 0x802c, 0xffffffff, 0xc0000000,
558 0x5448, 0xffffffff, 0x00000100,
559 0x55e4, 0xffffffff, 0x00000100,
560 0x160c, 0xffffffff, 0x00000100,
561 0x5644, 0xffffffff, 0x00000100,
562 0xc164, 0xffffffff, 0x00000100,
563 0x8a18, 0xffffffff, 0x00000100,
564 0x897c, 0xffffffff, 0x06000100,
565 0x8b28, 0xffffffff, 0x00000100,
566 0x9144, 0xffffffff, 0x00000100,
567 0x9a60, 0xffffffff, 0x00000100,
568 0x9868, 0xffffffff, 0x00000100,
569 0x8d58, 0xffffffff, 0x00000100,
570 0x9510, 0xffffffff, 0x00000100,
571 0x949c, 0xffffffff, 0x00000100,
572 0x9654, 0xffffffff, 0x00000100,
573 0x9030, 0xffffffff, 0x00000100,
574 0x9034, 0xffffffff, 0x00000100,
575 0x9038, 0xffffffff, 0x00000100,
576 0x903c, 0xffffffff, 0x00000100,
577 0x9040, 0xffffffff, 0x00000100,
578 0xa200, 0xffffffff, 0x00000100,
579 0xa204, 0xffffffff, 0x00000100,
580 0xa208, 0xffffffff, 0x00000100,
581 0xa20c, 0xffffffff, 0x00000100,
582 0x971c, 0xffffffff, 0x00000100,
583 0x977c, 0xffffffff, 0x00000100,
584 0x3f80, 0xffffffff, 0x00000100,
585 0xa210, 0xffffffff, 0x00000100,
586 0xa214, 0xffffffff, 0x00000100,
587 0x4d8, 0xffffffff, 0x00000100,
588 0x9784, 0xffffffff, 0x00000100,
589 0x9698, 0xffffffff, 0x00000100,
590 0x4d4, 0xffffffff, 0x00000200,
591 0x30cc, 0xffffffff, 0x00000100,
592 0xd0c0, 0xffffffff, 0xff000100,
593 0x802c, 0xffffffff, 0x40000000,
594 0x915c, 0xffffffff, 0x00010000,
595 0x9178, 0xffffffff, 0x00050000,
596 0x917c, 0xffffffff, 0x00030002,
597 0x918c, 0xffffffff, 0x00010004,
598 0x9190, 0xffffffff, 0x00070006,
599 0x9194, 0xffffffff, 0x00050000,
600 0x9198, 0xffffffff, 0x00030002,
601 0x91a8, 0xffffffff, 0x00010004,
602 0x91ac, 0xffffffff, 0x00070006,
603 0x91e8, 0xffffffff, 0x00000000,
604 0x9294, 0xffffffff, 0x00000000,
605 0x929c, 0xffffffff, 0x00000001,
606 0x802c, 0xffffffff, 0xc0000000
607};
608
609static const u32 juniper_mgcg_init[] =
610{
611 0x802c, 0xffffffff, 0xc0000000,
612 0x5448, 0xffffffff, 0x00000100,
613 0x55e4, 0xffffffff, 0x00000100,
614 0x160c, 0xffffffff, 0x00000100,
615 0x5644, 0xffffffff, 0x00000100,
616 0xc164, 0xffffffff, 0x00000100,
617 0x8a18, 0xffffffff, 0x00000100,
618 0x897c, 0xffffffff, 0x06000100,
619 0x8b28, 0xffffffff, 0x00000100,
620 0x9144, 0xffffffff, 0x00000100,
621 0x9a60, 0xffffffff, 0x00000100,
622 0x9868, 0xffffffff, 0x00000100,
623 0x8d58, 0xffffffff, 0x00000100,
624 0x9510, 0xffffffff, 0x00000100,
625 0x949c, 0xffffffff, 0x00000100,
626 0x9654, 0xffffffff, 0x00000100,
627 0x9030, 0xffffffff, 0x00000100,
628 0x9034, 0xffffffff, 0x00000100,
629 0x9038, 0xffffffff, 0x00000100,
630 0x903c, 0xffffffff, 0x00000100,
631 0x9040, 0xffffffff, 0x00000100,
632 0xa200, 0xffffffff, 0x00000100,
633 0xa204, 0xffffffff, 0x00000100,
634 0xa208, 0xffffffff, 0x00000100,
635 0xa20c, 0xffffffff, 0x00000100,
636 0x971c, 0xffffffff, 0x00000100,
637 0xd0c0, 0xffffffff, 0xff000100,
638 0x802c, 0xffffffff, 0x40000000,
639 0x915c, 0xffffffff, 0x00010000,
640 0x9160, 0xffffffff, 0x00030002,
641 0x9178, 0xffffffff, 0x00070000,
642 0x917c, 0xffffffff, 0x00030002,
643 0x9180, 0xffffffff, 0x00050004,
644 0x918c, 0xffffffff, 0x00010006,
645 0x9190, 0xffffffff, 0x00090008,
646 0x9194, 0xffffffff, 0x00070000,
647 0x9198, 0xffffffff, 0x00030002,
648 0x919c, 0xffffffff, 0x00050004,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91ac, 0xffffffff, 0x00090008,
651 0x91b0, 0xffffffff, 0x00070000,
652 0x91b4, 0xffffffff, 0x00030002,
653 0x91b8, 0xffffffff, 0x00050004,
654 0x91c4, 0xffffffff, 0x00010006,
655 0x91c8, 0xffffffff, 0x00090008,
656 0x91cc, 0xffffffff, 0x00070000,
657 0x91d0, 0xffffffff, 0x00030002,
658 0x91d4, 0xffffffff, 0x00050004,
659 0x91e0, 0xffffffff, 0x00010006,
660 0x91e4, 0xffffffff, 0x00090008,
661 0x91e8, 0xffffffff, 0x00000000,
662 0x91ec, 0xffffffff, 0x00070000,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x9200, 0xffffffff, 0x00010006,
666 0x9204, 0xffffffff, 0x00090008,
667 0x9208, 0xffffffff, 0x00070000,
668 0x920c, 0xffffffff, 0x00030002,
669 0x9210, 0xffffffff, 0x00050004,
670 0x921c, 0xffffffff, 0x00010006,
671 0x9220, 0xffffffff, 0x00090008,
672 0x9224, 0xffffffff, 0x00070000,
673 0x9228, 0xffffffff, 0x00030002,
674 0x922c, 0xffffffff, 0x00050004,
675 0x9238, 0xffffffff, 0x00010006,
676 0x923c, 0xffffffff, 0x00090008,
677 0x9240, 0xffffffff, 0x00070000,
678 0x9244, 0xffffffff, 0x00030002,
679 0x9248, 0xffffffff, 0x00050004,
680 0x9254, 0xffffffff, 0x00010006,
681 0x9258, 0xffffffff, 0x00090008,
682 0x925c, 0xffffffff, 0x00070000,
683 0x9260, 0xffffffff, 0x00030002,
684 0x9264, 0xffffffff, 0x00050004,
685 0x9270, 0xffffffff, 0x00010006,
686 0x9274, 0xffffffff, 0x00090008,
687 0x9278, 0xffffffff, 0x00070000,
688 0x927c, 0xffffffff, 0x00030002,
689 0x9280, 0xffffffff, 0x00050004,
690 0x928c, 0xffffffff, 0x00010006,
691 0x9290, 0xffffffff, 0x00090008,
692 0x9294, 0xffffffff, 0x00000000,
693 0x929c, 0xffffffff, 0x00000001,
694 0x802c, 0xffffffff, 0xc0000000,
695 0x977c, 0xffffffff, 0x00000100,
696 0x3f80, 0xffffffff, 0x00000100,
697 0xa210, 0xffffffff, 0x00000100,
698 0xa214, 0xffffffff, 0x00000100,
699 0x4d8, 0xffffffff, 0x00000100,
700 0x9784, 0xffffffff, 0x00000100,
701 0x9698, 0xffffffff, 0x00000100,
702 0x4d4, 0xffffffff, 0x00000200,
703 0x30cc, 0xffffffff, 0x00000100,
704 0x802c, 0xffffffff, 0xc0000000
705};
706
707static const u32 supersumo_golden_registers[] =
708{
709 0x5eb4, 0xffffffff, 0x00000002,
710 0x5c4, 0xffffffff, 0x00000001,
711 0x7030, 0xffffffff, 0x00000011,
712 0x7c30, 0xffffffff, 0x00000011,
713 0x6104, 0x01000300, 0x00000000,
714 0x5bc0, 0x00300000, 0x00000000,
715 0x8c04, 0xffffffff, 0x40600060,
716 0x8c08, 0xffffffff, 0x001c001c,
717 0x8c20, 0xffffffff, 0x00800080,
718 0x8c24, 0xffffffff, 0x00800080,
719 0x8c18, 0xffffffff, 0x20202078,
720 0x8c1c, 0xffffffff, 0x00001010,
721 0x918c, 0xffffffff, 0x00010006,
722 0x91a8, 0xffffffff, 0x00010006,
723 0x91c4, 0xffffffff, 0x00010006,
724 0x91e0, 0xffffffff, 0x00010006,
725 0x9200, 0xffffffff, 0x00010006,
726 0x9150, 0xffffffff, 0x6e944040,
727 0x917c, 0xffffffff, 0x00030002,
728 0x9180, 0xffffffff, 0x00050004,
729 0x9198, 0xffffffff, 0x00030002,
730 0x919c, 0xffffffff, 0x00050004,
731 0x91b4, 0xffffffff, 0x00030002,
732 0x91b8, 0xffffffff, 0x00050004,
733 0x91d0, 0xffffffff, 0x00030002,
734 0x91d4, 0xffffffff, 0x00050004,
735 0x91f0, 0xffffffff, 0x00030002,
736 0x91f4, 0xffffffff, 0x00050004,
737 0x915c, 0xffffffff, 0x00010000,
738 0x9160, 0xffffffff, 0x00030002,
739 0x3f90, 0xffff0000, 0xff000000,
740 0x9178, 0xffffffff, 0x00070000,
741 0x9194, 0xffffffff, 0x00070000,
742 0x91b0, 0xffffffff, 0x00070000,
743 0x91cc, 0xffffffff, 0x00070000,
744 0x91ec, 0xffffffff, 0x00070000,
745 0x9148, 0xffff0000, 0xff000000,
746 0x9190, 0xffffffff, 0x00090008,
747 0x91ac, 0xffffffff, 0x00090008,
748 0x91c8, 0xffffffff, 0x00090008,
749 0x91e4, 0xffffffff, 0x00090008,
750 0x9204, 0xffffffff, 0x00090008,
751 0x3f94, 0xffff0000, 0xff000000,
752 0x914c, 0xffff0000, 0xff000000,
753 0x929c, 0xffffffff, 0x00000001,
754 0x8a18, 0xffffffff, 0x00000100,
755 0x8b28, 0xffffffff, 0x00000100,
756 0x9144, 0xffffffff, 0x00000100,
757 0x5644, 0xffffffff, 0x00000100,
758 0x9b7c, 0xffffffff, 0x00000000,
759 0x8030, 0xffffffff, 0x0000100a,
760 0x8a14, 0xffffffff, 0x00000007,
761 0x8b24, 0xffffffff, 0x00ff0fff,
762 0x8b10, 0xffffffff, 0x00000000,
763 0x28a4c, 0x06000000, 0x06000000,
764 0x4d8, 0xffffffff, 0x00000100,
765 0x913c, 0xffff000f, 0x0100000a,
766 0x960c, 0xffffffff, 0x54763210,
767 0x88c4, 0xffffffff, 0x000000c2,
768 0x88d4, 0xffffffff, 0x00000010,
769 0x8974, 0xffffffff, 0x00000000,
770 0xc78, 0x00000080, 0x00000080,
771 0x5e78, 0xffffffff, 0x001000f0,
772 0xd02c, 0xffffffff, 0x08421000,
773 0xa008, 0xffffffff, 0x00010000,
774 0x8d00, 0xffffffff, 0x100e4848,
775 0x8d04, 0xffffffff, 0x00164745,
776 0x8c00, 0xffffffff, 0xe4000003,
777 0x8cf0, 0x1fffffff, 0x08e00620,
778 0x28350, 0xffffffff, 0x00000000,
779 0x9508, 0xffffffff, 0x00000002
780};
781
782static const u32 sumo_golden_registers[] =
783{
784 0x900c, 0x00ffffff, 0x0017071f,
785 0x8c18, 0xffffffff, 0x10101060,
786 0x8c1c, 0xffffffff, 0x00001010,
787 0x8c30, 0x0000000f, 0x00000005,
788 0x9688, 0x0000000f, 0x00000007
789};
790
791static const u32 wrestler_golden_registers[] =
792{
793 0x5eb4, 0xffffffff, 0x00000002,
794 0x5c4, 0xffffffff, 0x00000001,
795 0x7030, 0xffffffff, 0x00000011,
796 0x7c30, 0xffffffff, 0x00000011,
797 0x6104, 0x01000300, 0x00000000,
798 0x5bc0, 0x00300000, 0x00000000,
799 0x918c, 0xffffffff, 0x00010006,
800 0x91a8, 0xffffffff, 0x00010006,
801 0x9150, 0xffffffff, 0x6e944040,
802 0x917c, 0xffffffff, 0x00030002,
803 0x9198, 0xffffffff, 0x00030002,
804 0x915c, 0xffffffff, 0x00010000,
805 0x3f90, 0xffff0000, 0xff000000,
806 0x9178, 0xffffffff, 0x00070000,
807 0x9194, 0xffffffff, 0x00070000,
808 0x9148, 0xffff0000, 0xff000000,
809 0x9190, 0xffffffff, 0x00090008,
810 0x91ac, 0xffffffff, 0x00090008,
811 0x3f94, 0xffff0000, 0xff000000,
812 0x914c, 0xffff0000, 0xff000000,
813 0x929c, 0xffffffff, 0x00000001,
814 0x8a18, 0xffffffff, 0x00000100,
815 0x8b28, 0xffffffff, 0x00000100,
816 0x9144, 0xffffffff, 0x00000100,
817 0x9b7c, 0xffffffff, 0x00000000,
818 0x8030, 0xffffffff, 0x0000100a,
819 0x8a14, 0xffffffff, 0x00000001,
820 0x8b24, 0xffffffff, 0x00ff0fff,
821 0x8b10, 0xffffffff, 0x00000000,
822 0x28a4c, 0x06000000, 0x06000000,
823 0x4d8, 0xffffffff, 0x00000100,
824 0x913c, 0xffff000f, 0x0100000a,
825 0x960c, 0xffffffff, 0x54763210,
826 0x88c4, 0xffffffff, 0x000000c2,
827 0x88d4, 0xffffffff, 0x00000010,
828 0x8974, 0xffffffff, 0x00000000,
829 0xc78, 0x00000080, 0x00000080,
830 0x5e78, 0xffffffff, 0x001000f0,
831 0xd02c, 0xffffffff, 0x08421000,
832 0xa008, 0xffffffff, 0x00010000,
833 0x8d00, 0xffffffff, 0x100e4848,
834 0x8d04, 0xffffffff, 0x00164745,
835 0x8c00, 0xffffffff, 0xe4000003,
836 0x8cf0, 0x1fffffff, 0x08e00410,
837 0x28350, 0xffffffff, 0x00000000,
838 0x9508, 0xffffffff, 0x00000002,
839 0x900c, 0xffffffff, 0x0017071f,
840 0x8c18, 0xffffffff, 0x10101060,
841 0x8c1c, 0xffffffff, 0x00001010
842};
843
844static const u32 barts_golden_registers[] =
845{
846 0x5eb4, 0xffffffff, 0x00000002,
847 0x5e78, 0x8f311ff1, 0x001000f0,
848 0x3f90, 0xffff0000, 0xff000000,
849 0x9148, 0xffff0000, 0xff000000,
850 0x3f94, 0xffff0000, 0xff000000,
851 0x914c, 0xffff0000, 0xff000000,
852 0xc78, 0x00000080, 0x00000080,
853 0xbd4, 0x70073777, 0x00010001,
854 0xd02c, 0xbfffff1f, 0x08421000,
855 0xd0b8, 0x03773777, 0x02011003,
856 0x5bc0, 0x00200000, 0x50100000,
857 0x98f8, 0x33773777, 0x02011003,
858 0x98fc, 0xffffffff, 0x76543210,
859 0x7030, 0x31000311, 0x00000011,
860 0x2f48, 0x00000007, 0x02011003,
861 0x6b28, 0x00000010, 0x00000012,
862 0x7728, 0x00000010, 0x00000012,
863 0x10328, 0x00000010, 0x00000012,
864 0x10f28, 0x00000010, 0x00000012,
865 0x11b28, 0x00000010, 0x00000012,
866 0x12728, 0x00000010, 0x00000012,
867 0x240c, 0x000007ff, 0x00000380,
868 0x8a14, 0xf000001f, 0x00000007,
869 0x8b24, 0x3fff3fff, 0x00ff0fff,
870 0x8b10, 0x0000ff0f, 0x00000000,
871 0x28a4c, 0x07ffffff, 0x06000000,
872 0x10c, 0x00000001, 0x00010003,
873 0xa02c, 0xffffffff, 0x0000009b,
874 0x913c, 0x0000000f, 0x0100000a,
875 0x8d00, 0xffff7f7f, 0x100e4848,
876 0x8d04, 0x00ffffff, 0x00164745,
877 0x8c00, 0xfffc0003, 0xe4000003,
878 0x8c04, 0xf8ff00ff, 0x40600060,
879 0x8c08, 0x00ff00ff, 0x001c001c,
880 0x8cf0, 0x1fff1fff, 0x08e00620,
881 0x8c20, 0x0fff0fff, 0x00800080,
882 0x8c24, 0x0fff0fff, 0x00800080,
883 0x8c18, 0xffffffff, 0x20202078,
884 0x8c1c, 0x0000ffff, 0x00001010,
885 0x28350, 0x00000f01, 0x00000000,
886 0x9508, 0x3700001f, 0x00000002,
887 0x960c, 0xffffffff, 0x54763210,
888 0x88c4, 0x001f3ae3, 0x000000c2,
889 0x88d4, 0x0000001f, 0x00000010,
890 0x8974, 0xffffffff, 0x00000000
891};
892
893static const u32 turks_golden_registers[] =
894{
895 0x5eb4, 0xffffffff, 0x00000002,
896 0x5e78, 0x8f311ff1, 0x001000f0,
897 0x8c8, 0x00003000, 0x00001070,
898 0x8cc, 0x000fffff, 0x00040035,
899 0x3f90, 0xffff0000, 0xfff00000,
900 0x9148, 0xffff0000, 0xfff00000,
901 0x3f94, 0xffff0000, 0xfff00000,
902 0x914c, 0xffff0000, 0xfff00000,
903 0xc78, 0x00000080, 0x00000080,
904 0xbd4, 0x00073007, 0x00010002,
905 0xd02c, 0xbfffff1f, 0x08421000,
906 0xd0b8, 0x03773777, 0x02010002,
907 0x5bc0, 0x00200000, 0x50100000,
908 0x98f8, 0x33773777, 0x00010002,
909 0x98fc, 0xffffffff, 0x33221100,
910 0x7030, 0x31000311, 0x00000011,
911 0x2f48, 0x33773777, 0x00010002,
912 0x6b28, 0x00000010, 0x00000012,
913 0x7728, 0x00000010, 0x00000012,
914 0x10328, 0x00000010, 0x00000012,
915 0x10f28, 0x00000010, 0x00000012,
916 0x11b28, 0x00000010, 0x00000012,
917 0x12728, 0x00000010, 0x00000012,
918 0x240c, 0x000007ff, 0x00000380,
919 0x8a14, 0xf000001f, 0x00000007,
920 0x8b24, 0x3fff3fff, 0x00ff0fff,
921 0x8b10, 0x0000ff0f, 0x00000000,
922 0x28a4c, 0x07ffffff, 0x06000000,
923 0x10c, 0x00000001, 0x00010003,
924 0xa02c, 0xffffffff, 0x0000009b,
925 0x913c, 0x0000000f, 0x0100000a,
926 0x8d00, 0xffff7f7f, 0x100e4848,
927 0x8d04, 0x00ffffff, 0x00164745,
928 0x8c00, 0xfffc0003, 0xe4000003,
929 0x8c04, 0xf8ff00ff, 0x40600060,
930 0x8c08, 0x00ff00ff, 0x001c001c,
931 0x8cf0, 0x1fff1fff, 0x08e00410,
932 0x8c20, 0x0fff0fff, 0x00800080,
933 0x8c24, 0x0fff0fff, 0x00800080,
934 0x8c18, 0xffffffff, 0x20202078,
935 0x8c1c, 0x0000ffff, 0x00001010,
936 0x28350, 0x00000f01, 0x00000000,
937 0x9508, 0x3700001f, 0x00000002,
938 0x960c, 0xffffffff, 0x54763210,
939 0x88c4, 0x001f3ae3, 0x000000c2,
940 0x88d4, 0x0000001f, 0x00000010,
941 0x8974, 0xffffffff, 0x00000000
942};
943
944static const u32 caicos_golden_registers[] =
945{
946 0x5eb4, 0xffffffff, 0x00000002,
947 0x5e78, 0x8f311ff1, 0x001000f0,
948 0x8c8, 0x00003420, 0x00001450,
949 0x8cc, 0x000fffff, 0x00040035,
950 0x3f90, 0xffff0000, 0xfffc0000,
951 0x9148, 0xffff0000, 0xfffc0000,
952 0x3f94, 0xffff0000, 0xfffc0000,
953 0x914c, 0xffff0000, 0xfffc0000,
954 0xc78, 0x00000080, 0x00000080,
955 0xbd4, 0x00073007, 0x00010001,
956 0xd02c, 0xbfffff1f, 0x08421000,
957 0xd0b8, 0x03773777, 0x02010001,
958 0x5bc0, 0x00200000, 0x50100000,
959 0x98f8, 0x33773777, 0x02010001,
960 0x98fc, 0xffffffff, 0x33221100,
961 0x7030, 0x31000311, 0x00000011,
962 0x2f48, 0x33773777, 0x02010001,
963 0x6b28, 0x00000010, 0x00000012,
964 0x7728, 0x00000010, 0x00000012,
965 0x10328, 0x00000010, 0x00000012,
966 0x10f28, 0x00000010, 0x00000012,
967 0x11b28, 0x00000010, 0x00000012,
968 0x12728, 0x00000010, 0x00000012,
969 0x240c, 0x000007ff, 0x00000380,
970 0x8a14, 0xf000001f, 0x00000001,
971 0x8b24, 0x3fff3fff, 0x00ff0fff,
972 0x8b10, 0x0000ff0f, 0x00000000,
973 0x28a4c, 0x07ffffff, 0x06000000,
974 0x10c, 0x00000001, 0x00010003,
975 0xa02c, 0xffffffff, 0x0000009b,
976 0x913c, 0x0000000f, 0x0100000a,
977 0x8d00, 0xffff7f7f, 0x100e4848,
978 0x8d04, 0x00ffffff, 0x00164745,
979 0x8c00, 0xfffc0003, 0xe4000003,
980 0x8c04, 0xf8ff00ff, 0x40600060,
981 0x8c08, 0x00ff00ff, 0x001c001c,
982 0x8cf0, 0x1fff1fff, 0x08e00410,
983 0x8c20, 0x0fff0fff, 0x00800080,
984 0x8c24, 0x0fff0fff, 0x00800080,
985 0x8c18, 0xffffffff, 0x20202078,
986 0x8c1c, 0x0000ffff, 0x00001010,
987 0x28350, 0x00000f01, 0x00000000,
988 0x9508, 0x3700001f, 0x00000002,
989 0x960c, 0xffffffff, 0x54763210,
990 0x88c4, 0x001f3ae3, 0x000000c2,
991 0x88d4, 0x0000001f, 0x00000010,
992 0x8974, 0xffffffff, 0x00000000
993};
994
995static void evergreen_init_golden_registers(struct radeon_device *rdev)
996{
997 switch (rdev->family) {
998 case CHIP_CYPRESS:
999 case CHIP_HEMLOCK:
1000 radeon_program_register_sequence(rdev,
1001 evergreen_golden_registers,
1002 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1003 radeon_program_register_sequence(rdev,
1004 evergreen_golden_registers2,
1005 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1006 radeon_program_register_sequence(rdev,
1007 cypress_mgcg_init,
1008 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1009 break;
1010 case CHIP_JUNIPER:
1011 radeon_program_register_sequence(rdev,
1012 evergreen_golden_registers,
1013 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1014 radeon_program_register_sequence(rdev,
1015 evergreen_golden_registers2,
1016 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1017 radeon_program_register_sequence(rdev,
1018 juniper_mgcg_init,
1019 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1020 break;
1021 case CHIP_REDWOOD:
1022 radeon_program_register_sequence(rdev,
1023 evergreen_golden_registers,
1024 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1025 radeon_program_register_sequence(rdev,
1026 evergreen_golden_registers2,
1027 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1028 radeon_program_register_sequence(rdev,
1029 redwood_mgcg_init,
1030 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1031 break;
1032 case CHIP_CEDAR:
1033 radeon_program_register_sequence(rdev,
1034 cedar_golden_registers,
1035 (const u32)ARRAY_SIZE(cedar_golden_registers));
1036 radeon_program_register_sequence(rdev,
1037 evergreen_golden_registers2,
1038 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1039 radeon_program_register_sequence(rdev,
1040 cedar_mgcg_init,
1041 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1042 break;
1043 case CHIP_PALM:
1044 radeon_program_register_sequence(rdev,
1045 wrestler_golden_registers,
1046 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1047 break;
1048 case CHIP_SUMO:
1049 radeon_program_register_sequence(rdev,
1050 supersumo_golden_registers,
1051 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1052 break;
1053 case CHIP_SUMO2:
1054 radeon_program_register_sequence(rdev,
1055 supersumo_golden_registers,
1056 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057 radeon_program_register_sequence(rdev,
1058 sumo_golden_registers,
1059 (const u32)ARRAY_SIZE(sumo_golden_registers));
1060 break;
1061 case CHIP_BARTS:
1062 radeon_program_register_sequence(rdev,
1063 barts_golden_registers,
1064 (const u32)ARRAY_SIZE(barts_golden_registers));
1065 break;
1066 case CHIP_TURKS:
1067 radeon_program_register_sequence(rdev,
1068 turks_golden_registers,
1069 (const u32)ARRAY_SIZE(turks_golden_registers));
1070 break;
1071 case CHIP_CAICOS:
1072 radeon_program_register_sequence(rdev,
1073 caicos_golden_registers,
1074 (const u32)ARRAY_SIZE(caicos_golden_registers));
1075 break;
1076 default:
1077 break;
1078 }
1079}
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1092 u32 reg, u32 *val)
1093{
1094 switch (reg) {
1095 case GRBM_STATUS:
1096 case GRBM_STATUS_SE0:
1097 case GRBM_STATUS_SE1:
1098 case SRBM_STATUS:
1099 case SRBM_STATUS2:
1100 case DMA_STATUS_REG:
1101 case UVD_STATUS:
1102 *val = RREG32(reg);
1103 return 0;
1104 default:
1105 return -EINVAL;
1106 }
1107}
1108
1109void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1110 unsigned *bankh, unsigned *mtaspect,
1111 unsigned *tile_split)
1112{
1113 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1114 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1115 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1116 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1117 switch (*bankw) {
1118 default:
1119 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1120 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1121 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1122 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1123 }
1124 switch (*bankh) {
1125 default:
1126 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1127 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1128 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1129 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1130 }
1131 switch (*mtaspect) {
1132 default:
1133 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1134 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1135 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1136 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1137 }
1138}
1139
1140static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1141 u32 cntl_reg, u32 status_reg)
1142{
1143 int r, i;
1144 struct atom_clock_dividers dividers;
1145
1146 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1147 clock, false, ÷rs);
1148 if (r)
1149 return r;
1150
1151 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1152
1153 for (i = 0; i < 100; i++) {
1154 if (RREG32(status_reg) & DCLK_STATUS)
1155 break;
1156 mdelay(10);
1157 }
1158 if (i == 100)
1159 return -ETIMEDOUT;
1160
1161 return 0;
1162}
1163
1164int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1165{
1166 int r = 0;
1167 u32 cg_scratch = RREG32(CG_SCRATCH1);
1168
1169 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1170 if (r)
1171 goto done;
1172 cg_scratch &= 0xffff0000;
1173 cg_scratch |= vclk / 100;
1174
1175 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1176 if (r)
1177 goto done;
1178 cg_scratch &= 0x0000ffff;
1179 cg_scratch |= (dclk / 100) << 16;
1180
1181done:
1182 WREG32(CG_SCRATCH1, cg_scratch);
1183
1184 return r;
1185}
1186
1187int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1188{
1189
1190 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1191 int r;
1192
1193
1194 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1195 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1196 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1197
1198
1199 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1200
1201 if (!vclk || !dclk) {
1202
1203 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1204 return 0;
1205 }
1206
1207 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1208 16384, 0x03FFFFFF, 0, 128, 5,
1209 &fb_div, &vclk_div, &dclk_div);
1210 if (r)
1211 return r;
1212
1213
1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1215
1216
1217 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1219
1220
1221 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1222
1223 mdelay(1);
1224
1225 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1226 if (r)
1227 return r;
1228
1229
1230 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1231
1232
1233 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1234
1235
1236 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1237
1238
1239 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1240
1241 if (fb_div < 307200)
1242 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1243 else
1244 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1245
1246
1247 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1248 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1249 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1250
1251
1252 mdelay(15);
1253
1254
1255 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1256
1257 mdelay(15);
1258
1259
1260 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1261
1262 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1263 if (r)
1264 return r;
1265
1266
1267 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1268 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1269 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1270
1271 mdelay(100);
1272
1273 return 0;
1274}
1275
1276void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1277{
1278 int readrq;
1279 u16 v;
1280
1281 readrq = pcie_get_readrq(rdev->pdev);
1282 v = ffs(readrq) - 8;
1283
1284
1285
1286 if ((v == 0) || (v == 6) || (v == 7))
1287 pcie_set_readrq(rdev->pdev, 512);
1288}
1289
1290void dce4_program_fmt(struct drm_encoder *encoder)
1291{
1292 struct drm_device *dev = encoder->dev;
1293 struct radeon_device *rdev = dev->dev_private;
1294 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1295 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1296 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1297 int bpc = 0;
1298 u32 tmp = 0;
1299 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1300
1301 if (connector) {
1302 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1303 bpc = radeon_get_monitor_bpc(connector);
1304 dither = radeon_connector->dither;
1305 }
1306
1307
1308 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1309 return;
1310
1311
1312 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1313 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1314 return;
1315
1316 if (bpc == 0)
1317 return;
1318
1319 switch (bpc) {
1320 case 6:
1321 if (dither == RADEON_FMT_DITHER_ENABLE)
1322
1323 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1324 FMT_SPATIAL_DITHER_EN);
1325 else
1326 tmp |= FMT_TRUNCATE_EN;
1327 break;
1328 case 8:
1329 if (dither == RADEON_FMT_DITHER_ENABLE)
1330
1331 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1332 FMT_RGB_RANDOM_ENABLE |
1333 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1334 else
1335 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1336 break;
1337 case 10:
1338 default:
1339
1340 break;
1341 }
1342
1343 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1344}
1345
1346static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1347{
1348 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1349 return true;
1350 else
1351 return false;
1352}
1353
1354static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1355{
1356 u32 pos1, pos2;
1357
1358 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1359 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1360
1361 if (pos1 != pos2)
1362 return true;
1363 else
1364 return false;
1365}
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1376{
1377 unsigned i = 0;
1378
1379 if (crtc >= rdev->num_crtc)
1380 return;
1381
1382 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1383 return;
1384
1385
1386
1387
1388 while (dce4_is_in_vblank(rdev, crtc)) {
1389 if (i++ % 100 == 0) {
1390 if (!dce4_is_counter_moving(rdev, crtc))
1391 break;
1392 }
1393 }
1394
1395 while (!dce4_is_in_vblank(rdev, crtc)) {
1396 if (i++ % 100 == 0) {
1397 if (!dce4_is_counter_moving(rdev, crtc))
1398 break;
1399 }
1400 }
1401}
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1415 bool async)
1416{
1417 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1418 struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1419
1420
1421 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1422 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1423
1424 WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1425 fb->pitches[0] / fb->format->cpp[0]);
1426
1427 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1428 upper_32_bits(crtc_base));
1429 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1430 (u32)crtc_base);
1431
1432 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1433}
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1444{
1445 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1446
1447
1448 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1449 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1450}
1451
1452
1453int evergreen_get_temp(struct radeon_device *rdev)
1454{
1455 u32 temp, toffset;
1456 int actual_temp = 0;
1457
1458 if (rdev->family == CHIP_JUNIPER) {
1459 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1460 TOFFSET_SHIFT;
1461 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1462 TS0_ADC_DOUT_SHIFT;
1463
1464 if (toffset & 0x100)
1465 actual_temp = temp / 2 - (0x200 - toffset);
1466 else
1467 actual_temp = temp / 2 + toffset;
1468
1469 actual_temp = actual_temp * 1000;
1470
1471 } else {
1472 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1473 ASIC_T_SHIFT;
1474
1475 if (temp & 0x400)
1476 actual_temp = -256;
1477 else if (temp & 0x200)
1478 actual_temp = 255;
1479 else if (temp & 0x100) {
1480 actual_temp = temp & 0x1ff;
1481 actual_temp |= ~0x1ff;
1482 } else
1483 actual_temp = temp & 0xff;
1484
1485 actual_temp = (actual_temp * 1000) / 2;
1486 }
1487
1488 return actual_temp;
1489}
1490
1491int sumo_get_temp(struct radeon_device *rdev)
1492{
1493 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1494 int actual_temp = temp - 49;
1495
1496 return actual_temp * 1000;
1497}
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508void sumo_pm_init_profile(struct radeon_device *rdev)
1509{
1510 int idx;
1511
1512
1513 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1514 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1515 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1516 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1517
1518
1519 if (rdev->flags & RADEON_IS_MOBILITY)
1520 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1521 else
1522 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1523
1524 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1525 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1527 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1528
1529 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1533
1534 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1538
1539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1543
1544
1545 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1546 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1548 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1550 rdev->pm.power_state[idx].num_clock_modes - 1;
1551
1552 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1553 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1556 rdev->pm.power_state[idx].num_clock_modes - 1;
1557}
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568void btc_pm_init_profile(struct radeon_device *rdev)
1569{
1570 int idx;
1571
1572
1573 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1574 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1575 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1576 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1577
1578
1579
1580
1581 if (rdev->flags & RADEON_IS_MOBILITY)
1582 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1583 else
1584 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1585
1586 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1587 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1589 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1590
1591 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1592 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1594 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1595
1596 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1597 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1599 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1600
1601 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1602 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1604 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1605
1606 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1607 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1609 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1610
1611 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1612 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1614 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1615}
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625void evergreen_pm_misc(struct radeon_device *rdev)
1626{
1627 int req_ps_idx = rdev->pm.requested_power_state_index;
1628 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1629 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1630 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1631
1632 if (voltage->type == VOLTAGE_SW) {
1633
1634 if ((voltage->voltage & 0xff00) == 0xff00)
1635 return;
1636 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1637 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1638 rdev->pm.current_vddc = voltage->voltage;
1639 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1640 }
1641
1642
1643
1644
1645
1646 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1647 (rdev->family >= CHIP_BARTS) &&
1648 rdev->pm.active_crtc_count &&
1649 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1650 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1651 voltage = &rdev->pm.power_state[req_ps_idx].
1652 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1653
1654
1655 if ((voltage->vddci & 0xff00) == 0xff00)
1656 return;
1657 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1658 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1659 rdev->pm.current_vddci = voltage->vddci;
1660 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1661 }
1662 }
1663}
1664
1665
1666
1667
1668
1669
1670
1671
1672void evergreen_pm_prepare(struct radeon_device *rdev)
1673{
1674 struct drm_device *ddev = rdev->ddev;
1675 struct drm_crtc *crtc;
1676 struct radeon_crtc *radeon_crtc;
1677 u32 tmp;
1678
1679
1680 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1681 radeon_crtc = to_radeon_crtc(crtc);
1682 if (radeon_crtc->enabled) {
1683 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1684 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1685 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1686 }
1687 }
1688}
1689
1690
1691
1692
1693
1694
1695
1696
1697void evergreen_pm_finish(struct radeon_device *rdev)
1698{
1699 struct drm_device *ddev = rdev->ddev;
1700 struct drm_crtc *crtc;
1701 struct radeon_crtc *radeon_crtc;
1702 u32 tmp;
1703
1704
1705 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1706 radeon_crtc = to_radeon_crtc(crtc);
1707 if (radeon_crtc->enabled) {
1708 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1709 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1710 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1711 }
1712 }
1713}
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1725{
1726 if (hpd == RADEON_HPD_NONE)
1727 return false;
1728
1729 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1730}
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1741 enum radeon_hpd_id hpd)
1742{
1743 bool connected = evergreen_hpd_sense(rdev, hpd);
1744
1745 if (hpd == RADEON_HPD_NONE)
1746 return;
1747
1748 if (connected)
1749 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1750 else
1751 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1752}
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762void evergreen_hpd_init(struct radeon_device *rdev)
1763{
1764 struct drm_device *dev = rdev->ddev;
1765 struct drm_connector *connector;
1766 unsigned enabled = 0;
1767 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1768 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1769
1770 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1771 enum radeon_hpd_id hpd =
1772 to_radeon_connector(connector)->hpd.hpd;
1773
1774 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1775 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1776
1777
1778
1779
1780
1781 continue;
1782 }
1783
1784 if (hpd == RADEON_HPD_NONE)
1785 continue;
1786
1787 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1788 enabled |= 1 << hpd;
1789
1790 radeon_hpd_set_polarity(rdev, hpd);
1791 }
1792 radeon_irq_kms_enable_hpd(rdev, enabled);
1793}
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803void evergreen_hpd_fini(struct radeon_device *rdev)
1804{
1805 struct drm_device *dev = rdev->ddev;
1806 struct drm_connector *connector;
1807 unsigned disabled = 0;
1808
1809 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1810 enum radeon_hpd_id hpd =
1811 to_radeon_connector(connector)->hpd.hpd;
1812
1813 if (hpd == RADEON_HPD_NONE)
1814 continue;
1815
1816 WREG32(DC_HPDx_CONTROL(hpd), 0);
1817 disabled |= 1 << hpd;
1818 }
1819 radeon_irq_kms_disable_hpd(rdev, disabled);
1820}
1821
1822
1823
1824static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1825 struct radeon_crtc *radeon_crtc,
1826 struct drm_display_mode *mode,
1827 struct drm_display_mode *other_mode)
1828{
1829 u32 tmp, buffer_alloc, i;
1830 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852 if (radeon_crtc->base.enabled && mode) {
1853 if (other_mode) {
1854 tmp = 0;
1855 buffer_alloc = 1;
1856 } else {
1857 tmp = 2;
1858 buffer_alloc = 2;
1859 }
1860 } else {
1861 tmp = 0;
1862 buffer_alloc = 0;
1863 }
1864
1865
1866 if (radeon_crtc->crtc_id % 2)
1867 tmp += 4;
1868 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1869
1870 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1871 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1872 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1873 for (i = 0; i < rdev->usec_timeout; i++) {
1874 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1875 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1876 break;
1877 udelay(1);
1878 }
1879 }
1880
1881 if (radeon_crtc->base.enabled && mode) {
1882 switch (tmp) {
1883 case 0:
1884 case 4:
1885 default:
1886 if (ASIC_IS_DCE5(rdev))
1887 return 4096 * 2;
1888 else
1889 return 3840 * 2;
1890 case 1:
1891 case 5:
1892 if (ASIC_IS_DCE5(rdev))
1893 return 6144 * 2;
1894 else
1895 return 5760 * 2;
1896 case 2:
1897 case 6:
1898 if (ASIC_IS_DCE5(rdev))
1899 return 8192 * 2;
1900 else
1901 return 7680 * 2;
1902 case 3:
1903 case 7:
1904 if (ASIC_IS_DCE5(rdev))
1905 return 2048 * 2;
1906 else
1907 return 1920 * 2;
1908 }
1909 }
1910
1911
1912 return 0;
1913}
1914
1915u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1916{
1917 u32 tmp = RREG32(MC_SHARED_CHMAP);
1918
1919 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1920 case 0:
1921 default:
1922 return 1;
1923 case 1:
1924 return 2;
1925 case 2:
1926 return 4;
1927 case 3:
1928 return 8;
1929 }
1930}
1931
1932struct evergreen_wm_params {
1933 u32 dram_channels;
1934 u32 yclk;
1935 u32 sclk;
1936 u32 disp_clk;
1937 u32 src_width;
1938 u32 active_time;
1939 u32 blank_time;
1940 bool interlaced;
1941 fixed20_12 vsc;
1942 u32 num_heads;
1943 u32 bytes_per_pixel;
1944 u32 lb_size;
1945 u32 vtaps;
1946};
1947
1948static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1949{
1950
1951 fixed20_12 dram_efficiency;
1952 fixed20_12 yclk, dram_channels, bandwidth;
1953 fixed20_12 a;
1954
1955 a.full = dfixed_const(1000);
1956 yclk.full = dfixed_const(wm->yclk);
1957 yclk.full = dfixed_div(yclk, a);
1958 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1959 a.full = dfixed_const(10);
1960 dram_efficiency.full = dfixed_const(7);
1961 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1962 bandwidth.full = dfixed_mul(dram_channels, yclk);
1963 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1964
1965 return dfixed_trunc(bandwidth);
1966}
1967
1968static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1969{
1970
1971 fixed20_12 disp_dram_allocation;
1972 fixed20_12 yclk, dram_channels, bandwidth;
1973 fixed20_12 a;
1974
1975 a.full = dfixed_const(1000);
1976 yclk.full = dfixed_const(wm->yclk);
1977 yclk.full = dfixed_div(yclk, a);
1978 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1979 a.full = dfixed_const(10);
1980 disp_dram_allocation.full = dfixed_const(3);
1981 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1982 bandwidth.full = dfixed_mul(dram_channels, yclk);
1983 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1984
1985 return dfixed_trunc(bandwidth);
1986}
1987
1988static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1989{
1990
1991 fixed20_12 return_efficiency;
1992 fixed20_12 sclk, bandwidth;
1993 fixed20_12 a;
1994
1995 a.full = dfixed_const(1000);
1996 sclk.full = dfixed_const(wm->sclk);
1997 sclk.full = dfixed_div(sclk, a);
1998 a.full = dfixed_const(10);
1999 return_efficiency.full = dfixed_const(8);
2000 return_efficiency.full = dfixed_div(return_efficiency, a);
2001 a.full = dfixed_const(32);
2002 bandwidth.full = dfixed_mul(a, sclk);
2003 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2004
2005 return dfixed_trunc(bandwidth);
2006}
2007
2008static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2009{
2010
2011 fixed20_12 disp_clk_request_efficiency;
2012 fixed20_12 disp_clk, bandwidth;
2013 fixed20_12 a;
2014
2015 a.full = dfixed_const(1000);
2016 disp_clk.full = dfixed_const(wm->disp_clk);
2017 disp_clk.full = dfixed_div(disp_clk, a);
2018 a.full = dfixed_const(10);
2019 disp_clk_request_efficiency.full = dfixed_const(8);
2020 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2021 a.full = dfixed_const(32);
2022 bandwidth.full = dfixed_mul(a, disp_clk);
2023 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2024
2025 return dfixed_trunc(bandwidth);
2026}
2027
2028static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2029{
2030
2031 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2032 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2033 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2034
2035 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2036}
2037
2038static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2039{
2040
2041
2042
2043
2044 fixed20_12 bpp;
2045 fixed20_12 line_time;
2046 fixed20_12 src_width;
2047 fixed20_12 bandwidth;
2048 fixed20_12 a;
2049
2050 a.full = dfixed_const(1000);
2051 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2052 line_time.full = dfixed_div(line_time, a);
2053 bpp.full = dfixed_const(wm->bytes_per_pixel);
2054 src_width.full = dfixed_const(wm->src_width);
2055 bandwidth.full = dfixed_mul(src_width, bpp);
2056 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2057 bandwidth.full = dfixed_div(bandwidth, line_time);
2058
2059 return dfixed_trunc(bandwidth);
2060}
2061
2062static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2063{
2064
2065 u32 mc_latency = 2000;
2066 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2067 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2068 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2069 u32 dc_latency = 40000000 / wm->disp_clk;
2070 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2071 (wm->num_heads * cursor_line_pair_return_time);
2072 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2073 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2074 fixed20_12 a, b, c;
2075
2076 if (wm->num_heads == 0)
2077 return 0;
2078
2079 a.full = dfixed_const(2);
2080 b.full = dfixed_const(1);
2081 if ((wm->vsc.full > a.full) ||
2082 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2083 (wm->vtaps >= 5) ||
2084 ((wm->vsc.full >= a.full) && wm->interlaced))
2085 max_src_lines_per_dst_line = 4;
2086 else
2087 max_src_lines_per_dst_line = 2;
2088
2089 a.full = dfixed_const(available_bandwidth);
2090 b.full = dfixed_const(wm->num_heads);
2091 a.full = dfixed_div(a, b);
2092
2093 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2094
2095 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2096 b.full = dfixed_const(1000);
2097 c.full = dfixed_const(lb_fill_bw);
2098 b.full = dfixed_div(c, b);
2099 a.full = dfixed_div(a, b);
2100 line_fill_time = dfixed_trunc(a);
2101
2102 if (line_fill_time < wm->active_time)
2103 return latency;
2104 else
2105 return latency + (line_fill_time - wm->active_time);
2106
2107}
2108
2109static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2110{
2111 if (evergreen_average_bandwidth(wm) <=
2112 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2113 return true;
2114 else
2115 return false;
2116};
2117
2118static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2119{
2120 if (evergreen_average_bandwidth(wm) <=
2121 (evergreen_available_bandwidth(wm) / wm->num_heads))
2122 return true;
2123 else
2124 return false;
2125};
2126
2127static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2128{
2129 u32 lb_partitions = wm->lb_size / wm->src_width;
2130 u32 line_time = wm->active_time + wm->blank_time;
2131 u32 latency_tolerant_lines;
2132 u32 latency_hiding;
2133 fixed20_12 a;
2134
2135 a.full = dfixed_const(1);
2136 if (wm->vsc.full > a.full)
2137 latency_tolerant_lines = 1;
2138 else {
2139 if (lb_partitions <= (wm->vtaps + 1))
2140 latency_tolerant_lines = 1;
2141 else
2142 latency_tolerant_lines = 2;
2143 }
2144
2145 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2146
2147 if (evergreen_latency_watermark(wm) <= latency_hiding)
2148 return true;
2149 else
2150 return false;
2151}
2152
2153static void evergreen_program_watermarks(struct radeon_device *rdev,
2154 struct radeon_crtc *radeon_crtc,
2155 u32 lb_size, u32 num_heads)
2156{
2157 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2158 struct evergreen_wm_params wm_low, wm_high;
2159 u32 dram_channels;
2160 u32 active_time;
2161 u32 line_time = 0;
2162 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2163 u32 priority_a_mark = 0, priority_b_mark = 0;
2164 u32 priority_a_cnt = PRIORITY_OFF;
2165 u32 priority_b_cnt = PRIORITY_OFF;
2166 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2167 u32 tmp, arb_control3;
2168 fixed20_12 a, b, c;
2169
2170 if (radeon_crtc->base.enabled && num_heads && mode) {
2171 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2172 (u32)mode->clock);
2173 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2174 (u32)mode->clock);
2175 line_time = min(line_time, (u32)65535);
2176 priority_a_cnt = 0;
2177 priority_b_cnt = 0;
2178 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2179
2180
2181 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2182 wm_high.yclk =
2183 radeon_dpm_get_mclk(rdev, false) * 10;
2184 wm_high.sclk =
2185 radeon_dpm_get_sclk(rdev, false) * 10;
2186 } else {
2187 wm_high.yclk = rdev->pm.current_mclk * 10;
2188 wm_high.sclk = rdev->pm.current_sclk * 10;
2189 }
2190
2191 wm_high.disp_clk = mode->clock;
2192 wm_high.src_width = mode->crtc_hdisplay;
2193 wm_high.active_time = active_time;
2194 wm_high.blank_time = line_time - wm_high.active_time;
2195 wm_high.interlaced = false;
2196 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2197 wm_high.interlaced = true;
2198 wm_high.vsc = radeon_crtc->vsc;
2199 wm_high.vtaps = 1;
2200 if (radeon_crtc->rmx_type != RMX_OFF)
2201 wm_high.vtaps = 2;
2202 wm_high.bytes_per_pixel = 4;
2203 wm_high.lb_size = lb_size;
2204 wm_high.dram_channels = dram_channels;
2205 wm_high.num_heads = num_heads;
2206
2207
2208 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2209 wm_low.yclk =
2210 radeon_dpm_get_mclk(rdev, true) * 10;
2211 wm_low.sclk =
2212 radeon_dpm_get_sclk(rdev, true) * 10;
2213 } else {
2214 wm_low.yclk = rdev->pm.current_mclk * 10;
2215 wm_low.sclk = rdev->pm.current_sclk * 10;
2216 }
2217
2218 wm_low.disp_clk = mode->clock;
2219 wm_low.src_width = mode->crtc_hdisplay;
2220 wm_low.active_time = active_time;
2221 wm_low.blank_time = line_time - wm_low.active_time;
2222 wm_low.interlaced = false;
2223 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2224 wm_low.interlaced = true;
2225 wm_low.vsc = radeon_crtc->vsc;
2226 wm_low.vtaps = 1;
2227 if (radeon_crtc->rmx_type != RMX_OFF)
2228 wm_low.vtaps = 2;
2229 wm_low.bytes_per_pixel = 4;
2230 wm_low.lb_size = lb_size;
2231 wm_low.dram_channels = dram_channels;
2232 wm_low.num_heads = num_heads;
2233
2234
2235 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2236
2237 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2238
2239
2240
2241 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2242 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2243 !evergreen_check_latency_hiding(&wm_high) ||
2244 (rdev->disp_priority == 2)) {
2245 DRM_DEBUG_KMS("force priority a to high\n");
2246 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2247 }
2248 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2249 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2250 !evergreen_check_latency_hiding(&wm_low) ||
2251 (rdev->disp_priority == 2)) {
2252 DRM_DEBUG_KMS("force priority b to high\n");
2253 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2254 }
2255
2256 a.full = dfixed_const(1000);
2257 b.full = dfixed_const(mode->clock);
2258 b.full = dfixed_div(b, a);
2259 c.full = dfixed_const(latency_watermark_a);
2260 c.full = dfixed_mul(c, b);
2261 c.full = dfixed_mul(c, radeon_crtc->hsc);
2262 c.full = dfixed_div(c, a);
2263 a.full = dfixed_const(16);
2264 c.full = dfixed_div(c, a);
2265 priority_a_mark = dfixed_trunc(c);
2266 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2267
2268 a.full = dfixed_const(1000);
2269 b.full = dfixed_const(mode->clock);
2270 b.full = dfixed_div(b, a);
2271 c.full = dfixed_const(latency_watermark_b);
2272 c.full = dfixed_mul(c, b);
2273 c.full = dfixed_mul(c, radeon_crtc->hsc);
2274 c.full = dfixed_div(c, a);
2275 a.full = dfixed_const(16);
2276 c.full = dfixed_div(c, a);
2277 priority_b_mark = dfixed_trunc(c);
2278 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2279
2280
2281 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2282 }
2283
2284
2285 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2286 tmp = arb_control3;
2287 tmp &= ~LATENCY_WATERMARK_MASK(3);
2288 tmp |= LATENCY_WATERMARK_MASK(1);
2289 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2290 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2291 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2292 LATENCY_HIGH_WATERMARK(line_time)));
2293
2294 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2295 tmp &= ~LATENCY_WATERMARK_MASK(3);
2296 tmp |= LATENCY_WATERMARK_MASK(2);
2297 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2298 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2299 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2300 LATENCY_HIGH_WATERMARK(line_time)));
2301
2302 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2303
2304
2305 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2306 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2307
2308
2309 radeon_crtc->line_time = line_time;
2310 radeon_crtc->wm_high = latency_watermark_a;
2311 radeon_crtc->wm_low = latency_watermark_b;
2312}
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322void evergreen_bandwidth_update(struct radeon_device *rdev)
2323{
2324 struct drm_display_mode *mode0 = NULL;
2325 struct drm_display_mode *mode1 = NULL;
2326 u32 num_heads = 0, lb_size;
2327 int i;
2328
2329 if (!rdev->mode_info.mode_config_initialized)
2330 return;
2331
2332 radeon_update_display_priority(rdev);
2333
2334 for (i = 0; i < rdev->num_crtc; i++) {
2335 if (rdev->mode_info.crtcs[i]->base.enabled)
2336 num_heads++;
2337 }
2338 for (i = 0; i < rdev->num_crtc; i += 2) {
2339 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2340 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2341 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2342 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2343 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2344 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2345 }
2346}
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2358{
2359 unsigned i;
2360 u32 tmp;
2361
2362 for (i = 0; i < rdev->usec_timeout; i++) {
2363
2364 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2365 if (!tmp)
2366 return 0;
2367 udelay(1);
2368 }
2369 return -1;
2370}
2371
2372
2373
2374
2375void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2376{
2377 unsigned i;
2378 u32 tmp;
2379
2380 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2381
2382 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2383 for (i = 0; i < rdev->usec_timeout; i++) {
2384
2385 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2386 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2387 if (tmp == 2) {
2388 pr_warn("[drm] r600 flush TLB failed\n");
2389 return;
2390 }
2391 if (tmp) {
2392 return;
2393 }
2394 udelay(1);
2395 }
2396}
2397
2398static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2399{
2400 u32 tmp;
2401 int r;
2402
2403 if (rdev->gart.robj == NULL) {
2404 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2405 return -EINVAL;
2406 }
2407 r = radeon_gart_table_vram_pin(rdev);
2408 if (r)
2409 return r;
2410
2411 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2412 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2413 EFFECTIVE_L2_QUEUE_SIZE(7));
2414 WREG32(VM_L2_CNTL2, 0);
2415 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2416
2417 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2418 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2419 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2420 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2421 if (rdev->flags & RADEON_IS_IGP) {
2422 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2423 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2424 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2425 } else {
2426 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2427 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2428 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2429 if ((rdev->family == CHIP_JUNIPER) ||
2430 (rdev->family == CHIP_CYPRESS) ||
2431 (rdev->family == CHIP_HEMLOCK) ||
2432 (rdev->family == CHIP_BARTS))
2433 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2434 }
2435 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2439 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2440 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2441 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2442 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2443 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2444 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2445 (u32)(rdev->dummy_page.addr >> 12));
2446 WREG32(VM_CONTEXT1_CNTL, 0);
2447
2448 evergreen_pcie_gart_tlb_flush(rdev);
2449 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2450 (unsigned)(rdev->mc.gtt_size >> 20),
2451 (unsigned long long)rdev->gart.table_addr);
2452 rdev->gart.ready = true;
2453 return 0;
2454}
2455
2456static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2457{
2458 u32 tmp;
2459
2460
2461 WREG32(VM_CONTEXT0_CNTL, 0);
2462 WREG32(VM_CONTEXT1_CNTL, 0);
2463
2464
2465 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2466 EFFECTIVE_L2_QUEUE_SIZE(7));
2467 WREG32(VM_L2_CNTL2, 0);
2468 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2469
2470 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2471 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2472 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2473 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2474 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2475 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2476 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2477 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2478 radeon_gart_table_vram_unpin(rdev);
2479}
2480
2481static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2482{
2483 evergreen_pcie_gart_disable(rdev);
2484 radeon_gart_table_vram_free(rdev);
2485 radeon_gart_fini(rdev);
2486}
2487
2488
2489static void evergreen_agp_enable(struct radeon_device *rdev)
2490{
2491 u32 tmp;
2492
2493
2494 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2495 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2496 EFFECTIVE_L2_QUEUE_SIZE(7));
2497 WREG32(VM_L2_CNTL2, 0);
2498 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2499
2500 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2501 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2502 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2503 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2504 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2505 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2506 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2507 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2508 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2509 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2510 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2511 WREG32(VM_CONTEXT0_CNTL, 0);
2512 WREG32(VM_CONTEXT1_CNTL, 0);
2513}
2514
2515static const unsigned ni_dig_offsets[] =
2516{
2517 NI_DIG0_REGISTER_OFFSET,
2518 NI_DIG1_REGISTER_OFFSET,
2519 NI_DIG2_REGISTER_OFFSET,
2520 NI_DIG3_REGISTER_OFFSET,
2521 NI_DIG4_REGISTER_OFFSET,
2522 NI_DIG5_REGISTER_OFFSET
2523};
2524
2525static const unsigned ni_tx_offsets[] =
2526{
2527 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2533};
2534
2535static const unsigned evergreen_dp_offsets[] =
2536{
2537 EVERGREEN_DP0_REGISTER_OFFSET,
2538 EVERGREEN_DP1_REGISTER_OFFSET,
2539 EVERGREEN_DP2_REGISTER_OFFSET,
2540 EVERGREEN_DP3_REGISTER_OFFSET,
2541 EVERGREEN_DP4_REGISTER_OFFSET,
2542 EVERGREEN_DP5_REGISTER_OFFSET
2543};
2544
2545static const unsigned evergreen_disp_int_status[] =
2546{
2547 DISP_INTERRUPT_STATUS,
2548 DISP_INTERRUPT_STATUS_CONTINUE,
2549 DISP_INTERRUPT_STATUS_CONTINUE2,
2550 DISP_INTERRUPT_STATUS_CONTINUE3,
2551 DISP_INTERRUPT_STATUS_CONTINUE4,
2552 DISP_INTERRUPT_STATUS_CONTINUE5
2553};
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2566 unsigned crtc_id, unsigned *ret_dig_fe)
2567{
2568 unsigned i;
2569 unsigned dig_fe;
2570 unsigned dig_be;
2571 unsigned dig_en_be;
2572 unsigned uniphy_pll;
2573 unsigned digs_fe_selected;
2574 unsigned dig_be_mode;
2575 unsigned dig_fe_mask;
2576 bool is_enabled = false;
2577 bool found_crtc = false;
2578
2579
2580 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2581 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2582 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2583 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2584
2585 found_crtc = true;
2586 dig_fe_mask = 1 << i;
2587 dig_fe = i;
2588 break;
2589 }
2590 }
2591
2592 if (found_crtc) {
2593
2594 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2595 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2596
2597 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2598 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2599 if (dig_fe_mask & digs_fe_selected &&
2600
2601 dig_be_mode == NI_DIG_BE_DPSST) {
2602 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2603 ni_dig_offsets[i]);
2604 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2605 ni_tx_offsets[i]);
2606
2607 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2608 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2609 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2610 is_enabled = true;
2611 *ret_dig_fe = dig_fe;
2612 break;
2613 }
2614 }
2615 }
2616 }
2617
2618 return is_enabled;
2619}
2620
2621
2622
2623
2624
2625static void evergreen_blank_dp_output(struct radeon_device *rdev,
2626 unsigned dig_fe)
2627{
2628 unsigned stream_ctrl;
2629 unsigned fifo_ctrl;
2630 unsigned counter = 0;
2631
2632 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2633 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2634 return;
2635 }
2636
2637 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2638 evergreen_dp_offsets[dig_fe]);
2639 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2640 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2641 return;
2642 }
2643
2644 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2645 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2646 evergreen_dp_offsets[dig_fe], stream_ctrl);
2647
2648 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2649 evergreen_dp_offsets[dig_fe]);
2650 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2651 msleep(1);
2652 counter++;
2653 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2654 evergreen_dp_offsets[dig_fe]);
2655 }
2656 if (counter >= 32 )
2657 DRM_ERROR("counter exceeds %d\n", counter);
2658
2659 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2660 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2661 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2662
2663}
2664
2665void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2666{
2667 u32 crtc_enabled, tmp, frame_count, blackout;
2668 int i, j;
2669 unsigned dig_fe;
2670
2671 if (!ASIC_IS_NODCE(rdev)) {
2672 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2673 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2674
2675
2676 WREG32(VGA_RENDER_CONTROL, 0);
2677 }
2678
2679 for (i = 0; i < rdev->num_crtc; i++) {
2680 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2681 if (crtc_enabled) {
2682 save->crtc_enabled[i] = true;
2683 if (ASIC_IS_DCE6(rdev)) {
2684 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2685 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2686 radeon_wait_for_vblank(rdev, i);
2687 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2688 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2689 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2690 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2691 }
2692 } else {
2693 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2694 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2695 radeon_wait_for_vblank(rdev, i);
2696 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2697 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2698 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2699 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2700 }
2701 }
2702
2703 frame_count = radeon_get_vblank_counter(rdev, i);
2704 for (j = 0; j < rdev->usec_timeout; j++) {
2705 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2706 break;
2707 udelay(1);
2708 }
2709
2710
2711
2712
2713
2714
2715
2716 if (ASIC_IS_DCE5(rdev) &&
2717 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2718 evergreen_blank_dp_output(rdev, dig_fe);
2719
2720
2721 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2722 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2723 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2724 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2725 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2726 save->crtc_enabled[i] = false;
2727
2728 } else {
2729 save->crtc_enabled[i] = false;
2730 }
2731 }
2732
2733 radeon_mc_wait_for_idle(rdev);
2734
2735 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2736 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2737
2738 WREG32(BIF_FB_EN, 0);
2739
2740 blackout &= ~BLACKOUT_MODE_MASK;
2741 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2742 }
2743
2744 udelay(100);
2745
2746
2747 for (i = 0; i < rdev->num_crtc; i++) {
2748 if (save->crtc_enabled[i]) {
2749 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2750 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2751 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2752 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2753 }
2754 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2755 if (!(tmp & 1)) {
2756 tmp |= 1;
2757 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2758 }
2759 }
2760 }
2761}
2762
2763void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2764{
2765 u32 tmp, frame_count;
2766 int i, j;
2767
2768
2769 for (i = 0; i < rdev->num_crtc; i++) {
2770 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771 upper_32_bits(rdev->mc.vram_start));
2772 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2773 upper_32_bits(rdev->mc.vram_start));
2774 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2775 (u32)rdev->mc.vram_start);
2776 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2777 (u32)rdev->mc.vram_start);
2778 }
2779
2780 if (!ASIC_IS_NODCE(rdev)) {
2781 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2782 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2783 }
2784
2785
2786 for (i = 0; i < rdev->num_crtc; i++) {
2787 if (save->crtc_enabled[i]) {
2788 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2789 if ((tmp & 0x7) != 0) {
2790 tmp &= ~0x7;
2791 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2792 }
2793 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2794 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2795 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2796 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2797 }
2798 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2799 if (tmp & 1) {
2800 tmp &= ~1;
2801 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2802 }
2803 for (j = 0; j < rdev->usec_timeout; j++) {
2804 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2805 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2806 break;
2807 udelay(1);
2808 }
2809 }
2810 }
2811
2812
2813 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2814 tmp &= ~BLACKOUT_MODE_MASK;
2815 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2816
2817 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2818
2819 for (i = 0; i < rdev->num_crtc; i++) {
2820 if (save->crtc_enabled[i]) {
2821 if (ASIC_IS_DCE6(rdev)) {
2822 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2823 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2824 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2825 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2826 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2827 } else {
2828 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2829 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2830 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2831 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2832 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2833 }
2834
2835 frame_count = radeon_get_vblank_counter(rdev, i);
2836 for (j = 0; j < rdev->usec_timeout; j++) {
2837 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2838 break;
2839 udelay(1);
2840 }
2841 }
2842 }
2843 if (!ASIC_IS_NODCE(rdev)) {
2844
2845 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2846 mdelay(1);
2847 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2848 }
2849}
2850
2851void evergreen_mc_program(struct radeon_device *rdev)
2852{
2853 struct evergreen_mc_save save;
2854 u32 tmp;
2855 int i, j;
2856
2857
2858 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2859 WREG32((0x2c14 + j), 0x00000000);
2860 WREG32((0x2c18 + j), 0x00000000);
2861 WREG32((0x2c1c + j), 0x00000000);
2862 WREG32((0x2c20 + j), 0x00000000);
2863 WREG32((0x2c24 + j), 0x00000000);
2864 }
2865 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2866
2867 evergreen_mc_stop(rdev, &save);
2868 if (evergreen_mc_wait_for_idle(rdev)) {
2869 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2870 }
2871
2872 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2873
2874 if (rdev->flags & RADEON_IS_AGP) {
2875 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2876
2877 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2878 rdev->mc.vram_start >> 12);
2879 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2880 rdev->mc.gtt_end >> 12);
2881 } else {
2882
2883 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2884 rdev->mc.gtt_start >> 12);
2885 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2886 rdev->mc.vram_end >> 12);
2887 }
2888 } else {
2889 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2890 rdev->mc.vram_start >> 12);
2891 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2892 rdev->mc.vram_end >> 12);
2893 }
2894 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2895
2896 if ((rdev->family == CHIP_PALM) ||
2897 (rdev->family == CHIP_SUMO) ||
2898 (rdev->family == CHIP_SUMO2)) {
2899 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2900 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2901 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2902 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2903 }
2904 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2905 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2906 WREG32(MC_VM_FB_LOCATION, tmp);
2907 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2908 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2909 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2910 if (rdev->flags & RADEON_IS_AGP) {
2911 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2912 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2913 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2914 } else {
2915 WREG32(MC_VM_AGP_BASE, 0);
2916 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2917 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2918 }
2919 if (evergreen_mc_wait_for_idle(rdev)) {
2920 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2921 }
2922 evergreen_mc_resume(rdev, &save);
2923
2924
2925 rv515_vga_render_disable(rdev);
2926}
2927
2928
2929
2930
2931void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2932{
2933 struct radeon_ring *ring = &rdev->ring[ib->ring];
2934 u32 next_rptr;
2935
2936
2937 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2938 radeon_ring_write(ring, 1);
2939
2940 if (ring->rptr_save_reg) {
2941 next_rptr = ring->wptr + 3 + 4;
2942 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2943 radeon_ring_write(ring, ((ring->rptr_save_reg -
2944 PACKET3_SET_CONFIG_REG_START) >> 2));
2945 radeon_ring_write(ring, next_rptr);
2946 } else if (rdev->wb.enabled) {
2947 next_rptr = ring->wptr + 5 + 4;
2948 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2949 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2950 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2951 radeon_ring_write(ring, next_rptr);
2952 radeon_ring_write(ring, 0);
2953 }
2954
2955 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2956 radeon_ring_write(ring,
2957#ifdef __BIG_ENDIAN
2958 (2 << 0) |
2959#endif
2960 (ib->gpu_addr & 0xFFFFFFFC));
2961 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2962 radeon_ring_write(ring, ib->length_dw);
2963}
2964
2965
2966static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2967{
2968 const __be32 *fw_data;
2969 int i;
2970
2971 if (!rdev->me_fw || !rdev->pfp_fw)
2972 return -EINVAL;
2973
2974 r700_cp_stop(rdev);
2975 WREG32(CP_RB_CNTL,
2976#ifdef __BIG_ENDIAN
2977 BUF_SWAP_32BIT |
2978#endif
2979 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2980
2981 fw_data = (const __be32 *)rdev->pfp_fw->data;
2982 WREG32(CP_PFP_UCODE_ADDR, 0);
2983 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2984 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2985 WREG32(CP_PFP_UCODE_ADDR, 0);
2986
2987 fw_data = (const __be32 *)rdev->me_fw->data;
2988 WREG32(CP_ME_RAM_WADDR, 0);
2989 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2990 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2991
2992 WREG32(CP_PFP_UCODE_ADDR, 0);
2993 WREG32(CP_ME_RAM_WADDR, 0);
2994 WREG32(CP_ME_RAM_RADDR, 0);
2995 return 0;
2996}
2997
2998static int evergreen_cp_start(struct radeon_device *rdev)
2999{
3000 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3001 int r, i;
3002 uint32_t cp_me;
3003
3004 r = radeon_ring_lock(rdev, ring, 7);
3005 if (r) {
3006 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3007 return r;
3008 }
3009 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3010 radeon_ring_write(ring, 0x1);
3011 radeon_ring_write(ring, 0x0);
3012 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3013 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3014 radeon_ring_write(ring, 0);
3015 radeon_ring_write(ring, 0);
3016 radeon_ring_unlock_commit(rdev, ring, false);
3017
3018 cp_me = 0xff;
3019 WREG32(CP_ME_CNTL, cp_me);
3020
3021 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3022 if (r) {
3023 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3024 return r;
3025 }
3026
3027
3028 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3029 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3030
3031 for (i = 0; i < evergreen_default_size; i++)
3032 radeon_ring_write(ring, evergreen_default_state[i]);
3033
3034 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3035 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3036
3037
3038 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3039 radeon_ring_write(ring, 0);
3040
3041
3042 radeon_ring_write(ring, 0xc0026f00);
3043 radeon_ring_write(ring, 0x00000000);
3044 radeon_ring_write(ring, 0x00000000);
3045 radeon_ring_write(ring, 0x00000000);
3046
3047
3048 radeon_ring_write(ring, 0xc0036f00);
3049 radeon_ring_write(ring, 0x00000bc4);
3050 radeon_ring_write(ring, 0xffffffff);
3051 radeon_ring_write(ring, 0xffffffff);
3052 radeon_ring_write(ring, 0xffffffff);
3053
3054 radeon_ring_write(ring, 0xc0026900);
3055 radeon_ring_write(ring, 0x00000316);
3056 radeon_ring_write(ring, 0x0000000e);
3057 radeon_ring_write(ring, 0x00000010);
3058
3059 radeon_ring_unlock_commit(rdev, ring, false);
3060
3061 return 0;
3062}
3063
3064static int evergreen_cp_resume(struct radeon_device *rdev)
3065{
3066 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3067 u32 tmp;
3068 u32 rb_bufsz;
3069 int r;
3070
3071
3072 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3073 SOFT_RESET_PA |
3074 SOFT_RESET_SH |
3075 SOFT_RESET_VGT |
3076 SOFT_RESET_SPI |
3077 SOFT_RESET_SX));
3078 RREG32(GRBM_SOFT_RESET);
3079 mdelay(15);
3080 WREG32(GRBM_SOFT_RESET, 0);
3081 RREG32(GRBM_SOFT_RESET);
3082
3083
3084 rb_bufsz = order_base_2(ring->ring_size / 8);
3085 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3086#ifdef __BIG_ENDIAN
3087 tmp |= BUF_SWAP_32BIT;
3088#endif
3089 WREG32(CP_RB_CNTL, tmp);
3090 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3091 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3092
3093
3094 WREG32(CP_RB_WPTR_DELAY, 0);
3095
3096
3097 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3098 WREG32(CP_RB_RPTR_WR, 0);
3099 ring->wptr = 0;
3100 WREG32(CP_RB_WPTR, ring->wptr);
3101
3102
3103 WREG32(CP_RB_RPTR_ADDR,
3104 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3105 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3106 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3107
3108 if (rdev->wb.enabled)
3109 WREG32(SCRATCH_UMSK, 0xff);
3110 else {
3111 tmp |= RB_NO_UPDATE;
3112 WREG32(SCRATCH_UMSK, 0);
3113 }
3114
3115 mdelay(1);
3116 WREG32(CP_RB_CNTL, tmp);
3117
3118 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3119 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3120
3121 evergreen_cp_start(rdev);
3122 ring->ready = true;
3123 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3124 if (r) {
3125 ring->ready = false;
3126 return r;
3127 }
3128 return 0;
3129}
3130
3131
3132
3133
3134static void evergreen_gpu_init(struct radeon_device *rdev)
3135{
3136 u32 gb_addr_config;
3137 u32 mc_arb_ramcfg;
3138 u32 sx_debug_1;
3139 u32 smx_dc_ctl0;
3140 u32 sq_config;
3141 u32 sq_lds_resource_mgmt;
3142 u32 sq_gpr_resource_mgmt_1;
3143 u32 sq_gpr_resource_mgmt_2;
3144 u32 sq_gpr_resource_mgmt_3;
3145 u32 sq_thread_resource_mgmt;
3146 u32 sq_thread_resource_mgmt_2;
3147 u32 sq_stack_resource_mgmt_1;
3148 u32 sq_stack_resource_mgmt_2;
3149 u32 sq_stack_resource_mgmt_3;
3150 u32 vgt_cache_invalidation;
3151 u32 hdp_host_path_cntl, tmp;
3152 u32 disabled_rb_mask;
3153 int i, j, ps_thread_count;
3154
3155 switch (rdev->family) {
3156 case CHIP_CYPRESS:
3157 case CHIP_HEMLOCK:
3158 rdev->config.evergreen.num_ses = 2;
3159 rdev->config.evergreen.max_pipes = 4;
3160 rdev->config.evergreen.max_tile_pipes = 8;
3161 rdev->config.evergreen.max_simds = 10;
3162 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3163 rdev->config.evergreen.max_gprs = 256;
3164 rdev->config.evergreen.max_threads = 248;
3165 rdev->config.evergreen.max_gs_threads = 32;
3166 rdev->config.evergreen.max_stack_entries = 512;
3167 rdev->config.evergreen.sx_num_of_sets = 4;
3168 rdev->config.evergreen.sx_max_export_size = 256;
3169 rdev->config.evergreen.sx_max_export_pos_size = 64;
3170 rdev->config.evergreen.sx_max_export_smx_size = 192;
3171 rdev->config.evergreen.max_hw_contexts = 8;
3172 rdev->config.evergreen.sq_num_cf_insts = 2;
3173
3174 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3175 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3177 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3178 break;
3179 case CHIP_JUNIPER:
3180 rdev->config.evergreen.num_ses = 1;
3181 rdev->config.evergreen.max_pipes = 4;
3182 rdev->config.evergreen.max_tile_pipes = 4;
3183 rdev->config.evergreen.max_simds = 10;
3184 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3185 rdev->config.evergreen.max_gprs = 256;
3186 rdev->config.evergreen.max_threads = 248;
3187 rdev->config.evergreen.max_gs_threads = 32;
3188 rdev->config.evergreen.max_stack_entries = 512;
3189 rdev->config.evergreen.sx_num_of_sets = 4;
3190 rdev->config.evergreen.sx_max_export_size = 256;
3191 rdev->config.evergreen.sx_max_export_pos_size = 64;
3192 rdev->config.evergreen.sx_max_export_smx_size = 192;
3193 rdev->config.evergreen.max_hw_contexts = 8;
3194 rdev->config.evergreen.sq_num_cf_insts = 2;
3195
3196 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3197 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3198 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3199 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3200 break;
3201 case CHIP_REDWOOD:
3202 rdev->config.evergreen.num_ses = 1;
3203 rdev->config.evergreen.max_pipes = 4;
3204 rdev->config.evergreen.max_tile_pipes = 4;
3205 rdev->config.evergreen.max_simds = 5;
3206 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3207 rdev->config.evergreen.max_gprs = 256;
3208 rdev->config.evergreen.max_threads = 248;
3209 rdev->config.evergreen.max_gs_threads = 32;
3210 rdev->config.evergreen.max_stack_entries = 256;
3211 rdev->config.evergreen.sx_num_of_sets = 4;
3212 rdev->config.evergreen.sx_max_export_size = 256;
3213 rdev->config.evergreen.sx_max_export_pos_size = 64;
3214 rdev->config.evergreen.sx_max_export_smx_size = 192;
3215 rdev->config.evergreen.max_hw_contexts = 8;
3216 rdev->config.evergreen.sq_num_cf_insts = 2;
3217
3218 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3219 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3220 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3221 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3222 break;
3223 case CHIP_CEDAR:
3224 default:
3225 rdev->config.evergreen.num_ses = 1;
3226 rdev->config.evergreen.max_pipes = 2;
3227 rdev->config.evergreen.max_tile_pipes = 2;
3228 rdev->config.evergreen.max_simds = 2;
3229 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3230 rdev->config.evergreen.max_gprs = 256;
3231 rdev->config.evergreen.max_threads = 192;
3232 rdev->config.evergreen.max_gs_threads = 16;
3233 rdev->config.evergreen.max_stack_entries = 256;
3234 rdev->config.evergreen.sx_num_of_sets = 4;
3235 rdev->config.evergreen.sx_max_export_size = 128;
3236 rdev->config.evergreen.sx_max_export_pos_size = 32;
3237 rdev->config.evergreen.sx_max_export_smx_size = 96;
3238 rdev->config.evergreen.max_hw_contexts = 4;
3239 rdev->config.evergreen.sq_num_cf_insts = 1;
3240
3241 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3242 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3243 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3244 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3245 break;
3246 case CHIP_PALM:
3247 rdev->config.evergreen.num_ses = 1;
3248 rdev->config.evergreen.max_pipes = 2;
3249 rdev->config.evergreen.max_tile_pipes = 2;
3250 rdev->config.evergreen.max_simds = 2;
3251 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3252 rdev->config.evergreen.max_gprs = 256;
3253 rdev->config.evergreen.max_threads = 192;
3254 rdev->config.evergreen.max_gs_threads = 16;
3255 rdev->config.evergreen.max_stack_entries = 256;
3256 rdev->config.evergreen.sx_num_of_sets = 4;
3257 rdev->config.evergreen.sx_max_export_size = 128;
3258 rdev->config.evergreen.sx_max_export_pos_size = 32;
3259 rdev->config.evergreen.sx_max_export_smx_size = 96;
3260 rdev->config.evergreen.max_hw_contexts = 4;
3261 rdev->config.evergreen.sq_num_cf_insts = 1;
3262
3263 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3264 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3265 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3266 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3267 break;
3268 case CHIP_SUMO:
3269 rdev->config.evergreen.num_ses = 1;
3270 rdev->config.evergreen.max_pipes = 4;
3271 rdev->config.evergreen.max_tile_pipes = 4;
3272 if (rdev->pdev->device == 0x9648)
3273 rdev->config.evergreen.max_simds = 3;
3274 else if ((rdev->pdev->device == 0x9647) ||
3275 (rdev->pdev->device == 0x964a))
3276 rdev->config.evergreen.max_simds = 4;
3277 else
3278 rdev->config.evergreen.max_simds = 5;
3279 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3280 rdev->config.evergreen.max_gprs = 256;
3281 rdev->config.evergreen.max_threads = 248;
3282 rdev->config.evergreen.max_gs_threads = 32;
3283 rdev->config.evergreen.max_stack_entries = 256;
3284 rdev->config.evergreen.sx_num_of_sets = 4;
3285 rdev->config.evergreen.sx_max_export_size = 256;
3286 rdev->config.evergreen.sx_max_export_pos_size = 64;
3287 rdev->config.evergreen.sx_max_export_smx_size = 192;
3288 rdev->config.evergreen.max_hw_contexts = 8;
3289 rdev->config.evergreen.sq_num_cf_insts = 2;
3290
3291 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3292 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3293 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3294 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3295 break;
3296 case CHIP_SUMO2:
3297 rdev->config.evergreen.num_ses = 1;
3298 rdev->config.evergreen.max_pipes = 4;
3299 rdev->config.evergreen.max_tile_pipes = 4;
3300 rdev->config.evergreen.max_simds = 2;
3301 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3302 rdev->config.evergreen.max_gprs = 256;
3303 rdev->config.evergreen.max_threads = 248;
3304 rdev->config.evergreen.max_gs_threads = 32;
3305 rdev->config.evergreen.max_stack_entries = 512;
3306 rdev->config.evergreen.sx_num_of_sets = 4;
3307 rdev->config.evergreen.sx_max_export_size = 256;
3308 rdev->config.evergreen.sx_max_export_pos_size = 64;
3309 rdev->config.evergreen.sx_max_export_smx_size = 192;
3310 rdev->config.evergreen.max_hw_contexts = 4;
3311 rdev->config.evergreen.sq_num_cf_insts = 2;
3312
3313 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3314 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3315 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3316 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3317 break;
3318 case CHIP_BARTS:
3319 rdev->config.evergreen.num_ses = 2;
3320 rdev->config.evergreen.max_pipes = 4;
3321 rdev->config.evergreen.max_tile_pipes = 8;
3322 rdev->config.evergreen.max_simds = 7;
3323 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3324 rdev->config.evergreen.max_gprs = 256;
3325 rdev->config.evergreen.max_threads = 248;
3326 rdev->config.evergreen.max_gs_threads = 32;
3327 rdev->config.evergreen.max_stack_entries = 512;
3328 rdev->config.evergreen.sx_num_of_sets = 4;
3329 rdev->config.evergreen.sx_max_export_size = 256;
3330 rdev->config.evergreen.sx_max_export_pos_size = 64;
3331 rdev->config.evergreen.sx_max_export_smx_size = 192;
3332 rdev->config.evergreen.max_hw_contexts = 8;
3333 rdev->config.evergreen.sq_num_cf_insts = 2;
3334
3335 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3336 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3337 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3338 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3339 break;
3340 case CHIP_TURKS:
3341 rdev->config.evergreen.num_ses = 1;
3342 rdev->config.evergreen.max_pipes = 4;
3343 rdev->config.evergreen.max_tile_pipes = 4;
3344 rdev->config.evergreen.max_simds = 6;
3345 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3346 rdev->config.evergreen.max_gprs = 256;
3347 rdev->config.evergreen.max_threads = 248;
3348 rdev->config.evergreen.max_gs_threads = 32;
3349 rdev->config.evergreen.max_stack_entries = 256;
3350 rdev->config.evergreen.sx_num_of_sets = 4;
3351 rdev->config.evergreen.sx_max_export_size = 256;
3352 rdev->config.evergreen.sx_max_export_pos_size = 64;
3353 rdev->config.evergreen.sx_max_export_smx_size = 192;
3354 rdev->config.evergreen.max_hw_contexts = 8;
3355 rdev->config.evergreen.sq_num_cf_insts = 2;
3356
3357 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3358 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3359 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3360 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3361 break;
3362 case CHIP_CAICOS:
3363 rdev->config.evergreen.num_ses = 1;
3364 rdev->config.evergreen.max_pipes = 2;
3365 rdev->config.evergreen.max_tile_pipes = 2;
3366 rdev->config.evergreen.max_simds = 2;
3367 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3368 rdev->config.evergreen.max_gprs = 256;
3369 rdev->config.evergreen.max_threads = 192;
3370 rdev->config.evergreen.max_gs_threads = 16;
3371 rdev->config.evergreen.max_stack_entries = 256;
3372 rdev->config.evergreen.sx_num_of_sets = 4;
3373 rdev->config.evergreen.sx_max_export_size = 128;
3374 rdev->config.evergreen.sx_max_export_pos_size = 32;
3375 rdev->config.evergreen.sx_max_export_smx_size = 96;
3376 rdev->config.evergreen.max_hw_contexts = 4;
3377 rdev->config.evergreen.sq_num_cf_insts = 1;
3378
3379 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3380 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3381 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3382 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3383 break;
3384 }
3385
3386
3387 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3388 WREG32((0x2c14 + j), 0x00000000);
3389 WREG32((0x2c18 + j), 0x00000000);
3390 WREG32((0x2c1c + j), 0x00000000);
3391 WREG32((0x2c20 + j), 0x00000000);
3392 WREG32((0x2c24 + j), 0x00000000);
3393 }
3394
3395 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3396 WREG32(SRBM_INT_CNTL, 0x1);
3397 WREG32(SRBM_INT_ACK, 0x1);
3398
3399 evergreen_fix_pci_max_read_req_size(rdev);
3400
3401 RREG32(MC_SHARED_CHMAP);
3402 if ((rdev->family == CHIP_PALM) ||
3403 (rdev->family == CHIP_SUMO) ||
3404 (rdev->family == CHIP_SUMO2))
3405 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3406 else
3407 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3408
3409
3410
3411
3412
3413
3414
3415
3416 rdev->config.evergreen.tile_config = 0;
3417 switch (rdev->config.evergreen.max_tile_pipes) {
3418 case 1:
3419 default:
3420 rdev->config.evergreen.tile_config |= (0 << 0);
3421 break;
3422 case 2:
3423 rdev->config.evergreen.tile_config |= (1 << 0);
3424 break;
3425 case 4:
3426 rdev->config.evergreen.tile_config |= (2 << 0);
3427 break;
3428 case 8:
3429 rdev->config.evergreen.tile_config |= (3 << 0);
3430 break;
3431 }
3432
3433 if (rdev->flags & RADEON_IS_IGP)
3434 rdev->config.evergreen.tile_config |= 1 << 4;
3435 else {
3436 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3437 case 0:
3438 rdev->config.evergreen.tile_config |= 0 << 4;
3439 break;
3440 case 1:
3441 rdev->config.evergreen.tile_config |= 1 << 4;
3442 break;
3443 case 2:
3444 default:
3445 rdev->config.evergreen.tile_config |= 2 << 4;
3446 break;
3447 }
3448 }
3449 rdev->config.evergreen.tile_config |= 0 << 8;
3450 rdev->config.evergreen.tile_config |=
3451 ((gb_addr_config & 0x30000000) >> 28) << 12;
3452
3453 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3454 u32 efuse_straps_4;
3455 u32 efuse_straps_3;
3456
3457 efuse_straps_4 = RREG32_RCU(0x204);
3458 efuse_straps_3 = RREG32_RCU(0x203);
3459 tmp = (((efuse_straps_4 & 0xf) << 4) |
3460 ((efuse_straps_3 & 0xf0000000) >> 28));
3461 } else {
3462 tmp = 0;
3463 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3464 u32 rb_disable_bitmap;
3465
3466 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3468 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3469 tmp <<= 4;
3470 tmp |= rb_disable_bitmap;
3471 }
3472 }
3473
3474 disabled_rb_mask = tmp;
3475 tmp = 0;
3476 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3477 tmp |= (1 << i);
3478
3479 if ((disabled_rb_mask & tmp) == tmp) {
3480 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3481 disabled_rb_mask &= ~(1 << i);
3482 }
3483
3484 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3485 u32 simd_disable_bitmap;
3486
3487 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3489 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3490 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3491 tmp <<= 16;
3492 tmp |= simd_disable_bitmap;
3493 }
3494 rdev->config.evergreen.active_simds = hweight32(~tmp);
3495
3496 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3498
3499 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3500 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3501 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3502 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3503 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3504 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3505 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3506
3507 if ((rdev->config.evergreen.max_backends == 1) &&
3508 (rdev->flags & RADEON_IS_IGP)) {
3509 if ((disabled_rb_mask & 3) == 1) {
3510
3511 tmp = 0x11111111;
3512 } else {
3513
3514 tmp = 0x00000000;
3515 }
3516 } else {
3517 tmp = gb_addr_config & NUM_PIPES_MASK;
3518 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3519 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3520 }
3521 rdev->config.evergreen.backend_map = tmp;
3522 WREG32(GB_BACKEND_MAP, tmp);
3523
3524 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3525 WREG32(CGTS_TCC_DISABLE, 0);
3526 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3527 WREG32(CGTS_USER_TCC_DISABLE, 0);
3528
3529
3530 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3531 ROQ_IB2_START(0x2b)));
3532
3533 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3534
3535 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3536 SYNC_GRADIENT |
3537 SYNC_WALKER |
3538 SYNC_ALIGNER));
3539
3540 sx_debug_1 = RREG32(SX_DEBUG_1);
3541 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3542 WREG32(SX_DEBUG_1, sx_debug_1);
3543
3544
3545 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3546 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3547 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3548 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3549
3550 if (rdev->family <= CHIP_SUMO2)
3551 WREG32(SMX_SAR_CTL0, 0x00010000);
3552
3553 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3554 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3555 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3556
3557 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3558 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3559 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3560
3561 WREG32(VGT_NUM_INSTANCES, 1);
3562 WREG32(SPI_CONFIG_CNTL, 0);
3563 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3564 WREG32(CP_PERFMON_CNTL, 0);
3565
3566 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3567 FETCH_FIFO_HIWATER(0x4) |
3568 DONE_FIFO_HIWATER(0xe0) |
3569 ALU_UPDATE_FIFO_HIWATER(0x8)));
3570
3571 sq_config = RREG32(SQ_CONFIG);
3572 sq_config &= ~(PS_PRIO(3) |
3573 VS_PRIO(3) |
3574 GS_PRIO(3) |
3575 ES_PRIO(3));
3576 sq_config |= (VC_ENABLE |
3577 EXPORT_SRC_C |
3578 PS_PRIO(0) |
3579 VS_PRIO(1) |
3580 GS_PRIO(2) |
3581 ES_PRIO(3));
3582
3583 switch (rdev->family) {
3584 case CHIP_CEDAR:
3585 case CHIP_PALM:
3586 case CHIP_SUMO:
3587 case CHIP_SUMO2:
3588 case CHIP_CAICOS:
3589
3590 sq_config &= ~VC_ENABLE;
3591 break;
3592 default:
3593 break;
3594 }
3595
3596 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3597
3598 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3599 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3600 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3601 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3603 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3605
3606 switch (rdev->family) {
3607 case CHIP_CEDAR:
3608 case CHIP_PALM:
3609 case CHIP_SUMO:
3610 case CHIP_SUMO2:
3611 ps_thread_count = 96;
3612 break;
3613 default:
3614 ps_thread_count = 128;
3615 break;
3616 }
3617
3618 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3619 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3624
3625 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3631
3632 WREG32(SQ_CONFIG, sq_config);
3633 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3634 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3635 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3636 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3637 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3638 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3639 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3640 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3641 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3642 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3643
3644 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3645 FORCE_EOV_MAX_REZ_CNT(255)));
3646
3647 switch (rdev->family) {
3648 case CHIP_CEDAR:
3649 case CHIP_PALM:
3650 case CHIP_SUMO:
3651 case CHIP_SUMO2:
3652 case CHIP_CAICOS:
3653 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3654 break;
3655 default:
3656 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3657 break;
3658 }
3659 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3660 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3661
3662 WREG32(VGT_GS_VERTEX_REUSE, 16);
3663 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3664 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3665
3666 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3667 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3668
3669 WREG32(CB_PERF_CTR0_SEL_0, 0);
3670 WREG32(CB_PERF_CTR0_SEL_1, 0);
3671 WREG32(CB_PERF_CTR1_SEL_0, 0);
3672 WREG32(CB_PERF_CTR1_SEL_1, 0);
3673 WREG32(CB_PERF_CTR2_SEL_0, 0);
3674 WREG32(CB_PERF_CTR2_SEL_1, 0);
3675 WREG32(CB_PERF_CTR3_SEL_0, 0);
3676 WREG32(CB_PERF_CTR3_SEL_1, 0);
3677
3678
3679 WREG32(CB_COLOR0_BASE, 0);
3680 WREG32(CB_COLOR1_BASE, 0);
3681 WREG32(CB_COLOR2_BASE, 0);
3682 WREG32(CB_COLOR3_BASE, 0);
3683 WREG32(CB_COLOR4_BASE, 0);
3684 WREG32(CB_COLOR5_BASE, 0);
3685 WREG32(CB_COLOR6_BASE, 0);
3686 WREG32(CB_COLOR7_BASE, 0);
3687 WREG32(CB_COLOR8_BASE, 0);
3688 WREG32(CB_COLOR9_BASE, 0);
3689 WREG32(CB_COLOR10_BASE, 0);
3690 WREG32(CB_COLOR11_BASE, 0);
3691
3692
3693 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3694 WREG32(i, 0);
3695 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3696 WREG32(i, 0);
3697
3698 tmp = RREG32(HDP_MISC_CNTL);
3699 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3700 WREG32(HDP_MISC_CNTL, tmp);
3701
3702 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3703 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3704
3705 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3706
3707 udelay(50);
3708
3709}
3710
3711int evergreen_mc_init(struct radeon_device *rdev)
3712{
3713 u32 tmp;
3714 int chansize, numchan;
3715
3716
3717 rdev->mc.vram_is_ddr = true;
3718 if ((rdev->family == CHIP_PALM) ||
3719 (rdev->family == CHIP_SUMO) ||
3720 (rdev->family == CHIP_SUMO2))
3721 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3722 else
3723 tmp = RREG32(MC_ARB_RAMCFG);
3724 if (tmp & CHANSIZE_OVERRIDE) {
3725 chansize = 16;
3726 } else if (tmp & CHANSIZE_MASK) {
3727 chansize = 64;
3728 } else {
3729 chansize = 32;
3730 }
3731 tmp = RREG32(MC_SHARED_CHMAP);
3732 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3733 case 0:
3734 default:
3735 numchan = 1;
3736 break;
3737 case 1:
3738 numchan = 2;
3739 break;
3740 case 2:
3741 numchan = 4;
3742 break;
3743 case 3:
3744 numchan = 8;
3745 break;
3746 }
3747 rdev->mc.vram_width = numchan * chansize;
3748
3749 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3750 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3751
3752 if ((rdev->family == CHIP_PALM) ||
3753 (rdev->family == CHIP_SUMO) ||
3754 (rdev->family == CHIP_SUMO2)) {
3755
3756 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3757 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3758 } else {
3759
3760 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3762 }
3763 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3764 r700_vram_gtt_location(rdev, &rdev->mc);
3765 radeon_update_bandwidth_info(rdev);
3766
3767 return 0;
3768}
3769
3770void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3771{
3772 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3773 RREG32(GRBM_STATUS));
3774 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3775 RREG32(GRBM_STATUS_SE0));
3776 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3777 RREG32(GRBM_STATUS_SE1));
3778 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3779 RREG32(SRBM_STATUS));
3780 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3781 RREG32(SRBM_STATUS2));
3782 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3783 RREG32(CP_STALLED_STAT1));
3784 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3785 RREG32(CP_STALLED_STAT2));
3786 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3787 RREG32(CP_BUSY_STAT));
3788 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3789 RREG32(CP_STAT));
3790 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3791 RREG32(DMA_STATUS_REG));
3792 if (rdev->family >= CHIP_CAYMAN) {
3793 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3794 RREG32(DMA_STATUS_REG + 0x800));
3795 }
3796}
3797
3798bool evergreen_is_display_hung(struct radeon_device *rdev)
3799{
3800 u32 crtc_hung = 0;
3801 u32 crtc_status[6];
3802 u32 i, j, tmp;
3803
3804 for (i = 0; i < rdev->num_crtc; i++) {
3805 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3806 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3807 crtc_hung |= (1 << i);
3808 }
3809 }
3810
3811 for (j = 0; j < 10; j++) {
3812 for (i = 0; i < rdev->num_crtc; i++) {
3813 if (crtc_hung & (1 << i)) {
3814 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3815 if (tmp != crtc_status[i])
3816 crtc_hung &= ~(1 << i);
3817 }
3818 }
3819 if (crtc_hung == 0)
3820 return false;
3821 udelay(100);
3822 }
3823
3824 return true;
3825}
3826
3827u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3828{
3829 u32 reset_mask = 0;
3830 u32 tmp;
3831
3832
3833 tmp = RREG32(GRBM_STATUS);
3834 if (tmp & (PA_BUSY | SC_BUSY |
3835 SH_BUSY | SX_BUSY |
3836 TA_BUSY | VGT_BUSY |
3837 DB_BUSY | CB_BUSY |
3838 SPI_BUSY | VGT_BUSY_NO_DMA))
3839 reset_mask |= RADEON_RESET_GFX;
3840
3841 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3842 CP_BUSY | CP_COHERENCY_BUSY))
3843 reset_mask |= RADEON_RESET_CP;
3844
3845 if (tmp & GRBM_EE_BUSY)
3846 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3847
3848
3849 tmp = RREG32(DMA_STATUS_REG);
3850 if (!(tmp & DMA_IDLE))
3851 reset_mask |= RADEON_RESET_DMA;
3852
3853
3854 tmp = RREG32(SRBM_STATUS2);
3855 if (tmp & DMA_BUSY)
3856 reset_mask |= RADEON_RESET_DMA;
3857
3858
3859 tmp = RREG32(SRBM_STATUS);
3860 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3861 reset_mask |= RADEON_RESET_RLC;
3862
3863 if (tmp & IH_BUSY)
3864 reset_mask |= RADEON_RESET_IH;
3865
3866 if (tmp & SEM_BUSY)
3867 reset_mask |= RADEON_RESET_SEM;
3868
3869 if (tmp & GRBM_RQ_PENDING)
3870 reset_mask |= RADEON_RESET_GRBM;
3871
3872 if (tmp & VMC_BUSY)
3873 reset_mask |= RADEON_RESET_VMC;
3874
3875 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3876 MCC_BUSY | MCD_BUSY))
3877 reset_mask |= RADEON_RESET_MC;
3878
3879 if (evergreen_is_display_hung(rdev))
3880 reset_mask |= RADEON_RESET_DISPLAY;
3881
3882
3883 tmp = RREG32(VM_L2_STATUS);
3884 if (tmp & L2_BUSY)
3885 reset_mask |= RADEON_RESET_VMC;
3886
3887
3888 if (reset_mask & RADEON_RESET_MC) {
3889 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3890 reset_mask &= ~RADEON_RESET_MC;
3891 }
3892
3893 return reset_mask;
3894}
3895
3896static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3897{
3898 struct evergreen_mc_save save;
3899 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3900 u32 tmp;
3901
3902 if (reset_mask == 0)
3903 return;
3904
3905 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3906
3907 evergreen_print_gpu_status_regs(rdev);
3908
3909
3910 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3911
3912 if (reset_mask & RADEON_RESET_DMA) {
3913
3914 tmp = RREG32(DMA_RB_CNTL);
3915 tmp &= ~DMA_RB_ENABLE;
3916 WREG32(DMA_RB_CNTL, tmp);
3917 }
3918
3919 udelay(50);
3920
3921 evergreen_mc_stop(rdev, &save);
3922 if (evergreen_mc_wait_for_idle(rdev)) {
3923 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3924 }
3925
3926 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3927 grbm_soft_reset |= SOFT_RESET_DB |
3928 SOFT_RESET_CB |
3929 SOFT_RESET_PA |
3930 SOFT_RESET_SC |
3931 SOFT_RESET_SPI |
3932 SOFT_RESET_SX |
3933 SOFT_RESET_SH |
3934 SOFT_RESET_TC |
3935 SOFT_RESET_TA |
3936 SOFT_RESET_VC |
3937 SOFT_RESET_VGT;
3938 }
3939
3940 if (reset_mask & RADEON_RESET_CP) {
3941 grbm_soft_reset |= SOFT_RESET_CP |
3942 SOFT_RESET_VGT;
3943
3944 srbm_soft_reset |= SOFT_RESET_GRBM;
3945 }
3946
3947 if (reset_mask & RADEON_RESET_DMA)
3948 srbm_soft_reset |= SOFT_RESET_DMA;
3949
3950 if (reset_mask & RADEON_RESET_DISPLAY)
3951 srbm_soft_reset |= SOFT_RESET_DC;
3952
3953 if (reset_mask & RADEON_RESET_RLC)
3954 srbm_soft_reset |= SOFT_RESET_RLC;
3955
3956 if (reset_mask & RADEON_RESET_SEM)
3957 srbm_soft_reset |= SOFT_RESET_SEM;
3958
3959 if (reset_mask & RADEON_RESET_IH)
3960 srbm_soft_reset |= SOFT_RESET_IH;
3961
3962 if (reset_mask & RADEON_RESET_GRBM)
3963 srbm_soft_reset |= SOFT_RESET_GRBM;
3964
3965 if (reset_mask & RADEON_RESET_VMC)
3966 srbm_soft_reset |= SOFT_RESET_VMC;
3967
3968 if (!(rdev->flags & RADEON_IS_IGP)) {
3969 if (reset_mask & RADEON_RESET_MC)
3970 srbm_soft_reset |= SOFT_RESET_MC;
3971 }
3972
3973 if (grbm_soft_reset) {
3974 tmp = RREG32(GRBM_SOFT_RESET);
3975 tmp |= grbm_soft_reset;
3976 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3977 WREG32(GRBM_SOFT_RESET, tmp);
3978 tmp = RREG32(GRBM_SOFT_RESET);
3979
3980 udelay(50);
3981
3982 tmp &= ~grbm_soft_reset;
3983 WREG32(GRBM_SOFT_RESET, tmp);
3984 tmp = RREG32(GRBM_SOFT_RESET);
3985 }
3986
3987 if (srbm_soft_reset) {
3988 tmp = RREG32(SRBM_SOFT_RESET);
3989 tmp |= srbm_soft_reset;
3990 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3991 WREG32(SRBM_SOFT_RESET, tmp);
3992 tmp = RREG32(SRBM_SOFT_RESET);
3993
3994 udelay(50);
3995
3996 tmp &= ~srbm_soft_reset;
3997 WREG32(SRBM_SOFT_RESET, tmp);
3998 tmp = RREG32(SRBM_SOFT_RESET);
3999 }
4000
4001
4002 udelay(50);
4003
4004 evergreen_mc_resume(rdev, &save);
4005 udelay(50);
4006
4007 evergreen_print_gpu_status_regs(rdev);
4008}
4009
4010void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4011{
4012 struct evergreen_mc_save save;
4013 u32 tmp, i;
4014
4015 dev_info(rdev->dev, "GPU pci config reset\n");
4016
4017
4018
4019
4020 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4021 udelay(50);
4022
4023 tmp = RREG32(DMA_RB_CNTL);
4024 tmp &= ~DMA_RB_ENABLE;
4025 WREG32(DMA_RB_CNTL, tmp);
4026
4027
4028
4029 r600_rlc_stop(rdev);
4030
4031 udelay(50);
4032
4033
4034 rv770_set_clk_bypass_mode(rdev);
4035
4036 pci_clear_master(rdev->pdev);
4037
4038 evergreen_mc_stop(rdev, &save);
4039 if (evergreen_mc_wait_for_idle(rdev)) {
4040 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4041 }
4042
4043 radeon_pci_config_reset(rdev);
4044
4045 for (i = 0; i < rdev->usec_timeout; i++) {
4046 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4047 break;
4048 udelay(1);
4049 }
4050}
4051
4052int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4053{
4054 u32 reset_mask;
4055
4056 if (hard) {
4057 evergreen_gpu_pci_config_reset(rdev);
4058 return 0;
4059 }
4060
4061 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4062
4063 if (reset_mask)
4064 r600_set_bios_scratch_engine_hung(rdev, true);
4065
4066
4067 evergreen_gpu_soft_reset(rdev, reset_mask);
4068
4069 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4070
4071
4072 if (reset_mask && radeon_hard_reset)
4073 evergreen_gpu_pci_config_reset(rdev);
4074
4075 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4076
4077 if (!reset_mask)
4078 r600_set_bios_scratch_engine_hung(rdev, false);
4079
4080 return 0;
4081}
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4093{
4094 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4095
4096 if (!(reset_mask & (RADEON_RESET_GFX |
4097 RADEON_RESET_COMPUTE |
4098 RADEON_RESET_CP))) {
4099 radeon_ring_lockup_update(rdev, ring);
4100 return false;
4101 }
4102 return radeon_ring_test_lockup(rdev, ring);
4103}
4104
4105
4106
4107
4108#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4109#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4110
4111void sumo_rlc_fini(struct radeon_device *rdev)
4112{
4113 int r;
4114
4115
4116 if (rdev->rlc.save_restore_obj) {
4117 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4118 if (unlikely(r != 0))
4119 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4120 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4121 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4122
4123 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4124 rdev->rlc.save_restore_obj = NULL;
4125 }
4126
4127
4128 if (rdev->rlc.clear_state_obj) {
4129 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4130 if (unlikely(r != 0))
4131 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4132 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4133 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4134
4135 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4136 rdev->rlc.clear_state_obj = NULL;
4137 }
4138
4139
4140 if (rdev->rlc.cp_table_obj) {
4141 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4142 if (unlikely(r != 0))
4143 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4144 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4145 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4146
4147 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4148 rdev->rlc.cp_table_obj = NULL;
4149 }
4150}
4151
4152#define CP_ME_TABLE_SIZE 96
4153
4154int sumo_rlc_init(struct radeon_device *rdev)
4155{
4156 const u32 *src_ptr;
4157 volatile u32 *dst_ptr;
4158 u32 dws, data, i, j, k, reg_num;
4159 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4160 u64 reg_list_mc_addr;
4161 const struct cs_section_def *cs_data;
4162 int r;
4163
4164 src_ptr = rdev->rlc.reg_list;
4165 dws = rdev->rlc.reg_list_size;
4166 if (rdev->family >= CHIP_BONAIRE) {
4167 dws += (5 * 16) + 48 + 48 + 64;
4168 }
4169 cs_data = rdev->rlc.cs_data;
4170
4171 if (src_ptr) {
4172
4173 if (rdev->rlc.save_restore_obj == NULL) {
4174 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4175 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4176 NULL, &rdev->rlc.save_restore_obj);
4177 if (r) {
4178 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4179 return r;
4180 }
4181 }
4182
4183 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4184 if (unlikely(r != 0)) {
4185 sumo_rlc_fini(rdev);
4186 return r;
4187 }
4188 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4189 &rdev->rlc.save_restore_gpu_addr);
4190 if (r) {
4191 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4192 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4193 sumo_rlc_fini(rdev);
4194 return r;
4195 }
4196
4197 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4198 if (r) {
4199 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4200 sumo_rlc_fini(rdev);
4201 return r;
4202 }
4203
4204 dst_ptr = rdev->rlc.sr_ptr;
4205 if (rdev->family >= CHIP_TAHITI) {
4206
4207 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4208 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4209 } else {
4210
4211
4212
4213
4214
4215
4216 for (i = 0; i < dws; i++) {
4217 data = src_ptr[i] >> 2;
4218 i++;
4219 if (i < dws)
4220 data |= (src_ptr[i] >> 2) << 16;
4221 j = (((i - 1) * 3) / 2);
4222 dst_ptr[j] = cpu_to_le32(data);
4223 }
4224 j = ((i * 3) / 2);
4225 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4226 }
4227 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4228 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4229 }
4230
4231 if (cs_data) {
4232
4233 if (rdev->family >= CHIP_BONAIRE) {
4234 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4235 } else if (rdev->family >= CHIP_TAHITI) {
4236 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4237 dws = rdev->rlc.clear_state_size + (256 / 4);
4238 } else {
4239 reg_list_num = 0;
4240 dws = 0;
4241 for (i = 0; cs_data[i].section != NULL; i++) {
4242 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4243 reg_list_num++;
4244 dws += cs_data[i].section[j].reg_count;
4245 }
4246 }
4247 reg_list_blk_index = (3 * reg_list_num + 2);
4248 dws += reg_list_blk_index;
4249 rdev->rlc.clear_state_size = dws;
4250 }
4251
4252 if (rdev->rlc.clear_state_obj == NULL) {
4253 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4254 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4255 NULL, &rdev->rlc.clear_state_obj);
4256 if (r) {
4257 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4258 sumo_rlc_fini(rdev);
4259 return r;
4260 }
4261 }
4262 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4263 if (unlikely(r != 0)) {
4264 sumo_rlc_fini(rdev);
4265 return r;
4266 }
4267 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4268 &rdev->rlc.clear_state_gpu_addr);
4269 if (r) {
4270 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4271 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4272 sumo_rlc_fini(rdev);
4273 return r;
4274 }
4275
4276 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4277 if (r) {
4278 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4279 sumo_rlc_fini(rdev);
4280 return r;
4281 }
4282
4283 dst_ptr = rdev->rlc.cs_ptr;
4284 if (rdev->family >= CHIP_BONAIRE) {
4285 cik_get_csb_buffer(rdev, dst_ptr);
4286 } else if (rdev->family >= CHIP_TAHITI) {
4287 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4288 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4289 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4290 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4291 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4292 } else {
4293 reg_list_hdr_blk_index = 0;
4294 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4295 data = upper_32_bits(reg_list_mc_addr);
4296 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4297 reg_list_hdr_blk_index++;
4298 for (i = 0; cs_data[i].section != NULL; i++) {
4299 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4300 reg_num = cs_data[i].section[j].reg_count;
4301 data = reg_list_mc_addr & 0xffffffff;
4302 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4303 reg_list_hdr_blk_index++;
4304
4305 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4306 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4307 reg_list_hdr_blk_index++;
4308
4309 data = 0x08000000 | (reg_num * 4);
4310 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4311 reg_list_hdr_blk_index++;
4312
4313 for (k = 0; k < reg_num; k++) {
4314 data = cs_data[i].section[j].extent[k];
4315 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4316 }
4317 reg_list_mc_addr += reg_num * 4;
4318 reg_list_blk_index += reg_num;
4319 }
4320 }
4321 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4322 }
4323 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4324 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4325 }
4326
4327 if (rdev->rlc.cp_table_size) {
4328 if (rdev->rlc.cp_table_obj == NULL) {
4329 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4330 PAGE_SIZE, true,
4331 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4332 NULL, &rdev->rlc.cp_table_obj);
4333 if (r) {
4334 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4335 sumo_rlc_fini(rdev);
4336 return r;
4337 }
4338 }
4339
4340 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4341 if (unlikely(r != 0)) {
4342 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4343 sumo_rlc_fini(rdev);
4344 return r;
4345 }
4346 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4347 &rdev->rlc.cp_table_gpu_addr);
4348 if (r) {
4349 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4350 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4351 sumo_rlc_fini(rdev);
4352 return r;
4353 }
4354 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4355 if (r) {
4356 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4357 sumo_rlc_fini(rdev);
4358 return r;
4359 }
4360
4361 cik_init_cp_pg_table(rdev);
4362
4363 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4364 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4365
4366 }
4367
4368 return 0;
4369}
4370
4371static void evergreen_rlc_start(struct radeon_device *rdev)
4372{
4373 u32 mask = RLC_ENABLE;
4374
4375 if (rdev->flags & RADEON_IS_IGP) {
4376 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4377 }
4378
4379 WREG32(RLC_CNTL, mask);
4380}
4381
4382int evergreen_rlc_resume(struct radeon_device *rdev)
4383{
4384 u32 i;
4385 const __be32 *fw_data;
4386
4387 if (!rdev->rlc_fw)
4388 return -EINVAL;
4389
4390 r600_rlc_stop(rdev);
4391
4392 WREG32(RLC_HB_CNTL, 0);
4393
4394 if (rdev->flags & RADEON_IS_IGP) {
4395 if (rdev->family == CHIP_ARUBA) {
4396 u32 always_on_bitmap =
4397 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4398
4399 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4400 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4401 tmp = hweight32(~tmp);
4402 if (tmp == rdev->config.cayman.max_simds_per_se) {
4403 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4404 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4405 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4406 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4407 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4408 }
4409 } else {
4410 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4411 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4412 }
4413 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4414 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4415 } else {
4416 WREG32(RLC_HB_BASE, 0);
4417 WREG32(RLC_HB_RPTR, 0);
4418 WREG32(RLC_HB_WPTR, 0);
4419 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4420 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4421 }
4422 WREG32(RLC_MC_CNTL, 0);
4423 WREG32(RLC_UCODE_CNTL, 0);
4424
4425 fw_data = (const __be32 *)rdev->rlc_fw->data;
4426 if (rdev->family >= CHIP_ARUBA) {
4427 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4428 WREG32(RLC_UCODE_ADDR, i);
4429 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4430 }
4431 } else if (rdev->family >= CHIP_CAYMAN) {
4432 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4433 WREG32(RLC_UCODE_ADDR, i);
4434 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4435 }
4436 } else {
4437 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4438 WREG32(RLC_UCODE_ADDR, i);
4439 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4440 }
4441 }
4442 WREG32(RLC_UCODE_ADDR, 0);
4443
4444 evergreen_rlc_start(rdev);
4445
4446 return 0;
4447}
4448
4449
4450
4451u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4452{
4453 if (crtc >= rdev->num_crtc)
4454 return 0;
4455 else
4456 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4457}
4458
4459void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4460{
4461 int i;
4462 u32 tmp;
4463
4464 if (rdev->family >= CHIP_CAYMAN) {
4465 cayman_cp_int_cntl_setup(rdev, 0,
4466 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4467 cayman_cp_int_cntl_setup(rdev, 1, 0);
4468 cayman_cp_int_cntl_setup(rdev, 2, 0);
4469 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4470 WREG32(CAYMAN_DMA1_CNTL, tmp);
4471 } else
4472 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4473 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4474 WREG32(DMA_CNTL, tmp);
4475 WREG32(GRBM_INT_CNTL, 0);
4476 WREG32(SRBM_INT_CNTL, 0);
4477 for (i = 0; i < rdev->num_crtc; i++)
4478 WREG32(INT_MASK + crtc_offsets[i], 0);
4479 for (i = 0; i < rdev->num_crtc; i++)
4480 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4481
4482
4483 if (!ASIC_IS_DCE5(rdev))
4484 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4485 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4486
4487 for (i = 0; i < 6; i++)
4488 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4489}
4490
4491
4492int evergreen_irq_set(struct radeon_device *rdev)
4493{
4494 int i;
4495 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4496 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4497 u32 grbm_int_cntl = 0;
4498 u32 dma_cntl, dma_cntl1 = 0;
4499 u32 thermal_int = 0;
4500
4501 if (!rdev->irq.installed) {
4502 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4503 return -EINVAL;
4504 }
4505
4506 if (!rdev->ih.enabled) {
4507 r600_disable_interrupts(rdev);
4508
4509 evergreen_disable_interrupt_state(rdev);
4510 return 0;
4511 }
4512
4513 if (rdev->family == CHIP_ARUBA)
4514 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4515 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4516 else
4517 thermal_int = RREG32(CG_THERMAL_INT) &
4518 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4519
4520 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4521
4522 if (rdev->family >= CHIP_CAYMAN) {
4523
4524 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4525 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4526 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4527 }
4528 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4529 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4530 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4531 }
4532 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4533 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4534 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4535 }
4536 } else {
4537 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4538 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4539 cp_int_cntl |= RB_INT_ENABLE;
4540 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4541 }
4542 }
4543
4544 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4545 DRM_DEBUG("r600_irq_set: sw int dma\n");
4546 dma_cntl |= TRAP_ENABLE;
4547 }
4548
4549 if (rdev->family >= CHIP_CAYMAN) {
4550 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4551 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4552 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4553 dma_cntl1 |= TRAP_ENABLE;
4554 }
4555 }
4556
4557 if (rdev->irq.dpm_thermal) {
4558 DRM_DEBUG("dpm thermal\n");
4559 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4560 }
4561
4562 if (rdev->family >= CHIP_CAYMAN) {
4563 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4564 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4565 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4566 } else
4567 WREG32(CP_INT_CNTL, cp_int_cntl);
4568
4569 WREG32(DMA_CNTL, dma_cntl);
4570
4571 if (rdev->family >= CHIP_CAYMAN)
4572 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4573
4574 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4575
4576 for (i = 0; i < rdev->num_crtc; i++) {
4577 radeon_irq_kms_set_irq_n_enabled(
4578 rdev, INT_MASK + crtc_offsets[i],
4579 VBLANK_INT_MASK,
4580 rdev->irq.crtc_vblank_int[i] ||
4581 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4582 }
4583
4584 for (i = 0; i < rdev->num_crtc; i++)
4585 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4586
4587 for (i = 0; i < 6; i++) {
4588 radeon_irq_kms_set_irq_n_enabled(
4589 rdev, DC_HPDx_INT_CONTROL(i),
4590 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4591 rdev->irq.hpd[i], "HPD", i);
4592 }
4593
4594 if (rdev->family == CHIP_ARUBA)
4595 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4596 else
4597 WREG32(CG_THERMAL_INT, thermal_int);
4598
4599 for (i = 0; i < 6; i++) {
4600 radeon_irq_kms_set_irq_n_enabled(
4601 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4602 AFMT_AZ_FORMAT_WTRIG_MASK,
4603 rdev->irq.afmt[i], "HDMI", i);
4604 }
4605
4606
4607 RREG32(SRBM_STATUS);
4608
4609 return 0;
4610}
4611
4612
4613static void evergreen_irq_ack(struct radeon_device *rdev)
4614{
4615 int i, j;
4616 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4617 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4618 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4619
4620 for (i = 0; i < 6; i++) {
4621 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4622 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4623 if (i < rdev->num_crtc)
4624 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4625 }
4626
4627
4628 for (i = 0; i < rdev->num_crtc; i += 2) {
4629 for (j = i; j < (i + 2); j++) {
4630 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4631 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4632 GRPH_PFLIP_INT_CLEAR);
4633 }
4634
4635 for (j = i; j < (i + 2); j++) {
4636 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4637 WREG32(VBLANK_STATUS + crtc_offsets[j],
4638 VBLANK_ACK);
4639 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4640 WREG32(VLINE_STATUS + crtc_offsets[j],
4641 VLINE_ACK);
4642 }
4643 }
4644
4645 for (i = 0; i < 6; i++) {
4646 if (disp_int[i] & DC_HPD1_INTERRUPT)
4647 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4648 }
4649
4650 for (i = 0; i < 6; i++) {
4651 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4652 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4653 }
4654
4655 for (i = 0; i < 6; i++) {
4656 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4657 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4658 AFMT_AZ_FORMAT_WTRIG_ACK);
4659 }
4660}
4661
4662static void evergreen_irq_disable(struct radeon_device *rdev)
4663{
4664 r600_disable_interrupts(rdev);
4665
4666 mdelay(1);
4667 evergreen_irq_ack(rdev);
4668 evergreen_disable_interrupt_state(rdev);
4669}
4670
4671void evergreen_irq_suspend(struct radeon_device *rdev)
4672{
4673 evergreen_irq_disable(rdev);
4674 r600_rlc_stop(rdev);
4675}
4676
4677static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4678{
4679 u32 wptr, tmp;
4680
4681 if (rdev->wb.enabled)
4682 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4683 else
4684 wptr = RREG32(IH_RB_WPTR);
4685
4686 if (wptr & RB_OVERFLOW) {
4687 wptr &= ~RB_OVERFLOW;
4688
4689
4690
4691
4692 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4693 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4694 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4695 tmp = RREG32(IH_RB_CNTL);
4696 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4697 WREG32(IH_RB_CNTL, tmp);
4698 }
4699 return (wptr & rdev->ih.ptr_mask);
4700}
4701
4702int evergreen_irq_process(struct radeon_device *rdev)
4703{
4704 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4705 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4706 u32 crtc_idx, hpd_idx, afmt_idx;
4707 u32 mask;
4708 u32 wptr;
4709 u32 rptr;
4710 u32 src_id, src_data;
4711 u32 ring_index;
4712 bool queue_hotplug = false;
4713 bool queue_hdmi = false;
4714 bool queue_dp = false;
4715 bool queue_thermal = false;
4716 u32 status, addr;
4717 const char *event_name;
4718
4719 if (!rdev->ih.enabled || rdev->shutdown)
4720 return IRQ_NONE;
4721
4722 wptr = evergreen_get_ih_wptr(rdev);
4723
4724restart_ih:
4725
4726 if (atomic_xchg(&rdev->ih.lock, 1))
4727 return IRQ_NONE;
4728
4729 rptr = rdev->ih.rptr;
4730 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4731
4732
4733 rmb();
4734
4735
4736 evergreen_irq_ack(rdev);
4737
4738 while (rptr != wptr) {
4739
4740 ring_index = rptr / 4;
4741 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4742 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4743
4744 switch (src_id) {
4745 case 1:
4746 case 2:
4747 case 3:
4748 case 4:
4749 case 5:
4750 case 6:
4751 crtc_idx = src_id - 1;
4752
4753 if (src_data == 0) {
4754 mask = LB_D1_VBLANK_INTERRUPT;
4755 event_name = "vblank";
4756
4757 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4758 drm_handle_vblank(rdev->ddev, crtc_idx);
4759 rdev->pm.vblank_sync = true;
4760 wake_up(&rdev->irq.vblank_queue);
4761 }
4762 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4763 radeon_crtc_handle_vblank(rdev,
4764 crtc_idx);
4765 }
4766
4767 } else if (src_data == 1) {
4768 mask = LB_D1_VLINE_INTERRUPT;
4769 event_name = "vline";
4770 } else {
4771 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4772 src_id, src_data);
4773 break;
4774 }
4775
4776 if (!(disp_int[crtc_idx] & mask)) {
4777 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4778 crtc_idx + 1, event_name);
4779 }
4780
4781 disp_int[crtc_idx] &= ~mask;
4782 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4783
4784 break;
4785 case 8:
4786 case 10:
4787 case 12:
4788 case 14:
4789 case 16:
4790 case 18:
4791 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4792 if (radeon_use_pflipirq > 0)
4793 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4794 break;
4795 case 42:
4796 if (src_data <= 5) {
4797 hpd_idx = src_data;
4798 mask = DC_HPD1_INTERRUPT;
4799 queue_hotplug = true;
4800 event_name = "HPD";
4801
4802 } else if (src_data <= 11) {
4803 hpd_idx = src_data - 6;
4804 mask = DC_HPD1_RX_INTERRUPT;
4805 queue_dp = true;
4806 event_name = "HPD_RX";
4807
4808 } else {
4809 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4810 src_id, src_data);
4811 break;
4812 }
4813
4814 if (!(disp_int[hpd_idx] & mask))
4815 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4816
4817 disp_int[hpd_idx] &= ~mask;
4818 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4819
4820 break;
4821 case 44:
4822 afmt_idx = src_data;
4823 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4824 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4825
4826 if (afmt_idx > 5) {
4827 DRM_ERROR("Unhandled interrupt: %d %d\n",
4828 src_id, src_data);
4829 break;
4830 }
4831 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4832 queue_hdmi = true;
4833 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4834 break;
4835 case 96:
4836 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4837 WREG32(SRBM_INT_ACK, 0x1);
4838 break;
4839 case 124:
4840 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4841 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4842 break;
4843 case 146:
4844 case 147:
4845 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4846 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4847
4848 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4849 if (addr == 0x0 && status == 0x0)
4850 break;
4851 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4852 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4853 addr);
4854 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4855 status);
4856 cayman_vm_decode_fault(rdev, status, addr);
4857 break;
4858 case 176:
4859 case 177:
4860 case 178:
4861 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4862 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4863 break;
4864 case 181:
4865 DRM_DEBUG("IH: CP EOP\n");
4866 if (rdev->family >= CHIP_CAYMAN) {
4867 switch (src_data) {
4868 case 0:
4869 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4870 break;
4871 case 1:
4872 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4873 break;
4874 case 2:
4875 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4876 break;
4877 }
4878 } else
4879 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4880 break;
4881 case 224:
4882 DRM_DEBUG("IH: DMA trap\n");
4883 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4884 break;
4885 case 230:
4886 DRM_DEBUG("IH: thermal low to high\n");
4887 rdev->pm.dpm.thermal.high_to_low = false;
4888 queue_thermal = true;
4889 break;
4890 case 231:
4891 DRM_DEBUG("IH: thermal high to low\n");
4892 rdev->pm.dpm.thermal.high_to_low = true;
4893 queue_thermal = true;
4894 break;
4895 case 233:
4896 DRM_DEBUG("IH: GUI idle\n");
4897 break;
4898 case 244:
4899 if (rdev->family >= CHIP_CAYMAN) {
4900 DRM_DEBUG("IH: DMA1 trap\n");
4901 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4902 }
4903 break;
4904 default:
4905 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4906 break;
4907 }
4908
4909
4910 rptr += 16;
4911 rptr &= rdev->ih.ptr_mask;
4912 WREG32(IH_RB_RPTR, rptr);
4913 }
4914 if (queue_dp)
4915 schedule_work(&rdev->dp_work);
4916 if (queue_hotplug)
4917 schedule_delayed_work(&rdev->hotplug_work, 0);
4918 if (queue_hdmi)
4919 schedule_work(&rdev->audio_work);
4920 if (queue_thermal && rdev->pm.dpm_enabled)
4921 schedule_work(&rdev->pm.dpm.thermal.work);
4922 rdev->ih.rptr = rptr;
4923 atomic_set(&rdev->ih.lock, 0);
4924
4925
4926 wptr = evergreen_get_ih_wptr(rdev);
4927 if (wptr != rptr)
4928 goto restart_ih;
4929
4930 return IRQ_HANDLED;
4931}
4932
4933static void evergreen_uvd_init(struct radeon_device *rdev)
4934{
4935 int r;
4936
4937 if (!rdev->has_uvd)
4938 return;
4939
4940 r = radeon_uvd_init(rdev);
4941 if (r) {
4942 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4943
4944
4945
4946
4947
4948
4949 rdev->has_uvd = false;
4950 return;
4951 }
4952 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4953 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4954}
4955
4956static void evergreen_uvd_start(struct radeon_device *rdev)
4957{
4958 int r;
4959
4960 if (!rdev->has_uvd)
4961 return;
4962
4963 r = uvd_v2_2_resume(rdev);
4964 if (r) {
4965 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4966 goto error;
4967 }
4968 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4969 if (r) {
4970 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4971 goto error;
4972 }
4973 return;
4974
4975error:
4976 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4977}
4978
4979static void evergreen_uvd_resume(struct radeon_device *rdev)
4980{
4981 struct radeon_ring *ring;
4982 int r;
4983
4984 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4985 return;
4986
4987 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4988 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4989 if (r) {
4990 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4991 return;
4992 }
4993 r = uvd_v1_0_init(rdev);
4994 if (r) {
4995 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4996 return;
4997 }
4998}
4999
5000static int evergreen_startup(struct radeon_device *rdev)
5001{
5002 struct radeon_ring *ring;
5003 int r;
5004
5005
5006 evergreen_pcie_gen2_enable(rdev);
5007
5008 evergreen_program_aspm(rdev);
5009
5010
5011 r = r600_vram_scratch_init(rdev);
5012 if (r)
5013 return r;
5014
5015 evergreen_mc_program(rdev);
5016
5017 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5018 r = ni_mc_load_microcode(rdev);
5019 if (r) {
5020 DRM_ERROR("Failed to load MC firmware!\n");
5021 return r;
5022 }
5023 }
5024
5025 if (rdev->flags & RADEON_IS_AGP) {
5026 evergreen_agp_enable(rdev);
5027 } else {
5028 r = evergreen_pcie_gart_enable(rdev);
5029 if (r)
5030 return r;
5031 }
5032 evergreen_gpu_init(rdev);
5033
5034
5035 if (rdev->flags & RADEON_IS_IGP) {
5036 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5037 rdev->rlc.reg_list_size =
5038 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5039 rdev->rlc.cs_data = evergreen_cs_data;
5040 r = sumo_rlc_init(rdev);
5041 if (r) {
5042 DRM_ERROR("Failed to init rlc BOs!\n");
5043 return r;
5044 }
5045 }
5046
5047
5048 r = radeon_wb_init(rdev);
5049 if (r)
5050 return r;
5051
5052 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5053 if (r) {
5054 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5055 return r;
5056 }
5057
5058 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5059 if (r) {
5060 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5061 return r;
5062 }
5063
5064 evergreen_uvd_start(rdev);
5065
5066
5067 if (!rdev->irq.installed) {
5068 r = radeon_irq_kms_init(rdev);
5069 if (r)
5070 return r;
5071 }
5072
5073 r = r600_irq_init(rdev);
5074 if (r) {
5075 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5076 radeon_irq_kms_fini(rdev);
5077 return r;
5078 }
5079 evergreen_irq_set(rdev);
5080
5081 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5082 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5083 RADEON_CP_PACKET2);
5084 if (r)
5085 return r;
5086
5087 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5088 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5089 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5090 if (r)
5091 return r;
5092
5093 r = evergreen_cp_load_microcode(rdev);
5094 if (r)
5095 return r;
5096 r = evergreen_cp_resume(rdev);
5097 if (r)
5098 return r;
5099 r = r600_dma_resume(rdev);
5100 if (r)
5101 return r;
5102
5103 evergreen_uvd_resume(rdev);
5104
5105 r = radeon_ib_pool_init(rdev);
5106 if (r) {
5107 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5108 return r;
5109 }
5110
5111 r = radeon_audio_init(rdev);
5112 if (r) {
5113 DRM_ERROR("radeon: audio init failed\n");
5114 return r;
5115 }
5116
5117 return 0;
5118}
5119
5120int evergreen_resume(struct radeon_device *rdev)
5121{
5122 int r;
5123
5124
5125
5126
5127 if (radeon_asic_reset(rdev))
5128 dev_warn(rdev->dev, "GPU reset failed !\n");
5129
5130
5131
5132
5133
5134 atom_asic_init(rdev->mode_info.atom_context);
5135
5136
5137 evergreen_init_golden_registers(rdev);
5138
5139 if (rdev->pm.pm_method == PM_METHOD_DPM)
5140 radeon_pm_resume(rdev);
5141
5142 rdev->accel_working = true;
5143 r = evergreen_startup(rdev);
5144 if (r) {
5145 DRM_ERROR("evergreen startup failed on resume\n");
5146 rdev->accel_working = false;
5147 return r;
5148 }
5149
5150 return r;
5151
5152}
5153
5154int evergreen_suspend(struct radeon_device *rdev)
5155{
5156 radeon_pm_suspend(rdev);
5157 radeon_audio_fini(rdev);
5158 if (rdev->has_uvd) {
5159 uvd_v1_0_fini(rdev);
5160 radeon_uvd_suspend(rdev);
5161 }
5162 r700_cp_stop(rdev);
5163 r600_dma_stop(rdev);
5164 evergreen_irq_suspend(rdev);
5165 radeon_wb_disable(rdev);
5166 evergreen_pcie_gart_disable(rdev);
5167
5168 return 0;
5169}
5170
5171
5172
5173
5174
5175
5176
5177int evergreen_init(struct radeon_device *rdev)
5178{
5179 int r;
5180
5181
5182 if (!radeon_get_bios(rdev)) {
5183 if (ASIC_IS_AVIVO(rdev))
5184 return -EINVAL;
5185 }
5186
5187 if (!rdev->is_atom_bios) {
5188 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5189 return -EINVAL;
5190 }
5191 r = radeon_atombios_init(rdev);
5192 if (r)
5193 return r;
5194
5195
5196
5197 if (radeon_asic_reset(rdev))
5198 dev_warn(rdev->dev, "GPU reset failed !\n");
5199
5200 if (!radeon_card_posted(rdev)) {
5201 if (!rdev->bios) {
5202 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5203 return -EINVAL;
5204 }
5205 DRM_INFO("GPU not posted. posting now...\n");
5206 atom_asic_init(rdev->mode_info.atom_context);
5207 }
5208
5209 evergreen_init_golden_registers(rdev);
5210
5211 r600_scratch_init(rdev);
5212
5213 radeon_surface_init(rdev);
5214
5215 radeon_get_clock_info(rdev->ddev);
5216
5217 radeon_fence_driver_init(rdev);
5218
5219 if (rdev->flags & RADEON_IS_AGP) {
5220 r = radeon_agp_init(rdev);
5221 if (r)
5222 radeon_agp_disable(rdev);
5223 }
5224
5225 r = evergreen_mc_init(rdev);
5226 if (r)
5227 return r;
5228
5229 r = radeon_bo_init(rdev);
5230 if (r)
5231 return r;
5232
5233 if (ASIC_IS_DCE5(rdev)) {
5234 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5235 r = ni_init_microcode(rdev);
5236 if (r) {
5237 DRM_ERROR("Failed to load firmware!\n");
5238 return r;
5239 }
5240 }
5241 } else {
5242 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5243 r = r600_init_microcode(rdev);
5244 if (r) {
5245 DRM_ERROR("Failed to load firmware!\n");
5246 return r;
5247 }
5248 }
5249 }
5250
5251
5252 radeon_pm_init(rdev);
5253
5254 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5255 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5256
5257 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5258 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5259
5260 evergreen_uvd_init(rdev);
5261
5262 rdev->ih.ring_obj = NULL;
5263 r600_ih_ring_init(rdev, 64 * 1024);
5264
5265 r = r600_pcie_gart_init(rdev);
5266 if (r)
5267 return r;
5268
5269 rdev->accel_working = true;
5270 r = evergreen_startup(rdev);
5271 if (r) {
5272 dev_err(rdev->dev, "disabling GPU acceleration\n");
5273 r700_cp_fini(rdev);
5274 r600_dma_fini(rdev);
5275 r600_irq_fini(rdev);
5276 if (rdev->flags & RADEON_IS_IGP)
5277 sumo_rlc_fini(rdev);
5278 radeon_wb_fini(rdev);
5279 radeon_ib_pool_fini(rdev);
5280 radeon_irq_kms_fini(rdev);
5281 evergreen_pcie_gart_fini(rdev);
5282 rdev->accel_working = false;
5283 }
5284
5285
5286
5287
5288
5289 if (ASIC_IS_DCE5(rdev)) {
5290 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5291 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5292 return -EINVAL;
5293 }
5294 }
5295
5296 return 0;
5297}
5298
5299void evergreen_fini(struct radeon_device *rdev)
5300{
5301 radeon_pm_fini(rdev);
5302 radeon_audio_fini(rdev);
5303 r700_cp_fini(rdev);
5304 r600_dma_fini(rdev);
5305 r600_irq_fini(rdev);
5306 if (rdev->flags & RADEON_IS_IGP)
5307 sumo_rlc_fini(rdev);
5308 radeon_wb_fini(rdev);
5309 radeon_ib_pool_fini(rdev);
5310 radeon_irq_kms_fini(rdev);
5311 uvd_v1_0_fini(rdev);
5312 radeon_uvd_fini(rdev);
5313 evergreen_pcie_gart_fini(rdev);
5314 r600_vram_scratch_fini(rdev);
5315 radeon_gem_fini(rdev);
5316 radeon_fence_driver_fini(rdev);
5317 radeon_agp_fini(rdev);
5318 radeon_bo_fini(rdev);
5319 radeon_atombios_fini(rdev);
5320 kfree(rdev->bios);
5321 rdev->bios = NULL;
5322}
5323
5324void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5325{
5326 u32 link_width_cntl, speed_cntl;
5327
5328 if (radeon_pcie_gen2 == 0)
5329 return;
5330
5331 if (rdev->flags & RADEON_IS_IGP)
5332 return;
5333
5334 if (!(rdev->flags & RADEON_IS_PCIE))
5335 return;
5336
5337
5338 if (ASIC_IS_X2(rdev))
5339 return;
5340
5341 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5342 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5343 return;
5344
5345 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5346 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5347 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5348 return;
5349 }
5350
5351 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5352
5353 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5354 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5355
5356 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5357 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5358 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5359
5360 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5361 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5362 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5363
5364 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5365 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5366 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5367
5368 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5369 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5370 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5371
5372 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5373 speed_cntl |= LC_GEN2_EN_STRAP;
5374 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5375
5376 } else {
5377 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5378
5379 if (1)
5380 link_width_cntl |= LC_UPCONFIGURE_DIS;
5381 else
5382 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5383 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5384 }
5385}
5386
5387void evergreen_program_aspm(struct radeon_device *rdev)
5388{
5389 u32 data, orig;
5390 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5391 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5392
5393
5394
5395
5396
5397 bool fusion_platform = false;
5398
5399 if (radeon_aspm == 0)
5400 return;
5401
5402 if (!(rdev->flags & RADEON_IS_PCIE))
5403 return;
5404
5405 switch (rdev->family) {
5406 case CHIP_CYPRESS:
5407 case CHIP_HEMLOCK:
5408 case CHIP_JUNIPER:
5409 case CHIP_REDWOOD:
5410 case CHIP_CEDAR:
5411 case CHIP_SUMO:
5412 case CHIP_SUMO2:
5413 case CHIP_PALM:
5414 case CHIP_ARUBA:
5415 disable_l0s = true;
5416 break;
5417 default:
5418 disable_l0s = false;
5419 break;
5420 }
5421
5422 if (rdev->flags & RADEON_IS_IGP)
5423 fusion_platform = true;
5424
5425 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5426 if (fusion_platform)
5427 data &= ~MULTI_PIF;
5428 else
5429 data |= MULTI_PIF;
5430 if (data != orig)
5431 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5432
5433 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5434 if (fusion_platform)
5435 data &= ~MULTI_PIF;
5436 else
5437 data |= MULTI_PIF;
5438 if (data != orig)
5439 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5440
5441 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5442 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5443 if (!disable_l0s) {
5444 if (rdev->family >= CHIP_BARTS)
5445 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5446 else
5447 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5448 }
5449
5450 if (!disable_l1) {
5451 if (rdev->family >= CHIP_BARTS)
5452 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5453 else
5454 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5455
5456 if (!disable_plloff_in_l1) {
5457 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5458 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5459 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5460 if (data != orig)
5461 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5462
5463 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5464 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5465 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5466 if (data != orig)
5467 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5468
5469 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5470 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5471 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5472 if (data != orig)
5473 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5474
5475 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5476 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5477 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5478 if (data != orig)
5479 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5480
5481 if (rdev->family >= CHIP_BARTS) {
5482 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5483 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5484 data |= PLL_RAMP_UP_TIME_0(4);
5485 if (data != orig)
5486 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5487
5488 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5489 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5490 data |= PLL_RAMP_UP_TIME_1(4);
5491 if (data != orig)
5492 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5493
5494 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5495 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5496 data |= PLL_RAMP_UP_TIME_0(4);
5497 if (data != orig)
5498 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5499
5500 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5501 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5502 data |= PLL_RAMP_UP_TIME_1(4);
5503 if (data != orig)
5504 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5505 }
5506
5507 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5508 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5509 data |= LC_DYN_LANES_PWR_STATE(3);
5510 if (data != orig)
5511 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5512
5513 if (rdev->family >= CHIP_BARTS) {
5514 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5515 data &= ~LS2_EXIT_TIME_MASK;
5516 data |= LS2_EXIT_TIME(1);
5517 if (data != orig)
5518 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5519
5520 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5521 data &= ~LS2_EXIT_TIME_MASK;
5522 data |= LS2_EXIT_TIME(1);
5523 if (data != orig)
5524 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5525 }
5526 }
5527 }
5528
5529
5530 if (rdev->family < CHIP_BARTS)
5531 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5532
5533 if (pcie_lc_cntl != pcie_lc_cntl_old)
5534 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5535}
5536
