1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/firmware.h>
25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "amdgpu.h"
29#include "amdgpu_atombios.h"
30#include "amdgpu_ih.h"
31#include "amdgpu_uvd.h"
32#include "amdgpu_vce.h"
33#include "cikd.h"
34#include "atom.h"
35
36#include "cik.h"
37#include "gmc_v7_0.h"
38#include "cik_ih.h"
39#include "dce_v8_0.h"
40#include "gfx_v7_0.h"
41#include "cik_sdma.h"
42#include "uvd_v4_2.h"
43#include "vce_v2_0.h"
44#include "cik_dpm.h"
45
46#include "uvd/uvd_4_2_d.h"
47
48#include "smu/smu_7_0_1_d.h"
49#include "smu/smu_7_0_1_sh_mask.h"
50
51#include "dce/dce_8_0_d.h"
52#include "dce/dce_8_0_sh_mask.h"
53
54#include "bif/bif_4_1_d.h"
55#include "bif/bif_4_1_sh_mask.h"
56
57#include "gca/gfx_7_2_d.h"
58#include "gca/gfx_7_2_enum.h"
59#include "gca/gfx_7_2_sh_mask.h"
60
61#include "gmc/gmc_7_1_d.h"
62#include "gmc/gmc_7_1_sh_mask.h"
63
64#include "oss/oss_2_0_d.h"
65#include "oss/oss_2_0_sh_mask.h"
66
67#include "amdgpu_amdkfd.h"
68
69
70
71
72static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
73{
74 unsigned long flags;
75 u32 r;
76
77 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
78 WREG32(mmPCIE_INDEX, reg);
79 (void)RREG32(mmPCIE_INDEX);
80 r = RREG32(mmPCIE_DATA);
81 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
82 return r;
83}
84
85static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
86{
87 unsigned long flags;
88
89 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
90 WREG32(mmPCIE_INDEX, reg);
91 (void)RREG32(mmPCIE_INDEX);
92 WREG32(mmPCIE_DATA, v);
93 (void)RREG32(mmPCIE_DATA);
94 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
95}
96
97static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
98{
99 unsigned long flags;
100 u32 r;
101
102 spin_lock_irqsave(&adev->smc_idx_lock, flags);
103 WREG32(mmSMC_IND_INDEX_0, (reg));
104 r = RREG32(mmSMC_IND_DATA_0);
105 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
106 return r;
107}
108
109static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
110{
111 unsigned long flags;
112
113 spin_lock_irqsave(&adev->smc_idx_lock, flags);
114 WREG32(mmSMC_IND_INDEX_0, (reg));
115 WREG32(mmSMC_IND_DATA_0, (v));
116 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
117}
118
119static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
120{
121 unsigned long flags;
122 u32 r;
123
124 spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
125 WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
126 r = RREG32(mmUVD_CTX_DATA);
127 spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
128 return r;
129}
130
131static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
132{
133 unsigned long flags;
134
135 spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
136 WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
137 WREG32(mmUVD_CTX_DATA, (v));
138 spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
139}
140
141static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
142{
143 unsigned long flags;
144 u32 r;
145
146 spin_lock_irqsave(&adev->didt_idx_lock, flags);
147 WREG32(mmDIDT_IND_INDEX, (reg));
148 r = RREG32(mmDIDT_IND_DATA);
149 spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
150 return r;
151}
152
153static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
154{
155 unsigned long flags;
156
157 spin_lock_irqsave(&adev->didt_idx_lock, flags);
158 WREG32(mmDIDT_IND_INDEX, (reg));
159 WREG32(mmDIDT_IND_DATA, (v));
160 spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
161}
162
163static const u32 bonaire_golden_spm_registers[] =
164{
165 0xc200, 0xe0ffffff, 0xe0000000
166};
167
168static const u32 bonaire_golden_common_registers[] =
169{
170 0x31dc, 0xffffffff, 0x00000800,
171 0x31dd, 0xffffffff, 0x00000800,
172 0x31e6, 0xffffffff, 0x00007fbf,
173 0x31e7, 0xffffffff, 0x00007faf
174};
175
176static const u32 bonaire_golden_registers[] =
177{
178 0xcd5, 0x00000333, 0x00000333,
179 0xcd4, 0x000c0fc0, 0x00040200,
180 0x2684, 0x00010000, 0x00058208,
181 0xf000, 0xffff1fff, 0x00140000,
182 0xf080, 0xfdfc0fff, 0x00000100,
183 0xf08d, 0x40000000, 0x40000200,
184 0x260c, 0xffffffff, 0x00000000,
185 0x260d, 0xf00fffff, 0x00000400,
186 0x260e, 0x0002021c, 0x00020200,
187 0x31e, 0x00000080, 0x00000000,
188 0x16ec, 0x000000f0, 0x00000070,
189 0x16f0, 0xf0311fff, 0x80300000,
190 0x263e, 0x73773777, 0x12010001,
191 0xd43, 0x00810000, 0x408af000,
192 0x1c0c, 0x31000111, 0x00000011,
193 0xbd2, 0x73773777, 0x12010001,
194 0x883, 0x00007fb6, 0x0021a1b1,
195 0x884, 0x00007fb6, 0x002021b1,
196 0x860, 0x00007fb6, 0x00002191,
197 0x886, 0x00007fb6, 0x002121b1,
198 0x887, 0x00007fb6, 0x002021b1,
199 0x877, 0x00007fb6, 0x00002191,
200 0x878, 0x00007fb6, 0x00002191,
201 0xd8a, 0x0000003f, 0x0000000a,
202 0xd8b, 0x0000003f, 0x0000000a,
203 0xab9, 0x00073ffe, 0x000022a2,
204 0x903, 0x000007ff, 0x00000000,
205 0x2285, 0xf000003f, 0x00000007,
206 0x22fc, 0x00002001, 0x00000001,
207 0x22c9, 0xffffffff, 0x00ffffff,
208 0xc281, 0x0000ff0f, 0x00000000,
209 0xa293, 0x07ffffff, 0x06000000,
210 0x136, 0x00000fff, 0x00000100,
211 0xf9e, 0x00000001, 0x00000002,
212 0x2440, 0x03000000, 0x0362c688,
213 0x2300, 0x000000ff, 0x00000001,
214 0x390, 0x00001fff, 0x00001fff,
215 0x2418, 0x0000007f, 0x00000020,
216 0x2542, 0x00010000, 0x00010000,
217 0x2b05, 0x000003ff, 0x000000f3,
218 0x2b03, 0xffffffff, 0x00001032
219};
220
221static const u32 bonaire_mgcg_cgcg_init[] =
222{
223 0x3108, 0xffffffff, 0xfffffffc,
224 0xc200, 0xffffffff, 0xe0000000,
225 0xf0a8, 0xffffffff, 0x00000100,
226 0xf082, 0xffffffff, 0x00000100,
227 0xf0b0, 0xffffffff, 0xc0000100,
228 0xf0b2, 0xffffffff, 0xc0000100,
229 0xf0b1, 0xffffffff, 0xc0000100,
230 0x1579, 0xffffffff, 0x00600100,
231 0xf0a0, 0xffffffff, 0x00000100,
232 0xf085, 0xffffffff, 0x06000100,
233 0xf088, 0xffffffff, 0x00000100,
234 0xf086, 0xffffffff, 0x06000100,
235 0xf081, 0xffffffff, 0x00000100,
236 0xf0b8, 0xffffffff, 0x00000100,
237 0xf089, 0xffffffff, 0x00000100,
238 0xf080, 0xffffffff, 0x00000100,
239 0xf08c, 0xffffffff, 0x00000100,
240 0xf08d, 0xffffffff, 0x00000100,
241 0xf094, 0xffffffff, 0x00000100,
242 0xf095, 0xffffffff, 0x00000100,
243 0xf096, 0xffffffff, 0x00000100,
244 0xf097, 0xffffffff, 0x00000100,
245 0xf098, 0xffffffff, 0x00000100,
246 0xf09f, 0xffffffff, 0x00000100,
247 0xf09e, 0xffffffff, 0x00000100,
248 0xf084, 0xffffffff, 0x06000100,
249 0xf0a4, 0xffffffff, 0x00000100,
250 0xf09d, 0xffffffff, 0x00000100,
251 0xf0ad, 0xffffffff, 0x00000100,
252 0xf0ac, 0xffffffff, 0x00000100,
253 0xf09c, 0xffffffff, 0x00000100,
254 0xc200, 0xffffffff, 0xe0000000,
255 0xf008, 0xffffffff, 0x00010000,
256 0xf009, 0xffffffff, 0x00030002,
257 0xf00a, 0xffffffff, 0x00040007,
258 0xf00b, 0xffffffff, 0x00060005,
259 0xf00c, 0xffffffff, 0x00090008,
260 0xf00d, 0xffffffff, 0x00010000,
261 0xf00e, 0xffffffff, 0x00030002,
262 0xf00f, 0xffffffff, 0x00040007,
263 0xf010, 0xffffffff, 0x00060005,
264 0xf011, 0xffffffff, 0x00090008,
265 0xf012, 0xffffffff, 0x00010000,
266 0xf013, 0xffffffff, 0x00030002,
267 0xf014, 0xffffffff, 0x00040007,
268 0xf015, 0xffffffff, 0x00060005,
269 0xf016, 0xffffffff, 0x00090008,
270 0xf017, 0xffffffff, 0x00010000,
271 0xf018, 0xffffffff, 0x00030002,
272 0xf019, 0xffffffff, 0x00040007,
273 0xf01a, 0xffffffff, 0x00060005,
274 0xf01b, 0xffffffff, 0x00090008,
275 0xf01c, 0xffffffff, 0x00010000,
276 0xf01d, 0xffffffff, 0x00030002,
277 0xf01e, 0xffffffff, 0x00040007,
278 0xf01f, 0xffffffff, 0x00060005,
279 0xf020, 0xffffffff, 0x00090008,
280 0xf021, 0xffffffff, 0x00010000,
281 0xf022, 0xffffffff, 0x00030002,
282 0xf023, 0xffffffff, 0x00040007,
283 0xf024, 0xffffffff, 0x00060005,
284 0xf025, 0xffffffff, 0x00090008,
285 0xf026, 0xffffffff, 0x00010000,
286 0xf027, 0xffffffff, 0x00030002,
287 0xf028, 0xffffffff, 0x00040007,
288 0xf029, 0xffffffff, 0x00060005,
289 0xf02a, 0xffffffff, 0x00090008,
290 0xf000, 0xffffffff, 0x96e00200,
291 0x21c2, 0xffffffff, 0x00900100,
292 0x3109, 0xffffffff, 0x0020003f,
293 0xe, 0xffffffff, 0x0140001c,
294 0xf, 0x000f0000, 0x000f0000,
295 0x88, 0xffffffff, 0xc060000c,
296 0x89, 0xc0000fff, 0x00000100,
297 0x3e4, 0xffffffff, 0x00000100,
298 0x3e6, 0x00000101, 0x00000000,
299 0x82a, 0xffffffff, 0x00000104,
300 0x1579, 0xff000fff, 0x00000100,
301 0xc33, 0xc0000fff, 0x00000104,
302 0x3079, 0x00000001, 0x00000001,
303 0x3403, 0xff000ff0, 0x00000100,
304 0x3603, 0xff000ff0, 0x00000100
305};
306
307static const u32 spectre_golden_spm_registers[] =
308{
309 0xc200, 0xe0ffffff, 0xe0000000
310};
311
312static const u32 spectre_golden_common_registers[] =
313{
314 0x31dc, 0xffffffff, 0x00000800,
315 0x31dd, 0xffffffff, 0x00000800,
316 0x31e6, 0xffffffff, 0x00007fbf,
317 0x31e7, 0xffffffff, 0x00007faf
318};
319
320static const u32 spectre_golden_registers[] =
321{
322 0xf000, 0xffff1fff, 0x96940200,
323 0xf003, 0xffff0001, 0xff000000,
324 0xf080, 0xfffc0fff, 0x00000100,
325 0x1bb6, 0x00010101, 0x00010000,
326 0x260d, 0xf00fffff, 0x00000400,
327 0x260e, 0xfffffffc, 0x00020200,
328 0x16ec, 0x000000f0, 0x00000070,
329 0x16f0, 0xf0311fff, 0x80300000,
330 0x263e, 0x73773777, 0x12010001,
331 0x26df, 0x00ff0000, 0x00fc0000,
332 0xbd2, 0x73773777, 0x12010001,
333 0x2285, 0xf000003f, 0x00000007,
334 0x22c9, 0xffffffff, 0x00ffffff,
335 0xa0d4, 0x3f3f3fff, 0x00000082,
336 0xa0d5, 0x0000003f, 0x00000000,
337 0xf9e, 0x00000001, 0x00000002,
338 0x244f, 0xffff03df, 0x00000004,
339 0x31da, 0x00000008, 0x00000008,
340 0x2300, 0x000008ff, 0x00000800,
341 0x2542, 0x00010000, 0x00010000,
342 0x2b03, 0xffffffff, 0x54763210,
343 0x853e, 0x01ff01ff, 0x00000002,
344 0x8526, 0x007ff800, 0x00200000,
345 0x8057, 0xffffffff, 0x00000f40,
346 0xc24d, 0xffffffff, 0x00000001
347};
348
349static const u32 spectre_mgcg_cgcg_init[] =
350{
351 0x3108, 0xffffffff, 0xfffffffc,
352 0xc200, 0xffffffff, 0xe0000000,
353 0xf0a8, 0xffffffff, 0x00000100,
354 0xf082, 0xffffffff, 0x00000100,
355 0xf0b0, 0xffffffff, 0x00000100,
356 0xf0b2, 0xffffffff, 0x00000100,
357 0xf0b1, 0xffffffff, 0x00000100,
358 0x1579, 0xffffffff, 0x00600100,
359 0xf0a0, 0xffffffff, 0x00000100,
360 0xf085, 0xffffffff, 0x06000100,
361 0xf088, 0xffffffff, 0x00000100,
362 0xf086, 0xffffffff, 0x06000100,
363 0xf081, 0xffffffff, 0x00000100,
364 0xf0b8, 0xffffffff, 0x00000100,
365 0xf089, 0xffffffff, 0x00000100,
366 0xf080, 0xffffffff, 0x00000100,
367 0xf08c, 0xffffffff, 0x00000100,
368 0xf08d, 0xffffffff, 0x00000100,
369 0xf094, 0xffffffff, 0x00000100,
370 0xf095, 0xffffffff, 0x00000100,
371 0xf096, 0xffffffff, 0x00000100,
372 0xf097, 0xffffffff, 0x00000100,
373 0xf098, 0xffffffff, 0x00000100,
374 0xf09f, 0xffffffff, 0x00000100,
375 0xf09e, 0xffffffff, 0x00000100,
376 0xf084, 0xffffffff, 0x06000100,
377 0xf0a4, 0xffffffff, 0x00000100,
378 0xf09d, 0xffffffff, 0x00000100,
379 0xf0ad, 0xffffffff, 0x00000100,
380 0xf0ac, 0xffffffff, 0x00000100,
381 0xf09c, 0xffffffff, 0x00000100,
382 0xc200, 0xffffffff, 0xe0000000,
383 0xf008, 0xffffffff, 0x00010000,
384 0xf009, 0xffffffff, 0x00030002,
385 0xf00a, 0xffffffff, 0x00040007,
386 0xf00b, 0xffffffff, 0x00060005,
387 0xf00c, 0xffffffff, 0x00090008,
388 0xf00d, 0xffffffff, 0x00010000,
389 0xf00e, 0xffffffff, 0x00030002,
390 0xf00f, 0xffffffff, 0x00040007,
391 0xf010, 0xffffffff, 0x00060005,
392 0xf011, 0xffffffff, 0x00090008,
393 0xf012, 0xffffffff, 0x00010000,
394 0xf013, 0xffffffff, 0x00030002,
395 0xf014, 0xffffffff, 0x00040007,
396 0xf015, 0xffffffff, 0x00060005,
397 0xf016, 0xffffffff, 0x00090008,
398 0xf017, 0xffffffff, 0x00010000,
399 0xf018, 0xffffffff, 0x00030002,
400 0xf019, 0xffffffff, 0x00040007,
401 0xf01a, 0xffffffff, 0x00060005,
402 0xf01b, 0xffffffff, 0x00090008,
403 0xf01c, 0xffffffff, 0x00010000,
404 0xf01d, 0xffffffff, 0x00030002,
405 0xf01e, 0xffffffff, 0x00040007,
406 0xf01f, 0xffffffff, 0x00060005,
407 0xf020, 0xffffffff, 0x00090008,
408 0xf021, 0xffffffff, 0x00010000,
409 0xf022, 0xffffffff, 0x00030002,
410 0xf023, 0xffffffff, 0x00040007,
411 0xf024, 0xffffffff, 0x00060005,
412 0xf025, 0xffffffff, 0x00090008,
413 0xf026, 0xffffffff, 0x00010000,
414 0xf027, 0xffffffff, 0x00030002,
415 0xf028, 0xffffffff, 0x00040007,
416 0xf029, 0xffffffff, 0x00060005,
417 0xf02a, 0xffffffff, 0x00090008,
418 0xf02b, 0xffffffff, 0x00010000,
419 0xf02c, 0xffffffff, 0x00030002,
420 0xf02d, 0xffffffff, 0x00040007,
421 0xf02e, 0xffffffff, 0x00060005,
422 0xf02f, 0xffffffff, 0x00090008,
423 0xf000, 0xffffffff, 0x96e00200,
424 0x21c2, 0xffffffff, 0x00900100,
425 0x3109, 0xffffffff, 0x0020003f,
426 0xe, 0xffffffff, 0x0140001c,
427 0xf, 0x000f0000, 0x000f0000,
428 0x88, 0xffffffff, 0xc060000c,
429 0x89, 0xc0000fff, 0x00000100,
430 0x3e4, 0xffffffff, 0x00000100,
431 0x3e6, 0x00000101, 0x00000000,
432 0x82a, 0xffffffff, 0x00000104,
433 0x1579, 0xff000fff, 0x00000100,
434 0xc33, 0xc0000fff, 0x00000104,
435 0x3079, 0x00000001, 0x00000001,
436 0x3403, 0xff000ff0, 0x00000100,
437 0x3603, 0xff000ff0, 0x00000100
438};
439
440static const u32 kalindi_golden_spm_registers[] =
441{
442 0xc200, 0xe0ffffff, 0xe0000000
443};
444
445static const u32 kalindi_golden_common_registers[] =
446{
447 0x31dc, 0xffffffff, 0x00000800,
448 0x31dd, 0xffffffff, 0x00000800,
449 0x31e6, 0xffffffff, 0x00007fbf,
450 0x31e7, 0xffffffff, 0x00007faf
451};
452
453static const u32 kalindi_golden_registers[] =
454{
455 0xf000, 0xffffdfff, 0x6e944040,
456 0x1579, 0xff607fff, 0xfc000100,
457 0xf088, 0xff000fff, 0x00000100,
458 0xf089, 0xff000fff, 0x00000100,
459 0xf080, 0xfffc0fff, 0x00000100,
460 0x1bb6, 0x00010101, 0x00010000,
461 0x260c, 0xffffffff, 0x00000000,
462 0x260d, 0xf00fffff, 0x00000400,
463 0x16ec, 0x000000f0, 0x00000070,
464 0x16f0, 0xf0311fff, 0x80300000,
465 0x263e, 0x73773777, 0x12010001,
466 0x263f, 0xffffffff, 0x00000010,
467 0x26df, 0x00ff0000, 0x00fc0000,
468 0x200c, 0x00001f0f, 0x0000100a,
469 0xbd2, 0x73773777, 0x12010001,
470 0x902, 0x000fffff, 0x000c007f,
471 0x2285, 0xf000003f, 0x00000007,
472 0x22c9, 0x3fff3fff, 0x00ffcfff,
473 0xc281, 0x0000ff0f, 0x00000000,
474 0xa293, 0x07ffffff, 0x06000000,
475 0x136, 0x00000fff, 0x00000100,
476 0xf9e, 0x00000001, 0x00000002,
477 0x31da, 0x00000008, 0x00000008,
478 0x2300, 0x000000ff, 0x00000003,
479 0x853e, 0x01ff01ff, 0x00000002,
480 0x8526, 0x007ff800, 0x00200000,
481 0x8057, 0xffffffff, 0x00000f40,
482 0x2231, 0x001f3ae3, 0x00000082,
483 0x2235, 0x0000001f, 0x00000010,
484 0xc24d, 0xffffffff, 0x00000000
485};
486
487static const u32 kalindi_mgcg_cgcg_init[] =
488{
489 0x3108, 0xffffffff, 0xfffffffc,
490 0xc200, 0xffffffff, 0xe0000000,
491 0xf0a8, 0xffffffff, 0x00000100,
492 0xf082, 0xffffffff, 0x00000100,
493 0xf0b0, 0xffffffff, 0x00000100,
494 0xf0b2, 0xffffffff, 0x00000100,
495 0xf0b1, 0xffffffff, 0x00000100,
496 0x1579, 0xffffffff, 0x00600100,
497 0xf0a0, 0xffffffff, 0x00000100,
498 0xf085, 0xffffffff, 0x06000100,
499 0xf088, 0xffffffff, 0x00000100,
500 0xf086, 0xffffffff, 0x06000100,
501 0xf081, 0xffffffff, 0x00000100,
502 0xf0b8, 0xffffffff, 0x00000100,
503 0xf089, 0xffffffff, 0x00000100,
504 0xf080, 0xffffffff, 0x00000100,
505 0xf08c, 0xffffffff, 0x00000100,
506 0xf08d, 0xffffffff, 0x00000100,
507 0xf094, 0xffffffff, 0x00000100,
508 0xf095, 0xffffffff, 0x00000100,
509 0xf096, 0xffffffff, 0x00000100,
510 0xf097, 0xffffffff, 0x00000100,
511 0xf098, 0xffffffff, 0x00000100,
512 0xf09f, 0xffffffff, 0x00000100,
513 0xf09e, 0xffffffff, 0x00000100,
514 0xf084, 0xffffffff, 0x06000100,
515 0xf0a4, 0xffffffff, 0x00000100,
516 0xf09d, 0xffffffff, 0x00000100,
517 0xf0ad, 0xffffffff, 0x00000100,
518 0xf0ac, 0xffffffff, 0x00000100,
519 0xf09c, 0xffffffff, 0x00000100,
520 0xc200, 0xffffffff, 0xe0000000,
521 0xf008, 0xffffffff, 0x00010000,
522 0xf009, 0xffffffff, 0x00030002,
523 0xf00a, 0xffffffff, 0x00040007,
524 0xf00b, 0xffffffff, 0x00060005,
525 0xf00c, 0xffffffff, 0x00090008,
526 0xf00d, 0xffffffff, 0x00010000,
527 0xf00e, 0xffffffff, 0x00030002,
528 0xf00f, 0xffffffff, 0x00040007,
529 0xf010, 0xffffffff, 0x00060005,
530 0xf011, 0xffffffff, 0x00090008,
531 0xf000, 0xffffffff, 0x96e00200,
532 0x21c2, 0xffffffff, 0x00900100,
533 0x3109, 0xffffffff, 0x0020003f,
534 0xe, 0xffffffff, 0x0140001c,
535 0xf, 0x000f0000, 0x000f0000,
536 0x88, 0xffffffff, 0xc060000c,
537 0x89, 0xc0000fff, 0x00000100,
538 0x82a, 0xffffffff, 0x00000104,
539 0x1579, 0xff000fff, 0x00000100,
540 0xc33, 0xc0000fff, 0x00000104,
541 0x3079, 0x00000001, 0x00000001,
542 0x3403, 0xff000ff0, 0x00000100,
543 0x3603, 0xff000ff0, 0x00000100
544};
545
546static const u32 hawaii_golden_spm_registers[] =
547{
548 0xc200, 0xe0ffffff, 0xe0000000
549};
550
551static const u32 hawaii_golden_common_registers[] =
552{
553 0xc200, 0xffffffff, 0xe0000000,
554 0xa0d4, 0xffffffff, 0x3a00161a,
555 0xa0d5, 0xffffffff, 0x0000002e,
556 0x2684, 0xffffffff, 0x00018208,
557 0x263e, 0xffffffff, 0x12011003
558};
559
560static const u32 hawaii_golden_registers[] =
561{
562 0xcd5, 0x00000333, 0x00000333,
563 0x2684, 0x00010000, 0x00058208,
564 0x260c, 0xffffffff, 0x00000000,
565 0x260d, 0xf00fffff, 0x00000400,
566 0x260e, 0x0002021c, 0x00020200,
567 0x31e, 0x00000080, 0x00000000,
568 0x16ec, 0x000000f0, 0x00000070,
569 0x16f0, 0xf0311fff, 0x80300000,
570 0xd43, 0x00810000, 0x408af000,
571 0x1c0c, 0x31000111, 0x00000011,
572 0xbd2, 0x73773777, 0x12010001,
573 0x848, 0x0000007f, 0x0000001b,
574 0x877, 0x00007fb6, 0x00002191,
575 0xd8a, 0x0000003f, 0x0000000a,
576 0xd8b, 0x0000003f, 0x0000000a,
577 0xab9, 0x00073ffe, 0x000022a2,
578 0x903, 0x000007ff, 0x00000000,
579 0x22fc, 0x00002001, 0x00000001,
580 0x22c9, 0xffffffff, 0x00ffffff,
581 0xc281, 0x0000ff0f, 0x00000000,
582 0xa293, 0x07ffffff, 0x06000000,
583 0xf9e, 0x00000001, 0x00000002,
584 0x31da, 0x00000008, 0x00000008,
585 0x31dc, 0x00000f00, 0x00000800,
586 0x31dd, 0x00000f00, 0x00000800,
587 0x31e6, 0x00ffffff, 0x00ff7fbf,
588 0x31e7, 0x00ffffff, 0x00ff7faf,
589 0x2300, 0x000000ff, 0x00000800,
590 0x390, 0x00001fff, 0x00001fff,
591 0x2418, 0x0000007f, 0x00000020,
592 0x2542, 0x00010000, 0x00010000,
593 0x2b80, 0x00100000, 0x000ff07c,
594 0x2b05, 0x000003ff, 0x0000000f,
595 0x2b04, 0xffffffff, 0x7564fdec,
596 0x2b03, 0xffffffff, 0x3120b9a8,
597 0x2b02, 0x20000000, 0x0f9c0000
598};
599
600static const u32 hawaii_mgcg_cgcg_init[] =
601{
602 0x3108, 0xffffffff, 0xfffffffd,
603 0xc200, 0xffffffff, 0xe0000000,
604 0xf0a8, 0xffffffff, 0x00000100,
605 0xf082, 0xffffffff, 0x00000100,
606 0xf0b0, 0xffffffff, 0x00000100,
607 0xf0b2, 0xffffffff, 0x00000100,
608 0xf0b1, 0xffffffff, 0x00000100,
609 0x1579, 0xffffffff, 0x00200100,
610 0xf0a0, 0xffffffff, 0x00000100,
611 0xf085, 0xffffffff, 0x06000100,
612 0xf088, 0xffffffff, 0x00000100,
613 0xf086, 0xffffffff, 0x06000100,
614 0xf081, 0xffffffff, 0x00000100,
615 0xf0b8, 0xffffffff, 0x00000100,
616 0xf089, 0xffffffff, 0x00000100,
617 0xf080, 0xffffffff, 0x00000100,
618 0xf08c, 0xffffffff, 0x00000100,
619 0xf08d, 0xffffffff, 0x00000100,
620 0xf094, 0xffffffff, 0x00000100,
621 0xf095, 0xffffffff, 0x00000100,
622 0xf096, 0xffffffff, 0x00000100,
623 0xf097, 0xffffffff, 0x00000100,
624 0xf098, 0xffffffff, 0x00000100,
625 0xf09f, 0xffffffff, 0x00000100,
626 0xf09e, 0xffffffff, 0x00000100,
627 0xf084, 0xffffffff, 0x06000100,
628 0xf0a4, 0xffffffff, 0x00000100,
629 0xf09d, 0xffffffff, 0x00000100,
630 0xf0ad, 0xffffffff, 0x00000100,
631 0xf0ac, 0xffffffff, 0x00000100,
632 0xf09c, 0xffffffff, 0x00000100,
633 0xc200, 0xffffffff, 0xe0000000,
634 0xf008, 0xffffffff, 0x00010000,
635 0xf009, 0xffffffff, 0x00030002,
636 0xf00a, 0xffffffff, 0x00040007,
637 0xf00b, 0xffffffff, 0x00060005,
638 0xf00c, 0xffffffff, 0x00090008,
639 0xf00d, 0xffffffff, 0x00010000,
640 0xf00e, 0xffffffff, 0x00030002,
641 0xf00f, 0xffffffff, 0x00040007,
642 0xf010, 0xffffffff, 0x00060005,
643 0xf011, 0xffffffff, 0x00090008,
644 0xf012, 0xffffffff, 0x00010000,
645 0xf013, 0xffffffff, 0x00030002,
646 0xf014, 0xffffffff, 0x00040007,
647 0xf015, 0xffffffff, 0x00060005,
648 0xf016, 0xffffffff, 0x00090008,
649 0xf017, 0xffffffff, 0x00010000,
650 0xf018, 0xffffffff, 0x00030002,
651 0xf019, 0xffffffff, 0x00040007,
652 0xf01a, 0xffffffff, 0x00060005,
653 0xf01b, 0xffffffff, 0x00090008,
654 0xf01c, 0xffffffff, 0x00010000,
655 0xf01d, 0xffffffff, 0x00030002,
656 0xf01e, 0xffffffff, 0x00040007,
657 0xf01f, 0xffffffff, 0x00060005,
658 0xf020, 0xffffffff, 0x00090008,
659 0xf021, 0xffffffff, 0x00010000,
660 0xf022, 0xffffffff, 0x00030002,
661 0xf023, 0xffffffff, 0x00040007,
662 0xf024, 0xffffffff, 0x00060005,
663 0xf025, 0xffffffff, 0x00090008,
664 0xf026, 0xffffffff, 0x00010000,
665 0xf027, 0xffffffff, 0x00030002,
666 0xf028, 0xffffffff, 0x00040007,
667 0xf029, 0xffffffff, 0x00060005,
668 0xf02a, 0xffffffff, 0x00090008,
669 0xf02b, 0xffffffff, 0x00010000,
670 0xf02c, 0xffffffff, 0x00030002,
671 0xf02d, 0xffffffff, 0x00040007,
672 0xf02e, 0xffffffff, 0x00060005,
673 0xf02f, 0xffffffff, 0x00090008,
674 0xf030, 0xffffffff, 0x00010000,
675 0xf031, 0xffffffff, 0x00030002,
676 0xf032, 0xffffffff, 0x00040007,
677 0xf033, 0xffffffff, 0x00060005,
678 0xf034, 0xffffffff, 0x00090008,
679 0xf035, 0xffffffff, 0x00010000,
680 0xf036, 0xffffffff, 0x00030002,
681 0xf037, 0xffffffff, 0x00040007,
682 0xf038, 0xffffffff, 0x00060005,
683 0xf039, 0xffffffff, 0x00090008,
684 0xf03a, 0xffffffff, 0x00010000,
685 0xf03b, 0xffffffff, 0x00030002,
686 0xf03c, 0xffffffff, 0x00040007,
687 0xf03d, 0xffffffff, 0x00060005,
688 0xf03e, 0xffffffff, 0x00090008,
689 0x30c6, 0xffffffff, 0x00020200,
690 0xcd4, 0xffffffff, 0x00000200,
691 0x570, 0xffffffff, 0x00000400,
692 0x157a, 0xffffffff, 0x00000000,
693 0xbd4, 0xffffffff, 0x00000902,
694 0xf000, 0xffffffff, 0x96940200,
695 0x21c2, 0xffffffff, 0x00900100,
696 0x3109, 0xffffffff, 0x0020003f,
697 0xe, 0xffffffff, 0x0140001c,
698 0xf, 0x000f0000, 0x000f0000,
699 0x88, 0xffffffff, 0xc060000c,
700 0x89, 0xc0000fff, 0x00000100,
701 0x3e4, 0xffffffff, 0x00000100,
702 0x3e6, 0x00000101, 0x00000000,
703 0x82a, 0xffffffff, 0x00000104,
704 0x1579, 0xff000fff, 0x00000100,
705 0xc33, 0xc0000fff, 0x00000104,
706 0x3079, 0x00000001, 0x00000001,
707 0x3403, 0xff000ff0, 0x00000100,
708 0x3603, 0xff000ff0, 0x00000100
709};
710
711static const u32 godavari_golden_registers[] =
712{
713 0x1579, 0xff607fff, 0xfc000100,
714 0x1bb6, 0x00010101, 0x00010000,
715 0x260c, 0xffffffff, 0x00000000,
716 0x260c0, 0xf00fffff, 0x00000400,
717 0x184c, 0xffffffff, 0x00010000,
718 0x16ec, 0x000000f0, 0x00000070,
719 0x16f0, 0xf0311fff, 0x80300000,
720 0x263e, 0x73773777, 0x12010001,
721 0x263f, 0xffffffff, 0x00000010,
722 0x200c, 0x00001f0f, 0x0000100a,
723 0xbd2, 0x73773777, 0x12010001,
724 0x902, 0x000fffff, 0x000c007f,
725 0x2285, 0xf000003f, 0x00000007,
726 0x22c9, 0xffffffff, 0x00ff0fff,
727 0xc281, 0x0000ff0f, 0x00000000,
728 0xa293, 0x07ffffff, 0x06000000,
729 0x136, 0x00000fff, 0x00000100,
730 0x3405, 0x00010000, 0x00810001,
731 0x3605, 0x00010000, 0x00810001,
732 0xf9e, 0x00000001, 0x00000002,
733 0x31da, 0x00000008, 0x00000008,
734 0x31dc, 0x00000f00, 0x00000800,
735 0x31dd, 0x00000f00, 0x00000800,
736 0x31e6, 0x00ffffff, 0x00ff7fbf,
737 0x31e7, 0x00ffffff, 0x00ff7faf,
738 0x2300, 0x000000ff, 0x00000001,
739 0x853e, 0x01ff01ff, 0x00000002,
740 0x8526, 0x007ff800, 0x00200000,
741 0x8057, 0xffffffff, 0x00000f40,
742 0x2231, 0x001f3ae3, 0x00000082,
743 0x2235, 0x0000001f, 0x00000010,
744 0xc24d, 0xffffffff, 0x00000000
745};
746
747static void cik_init_golden_registers(struct amdgpu_device *adev)
748{
749
750 mutex_lock(&adev->grbm_idx_mutex);
751
752 switch (adev->asic_type) {
753 case CHIP_BONAIRE:
754 amdgpu_program_register_sequence(adev,
755 bonaire_mgcg_cgcg_init,
756 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
757 amdgpu_program_register_sequence(adev,
758 bonaire_golden_registers,
759 (const u32)ARRAY_SIZE(bonaire_golden_registers));
760 amdgpu_program_register_sequence(adev,
761 bonaire_golden_common_registers,
762 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
763 amdgpu_program_register_sequence(adev,
764 bonaire_golden_spm_registers,
765 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
766 break;
767 case CHIP_KABINI:
768 amdgpu_program_register_sequence(adev,
769 kalindi_mgcg_cgcg_init,
770 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
771 amdgpu_program_register_sequence(adev,
772 kalindi_golden_registers,
773 (const u32)ARRAY_SIZE(kalindi_golden_registers));
774 amdgpu_program_register_sequence(adev,
775 kalindi_golden_common_registers,
776 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
777 amdgpu_program_register_sequence(adev,
778 kalindi_golden_spm_registers,
779 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
780 break;
781 case CHIP_MULLINS:
782 amdgpu_program_register_sequence(adev,
783 kalindi_mgcg_cgcg_init,
784 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
785 amdgpu_program_register_sequence(adev,
786 godavari_golden_registers,
787 (const u32)ARRAY_SIZE(godavari_golden_registers));
788 amdgpu_program_register_sequence(adev,
789 kalindi_golden_common_registers,
790 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
791 amdgpu_program_register_sequence(adev,
792 kalindi_golden_spm_registers,
793 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
794 break;
795 case CHIP_KAVERI:
796 amdgpu_program_register_sequence(adev,
797 spectre_mgcg_cgcg_init,
798 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
799 amdgpu_program_register_sequence(adev,
800 spectre_golden_registers,
801 (const u32)ARRAY_SIZE(spectre_golden_registers));
802 amdgpu_program_register_sequence(adev,
803 spectre_golden_common_registers,
804 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
805 amdgpu_program_register_sequence(adev,
806 spectre_golden_spm_registers,
807 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
808 break;
809 case CHIP_HAWAII:
810 amdgpu_program_register_sequence(adev,
811 hawaii_mgcg_cgcg_init,
812 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
813 amdgpu_program_register_sequence(adev,
814 hawaii_golden_registers,
815 (const u32)ARRAY_SIZE(hawaii_golden_registers));
816 amdgpu_program_register_sequence(adev,
817 hawaii_golden_common_registers,
818 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
819 amdgpu_program_register_sequence(adev,
820 hawaii_golden_spm_registers,
821 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
822 break;
823 default:
824 break;
825 }
826 mutex_unlock(&adev->grbm_idx_mutex);
827}
828
829
830
831
832
833
834
835
836
837static u32 cik_get_xclk(struct amdgpu_device *adev)
838{
839 u32 reference_clock = adev->clock.spll.reference_freq;
840
841 if (adev->flags & AMD_IS_APU) {
842 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
843 return reference_clock / 2;
844 } else {
845 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
846 return reference_clock / 4;
847 }
848 return reference_clock;
849}
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864void cik_srbm_select(struct amdgpu_device *adev,
865 u32 me, u32 pipe, u32 queue, u32 vmid)
866{
867 u32 srbm_gfx_cntl =
868 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
869 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
870 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
871 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
872 WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
873}
874
875static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
876{
877 uint32_t tmp;
878
879 tmp = RREG32(mmCONFIG_CNTL);
880 if (state == false)
881 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
882 else
883 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
884 WREG32(mmCONFIG_CNTL, tmp);
885}
886
887static bool cik_read_disabled_bios(struct amdgpu_device *adev)
888{
889 u32 bus_cntl;
890 u32 d1vga_control = 0;
891 u32 d2vga_control = 0;
892 u32 vga_render_control = 0;
893 u32 rom_cntl;
894 bool r;
895
896 bus_cntl = RREG32(mmBUS_CNTL);
897 if (adev->mode_info.num_crtc) {
898 d1vga_control = RREG32(mmD1VGA_CONTROL);
899 d2vga_control = RREG32(mmD2VGA_CONTROL);
900 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
901 }
902 rom_cntl = RREG32_SMC(ixROM_CNTL);
903
904
905 WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
906 if (adev->mode_info.num_crtc) {
907
908 WREG32(mmD1VGA_CONTROL,
909 (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
910 D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
911 WREG32(mmD2VGA_CONTROL,
912 (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
913 D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
914 WREG32(mmVGA_RENDER_CONTROL,
915 (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
916 }
917 WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
918
919 r = amdgpu_read_bios(adev);
920
921
922 WREG32(mmBUS_CNTL, bus_cntl);
923 if (adev->mode_info.num_crtc) {
924 WREG32(mmD1VGA_CONTROL, d1vga_control);
925 WREG32(mmD2VGA_CONTROL, d2vga_control);
926 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
927 }
928 WREG32_SMC(ixROM_CNTL, rom_cntl);
929 return r;
930}
931
932static struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
933 {mmGRBM_STATUS, false},
934 {mmGB_ADDR_CONFIG, false},
935 {mmMC_ARB_RAMCFG, false},
936 {mmGB_TILE_MODE0, false},
937 {mmGB_TILE_MODE1, false},
938 {mmGB_TILE_MODE2, false},
939 {mmGB_TILE_MODE3, false},
940 {mmGB_TILE_MODE4, false},
941 {mmGB_TILE_MODE5, false},
942 {mmGB_TILE_MODE6, false},
943 {mmGB_TILE_MODE7, false},
944 {mmGB_TILE_MODE8, false},
945 {mmGB_TILE_MODE9, false},
946 {mmGB_TILE_MODE10, false},
947 {mmGB_TILE_MODE11, false},
948 {mmGB_TILE_MODE12, false},
949 {mmGB_TILE_MODE13, false},
950 {mmGB_TILE_MODE14, false},
951 {mmGB_TILE_MODE15, false},
952 {mmGB_TILE_MODE16, false},
953 {mmGB_TILE_MODE17, false},
954 {mmGB_TILE_MODE18, false},
955 {mmGB_TILE_MODE19, false},
956 {mmGB_TILE_MODE20, false},
957 {mmGB_TILE_MODE21, false},
958 {mmGB_TILE_MODE22, false},
959 {mmGB_TILE_MODE23, false},
960 {mmGB_TILE_MODE24, false},
961 {mmGB_TILE_MODE25, false},
962 {mmGB_TILE_MODE26, false},
963 {mmGB_TILE_MODE27, false},
964 {mmGB_TILE_MODE28, false},
965 {mmGB_TILE_MODE29, false},
966 {mmGB_TILE_MODE30, false},
967 {mmGB_TILE_MODE31, false},
968 {mmGB_MACROTILE_MODE0, false},
969 {mmGB_MACROTILE_MODE1, false},
970 {mmGB_MACROTILE_MODE2, false},
971 {mmGB_MACROTILE_MODE3, false},
972 {mmGB_MACROTILE_MODE4, false},
973 {mmGB_MACROTILE_MODE5, false},
974 {mmGB_MACROTILE_MODE6, false},
975 {mmGB_MACROTILE_MODE7, false},
976 {mmGB_MACROTILE_MODE8, false},
977 {mmGB_MACROTILE_MODE9, false},
978 {mmGB_MACROTILE_MODE10, false},
979 {mmGB_MACROTILE_MODE11, false},
980 {mmGB_MACROTILE_MODE12, false},
981 {mmGB_MACROTILE_MODE13, false},
982 {mmGB_MACROTILE_MODE14, false},
983 {mmGB_MACROTILE_MODE15, false},
984 {mmCC_RB_BACKEND_DISABLE, false, true},
985 {mmGC_USER_RB_BACKEND_DISABLE, false, true},
986 {mmGB_BACKEND_MAP, false, false},
987 {mmPA_SC_RASTER_CONFIG, false, true},
988 {mmPA_SC_RASTER_CONFIG_1, false, true},
989};
990
991static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
992 u32 se_num, u32 sh_num,
993 u32 reg_offset)
994{
995 uint32_t val;
996
997 mutex_lock(&adev->grbm_idx_mutex);
998 if (se_num != 0xffffffff || sh_num != 0xffffffff)
999 gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1000
1001 val = RREG32(reg_offset);
1002
1003 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1004 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1005 mutex_unlock(&adev->grbm_idx_mutex);
1006 return val;
1007}
1008
1009static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1010 u32 sh_num, u32 reg_offset, u32 *value)
1011{
1012 uint32_t i;
1013
1014 *value = 0;
1015 for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1016 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1017 continue;
1018
1019 if (!cik_allowed_read_registers[i].untouched)
1020 *value = cik_allowed_read_registers[i].grbm_indexed ?
1021 cik_read_indexed_register(adev, se_num,
1022 sh_num, reg_offset) :
1023 RREG32(reg_offset);
1024 return 0;
1025 }
1026 return -EINVAL;
1027}
1028
1029static void cik_print_gpu_status_regs(struct amdgpu_device *adev)
1030{
1031 dev_info(adev->dev, " GRBM_STATUS=0x%08X\n",
1032 RREG32(mmGRBM_STATUS));
1033 dev_info(adev->dev, " GRBM_STATUS2=0x%08X\n",
1034 RREG32(mmGRBM_STATUS2));
1035 dev_info(adev->dev, " GRBM_STATUS_SE0=0x%08X\n",
1036 RREG32(mmGRBM_STATUS_SE0));
1037 dev_info(adev->dev, " GRBM_STATUS_SE1=0x%08X\n",
1038 RREG32(mmGRBM_STATUS_SE1));
1039 dev_info(adev->dev, " GRBM_STATUS_SE2=0x%08X\n",
1040 RREG32(mmGRBM_STATUS_SE2));
1041 dev_info(adev->dev, " GRBM_STATUS_SE3=0x%08X\n",
1042 RREG32(mmGRBM_STATUS_SE3));
1043 dev_info(adev->dev, " SRBM_STATUS=0x%08X\n",
1044 RREG32(mmSRBM_STATUS));
1045 dev_info(adev->dev, " SRBM_STATUS2=0x%08X\n",
1046 RREG32(mmSRBM_STATUS2));
1047 dev_info(adev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
1048 RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
1049 dev_info(adev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
1050 RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
1051 dev_info(adev->dev, " CP_STAT = 0x%08x\n", RREG32(mmCP_STAT));
1052 dev_info(adev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
1053 RREG32(mmCP_STALLED_STAT1));
1054 dev_info(adev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
1055 RREG32(mmCP_STALLED_STAT2));
1056 dev_info(adev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
1057 RREG32(mmCP_STALLED_STAT3));
1058 dev_info(adev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
1059 RREG32(mmCP_CPF_BUSY_STAT));
1060 dev_info(adev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
1061 RREG32(mmCP_CPF_STALLED_STAT1));
1062 dev_info(adev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(mmCP_CPF_STATUS));
1063 dev_info(adev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(mmCP_CPC_BUSY_STAT));
1064 dev_info(adev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
1065 RREG32(mmCP_CPC_STALLED_STAT1));
1066 dev_info(adev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(mmCP_CPC_STATUS));
1067}
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev)
1079{
1080 u32 reset_mask = 0;
1081 u32 tmp;
1082
1083
1084 tmp = RREG32(mmGRBM_STATUS);
1085 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
1086 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
1087 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
1088 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
1089 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
1090 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK))
1091 reset_mask |= AMDGPU_RESET_GFX;
1092
1093 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK))
1094 reset_mask |= AMDGPU_RESET_CP;
1095
1096
1097 tmp = RREG32(mmGRBM_STATUS2);
1098 if (tmp & GRBM_STATUS2__RLC_BUSY_MASK)
1099 reset_mask |= AMDGPU_RESET_RLC;
1100
1101
1102 tmp = RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
1103 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1104 reset_mask |= AMDGPU_RESET_DMA;
1105
1106
1107 tmp = RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
1108 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1109 reset_mask |= AMDGPU_RESET_DMA1;
1110
1111
1112 tmp = RREG32(mmSRBM_STATUS2);
1113 if (tmp & SRBM_STATUS2__SDMA_BUSY_MASK)
1114 reset_mask |= AMDGPU_RESET_DMA;
1115
1116 if (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)
1117 reset_mask |= AMDGPU_RESET_DMA1;
1118
1119
1120 tmp = RREG32(mmSRBM_STATUS);
1121
1122 if (tmp & SRBM_STATUS__IH_BUSY_MASK)
1123 reset_mask |= AMDGPU_RESET_IH;
1124
1125 if (tmp & SRBM_STATUS__SEM_BUSY_MASK)
1126 reset_mask |= AMDGPU_RESET_SEM;
1127
1128 if (tmp & SRBM_STATUS__GRBM_RQ_PENDING_MASK)
1129 reset_mask |= AMDGPU_RESET_GRBM;
1130
1131 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1132 reset_mask |= AMDGPU_RESET_VMC;
1133
1134 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1135 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK))
1136 reset_mask |= AMDGPU_RESET_MC;
1137
1138 if (amdgpu_display_is_display_hung(adev))
1139 reset_mask |= AMDGPU_RESET_DISPLAY;
1140
1141
1142 if (reset_mask & AMDGPU_RESET_MC) {
1143 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1144 reset_mask &= ~AMDGPU_RESET_MC;
1145 }
1146
1147 return reset_mask;
1148}
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158static void cik_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask)
1159{
1160 struct amdgpu_mode_mc_save save;
1161 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1162 u32 tmp;
1163
1164 if (reset_mask == 0)
1165 return;
1166
1167 dev_info(adev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1168
1169 cik_print_gpu_status_regs(adev);
1170 dev_info(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1171 RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR));
1172 dev_info(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1173 RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS));
1174
1175
1176
1177
1178 gfx_v7_0_rlc_stop(adev);
1179
1180
1181 WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
1182
1183
1184 WREG32(mmCP_MEC_CNTL, CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
1185
1186 if (reset_mask & AMDGPU_RESET_DMA) {
1187
1188 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1189 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1190 WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1191 }
1192 if (reset_mask & AMDGPU_RESET_DMA1) {
1193
1194 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1195 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1196 WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1197 }
1198
1199 gmc_v7_0_mc_stop(adev, &save);
1200 if (amdgpu_asic_wait_for_mc_idle(adev)) {
1201 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
1202 }
1203
1204 if (reset_mask & (AMDGPU_RESET_GFX | AMDGPU_RESET_COMPUTE | AMDGPU_RESET_CP))
1205 grbm_soft_reset = GRBM_SOFT_RESET__SOFT_RESET_CP_MASK |
1206 GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK;
1207
1208 if (reset_mask & AMDGPU_RESET_CP) {
1209 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK;
1210
1211 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
1212 }
1213
1214 if (reset_mask & AMDGPU_RESET_DMA)
1215 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;
1216
1217 if (reset_mask & AMDGPU_RESET_DMA1)
1218 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
1219
1220 if (reset_mask & AMDGPU_RESET_DISPLAY)
1221 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
1222
1223 if (reset_mask & AMDGPU_RESET_RLC)
1224 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
1225
1226 if (reset_mask & AMDGPU_RESET_SEM)
1227 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SEM_MASK;
1228
1229 if (reset_mask & AMDGPU_RESET_IH)
1230 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
1231
1232 if (reset_mask & AMDGPU_RESET_GRBM)
1233 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
1234
1235 if (reset_mask & AMDGPU_RESET_VMC)
1236 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_VMC_MASK;
1237
1238 if (!(adev->flags & AMD_IS_APU)) {
1239 if (reset_mask & AMDGPU_RESET_MC)
1240 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_MC_MASK;
1241 }
1242
1243 if (grbm_soft_reset) {
1244 tmp = RREG32(mmGRBM_SOFT_RESET);
1245 tmp |= grbm_soft_reset;
1246 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1247 WREG32(mmGRBM_SOFT_RESET, tmp);
1248 tmp = RREG32(mmGRBM_SOFT_RESET);
1249
1250 udelay(50);
1251
1252 tmp &= ~grbm_soft_reset;
1253 WREG32(mmGRBM_SOFT_RESET, tmp);
1254 tmp = RREG32(mmGRBM_SOFT_RESET);
1255 }
1256
1257 if (srbm_soft_reset) {
1258 tmp = RREG32(mmSRBM_SOFT_RESET);
1259 tmp |= srbm_soft_reset;
1260 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1261 WREG32(mmSRBM_SOFT_RESET, tmp);
1262 tmp = RREG32(mmSRBM_SOFT_RESET);
1263
1264 udelay(50);
1265
1266 tmp &= ~srbm_soft_reset;
1267 WREG32(mmSRBM_SOFT_RESET, tmp);
1268 tmp = RREG32(mmSRBM_SOFT_RESET);
1269 }
1270
1271
1272 udelay(50);
1273
1274 gmc_v7_0_mc_resume(adev, &save);
1275 udelay(50);
1276
1277 cik_print_gpu_status_regs(adev);
1278}
1279
1280struct kv_reset_save_regs {
1281 u32 gmcon_reng_execute;
1282 u32 gmcon_misc;
1283 u32 gmcon_misc3;
1284};
1285
1286static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1287 struct kv_reset_save_regs *save)
1288{
1289 save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1290 save->gmcon_misc = RREG32(mmGMCON_MISC);
1291 save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1292
1293 WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1294 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1295 WREG32(mmGMCON_MISC, save->gmcon_misc &
1296 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1297 GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1298}
1299
1300static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1301 struct kv_reset_save_regs *save)
1302{
1303 int i;
1304
1305 WREG32(mmGMCON_PGFSM_WRITE, 0);
1306 WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1307
1308 for (i = 0; i < 5; i++)
1309 WREG32(mmGMCON_PGFSM_WRITE, 0);
1310
1311 WREG32(mmGMCON_PGFSM_WRITE, 0);
1312 WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1313
1314 for (i = 0; i < 5; i++)
1315 WREG32(mmGMCON_PGFSM_WRITE, 0);
1316
1317 WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1318 WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1319
1320 for (i = 0; i < 5; i++)
1321 WREG32(mmGMCON_PGFSM_WRITE, 0);
1322
1323 WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1324 WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1325
1326 for (i = 0; i < 5; i++)
1327 WREG32(mmGMCON_PGFSM_WRITE, 0);
1328
1329 WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1330 WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1331
1332 for (i = 0; i < 5; i++)
1333 WREG32(mmGMCON_PGFSM_WRITE, 0);
1334
1335 WREG32(mmGMCON_PGFSM_WRITE, 0);
1336 WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1337
1338 for (i = 0; i < 5; i++)
1339 WREG32(mmGMCON_PGFSM_WRITE, 0);
1340
1341 WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1342 WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1343
1344 for (i = 0; i < 5; i++)
1345 WREG32(mmGMCON_PGFSM_WRITE, 0);
1346
1347 WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1348 WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1349
1350 for (i = 0; i < 5; i++)
1351 WREG32(mmGMCON_PGFSM_WRITE, 0);
1352
1353 WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1354 WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1355
1356 for (i = 0; i < 5; i++)
1357 WREG32(mmGMCON_PGFSM_WRITE, 0);
1358
1359 WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1360 WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1361
1362 for (i = 0; i < 5; i++)
1363 WREG32(mmGMCON_PGFSM_WRITE, 0);
1364
1365 WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1366 WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1367
1368 WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1369 WREG32(mmGMCON_MISC, save->gmcon_misc);
1370 WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1371}
1372
1373static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1374{
1375 struct amdgpu_mode_mc_save save;
1376 struct kv_reset_save_regs kv_save = { 0 };
1377 u32 tmp, i;
1378
1379 dev_info(adev->dev, "GPU pci config reset\n");
1380
1381
1382
1383
1384
1385
1386 WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK |
1387 CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
1388
1389
1390 WREG32(mmCP_MEC_CNTL,
1391 CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
1392
1393
1394 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1395 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1396 WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1397
1398 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1399 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1400 WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1401
1402
1403
1404 gfx_v7_0_rlc_stop(adev);
1405
1406 udelay(50);
1407
1408
1409 gmc_v7_0_mc_stop(adev, &save);
1410 if (amdgpu_asic_wait_for_mc_idle(adev)) {
1411 dev_warn(adev->dev, "Wait for MC idle timed out !\n");
1412 }
1413
1414 if (adev->flags & AMD_IS_APU)
1415 kv_save_regs_for_reset(adev, &kv_save);
1416
1417
1418 pci_clear_master(adev->pdev);
1419
1420 amdgpu_pci_config_reset(adev);
1421
1422 udelay(100);
1423
1424
1425 for (i = 0; i < adev->usec_timeout; i++) {
1426 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
1427 break;
1428 udelay(1);
1429 }
1430
1431
1432 if (adev->flags & AMD_IS_APU)
1433 kv_restore_regs_for_reset(adev, &kv_save);
1434}
1435
1436static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1437{
1438 u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1439
1440 if (hung)
1441 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1442 else
1443 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1444
1445 WREG32(mmBIOS_SCRATCH_3, tmp);
1446}
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457static int cik_asic_reset(struct amdgpu_device *adev)
1458{
1459 u32 reset_mask;
1460
1461 reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1462
1463 if (reset_mask)
1464 cik_set_bios_scratch_engine_hung(adev, true);
1465
1466
1467 cik_gpu_soft_reset(adev, reset_mask);
1468
1469 reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1470
1471
1472 if (reset_mask && amdgpu_hard_reset)
1473 cik_gpu_pci_config_reset(adev);
1474
1475 reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1476
1477 if (!reset_mask)
1478 cik_set_bios_scratch_engine_hung(adev, false);
1479
1480 return 0;
1481}
1482
1483static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1484 u32 cntl_reg, u32 status_reg)
1485{
1486 int r, i;
1487 struct atom_clock_dividers dividers;
1488 uint32_t tmp;
1489
1490 r = amdgpu_atombios_get_clock_dividers(adev,
1491 COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1492 clock, false, ÷rs);
1493 if (r)
1494 return r;
1495
1496 tmp = RREG32_SMC(cntl_reg);
1497 tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1498 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1499 tmp |= dividers.post_divider;
1500 WREG32_SMC(cntl_reg, tmp);
1501
1502 for (i = 0; i < 100; i++) {
1503 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1504 break;
1505 mdelay(10);
1506 }
1507 if (i == 100)
1508 return -ETIMEDOUT;
1509
1510 return 0;
1511}
1512
1513static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1514{
1515 int r = 0;
1516
1517 r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1518 if (r)
1519 return r;
1520
1521 r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1522 return r;
1523}
1524
1525static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1526{
1527 int r, i;
1528 struct atom_clock_dividers dividers;
1529 u32 tmp;
1530
1531 r = amdgpu_atombios_get_clock_dividers(adev,
1532 COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1533 ecclk, false, ÷rs);
1534 if (r)
1535 return r;
1536
1537 for (i = 0; i < 100; i++) {
1538 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1539 break;
1540 mdelay(10);
1541 }
1542 if (i == 100)
1543 return -ETIMEDOUT;
1544
1545 tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1546 tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1547 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1548 tmp |= dividers.post_divider;
1549 WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1550
1551 for (i = 0; i < 100; i++) {
1552 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1553 break;
1554 mdelay(10);
1555 }
1556 if (i == 100)
1557 return -ETIMEDOUT;
1558
1559 return 0;
1560}
1561
1562static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1563{
1564 struct pci_dev *root = adev->pdev->bus->self;
1565 int bridge_pos, gpu_pos;
1566 u32 speed_cntl, mask, current_data_rate;
1567 int ret, i;
1568 u16 tmp16;
1569
1570 if (pci_is_root_bus(adev->pdev->bus))
1571 return;
1572
1573 if (amdgpu_pcie_gen2 == 0)
1574 return;
1575
1576 if (adev->flags & AMD_IS_APU)
1577 return;
1578
1579 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
1580 if (ret != 0)
1581 return;
1582
1583 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
1584 return;
1585
1586 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1587 current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1588 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1589 if (mask & DRM_PCIE_SPEED_80) {
1590 if (current_data_rate == 2) {
1591 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1592 return;
1593 }
1594 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1595 } else if (mask & DRM_PCIE_SPEED_50) {
1596 if (current_data_rate == 1) {
1597 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1598 return;
1599 }
1600 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1601 }
1602
1603 bridge_pos = pci_pcie_cap(root);
1604 if (!bridge_pos)
1605 return;
1606
1607 gpu_pos = pci_pcie_cap(adev->pdev);
1608 if (!gpu_pos)
1609 return;
1610
1611 if (mask & DRM_PCIE_SPEED_80) {
1612
1613 if (current_data_rate != 2) {
1614 u16 bridge_cfg, gpu_cfg;
1615 u16 bridge_cfg2, gpu_cfg2;
1616 u32 max_lw, current_lw, tmp;
1617
1618 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1619 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1620
1621 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1622 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1623
1624 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1625 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1626
1627 tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1628 max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1629 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1630 current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1631 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1632
1633 if (current_lw < max_lw) {
1634 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1635 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1636 tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1637 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1638 tmp |= (max_lw <<
1639 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1640 tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1641 PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1642 PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1643 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1644 }
1645 }
1646
1647 for (i = 0; i < 10; i++) {
1648
1649 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1650 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1651 break;
1652
1653 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1654 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1655
1656 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1657 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1658
1659 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1660 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1661 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1662
1663 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1664 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1665 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1666
1667 mdelay(100);
1668
1669
1670 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1671 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1672 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1673 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1674
1675 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1676 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1677 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1678 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1679
1680
1681 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1682 tmp16 &= ~((1 << 4) | (7 << 9));
1683 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1684 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1685
1686 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1687 tmp16 &= ~((1 << 4) | (7 << 9));
1688 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1689 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1690
1691 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1692 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1693 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1694 }
1695 }
1696 }
1697
1698
1699 speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1700 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1701 speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1702 WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1703
1704 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1705 tmp16 &= ~0xf;
1706 if (mask & DRM_PCIE_SPEED_80)
1707 tmp16 |= 3;
1708 else if (mask & DRM_PCIE_SPEED_50)
1709 tmp16 |= 2;
1710 else
1711 tmp16 |= 1;
1712 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1713
1714 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1715 speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1716 WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1717
1718 for (i = 0; i < adev->usec_timeout; i++) {
1719 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1720 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1721 break;
1722 udelay(1);
1723 }
1724}
1725
1726static void cik_program_aspm(struct amdgpu_device *adev)
1727{
1728 u32 data, orig;
1729 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1730 bool disable_clkreq = false;
1731
1732 if (amdgpu_aspm == 0)
1733 return;
1734
1735
1736 if (adev->flags & AMD_IS_APU)
1737 return;
1738
1739 orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1740 data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1741 data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1742 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1743 if (orig != data)
1744 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1745
1746 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1747 data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1748 if (orig != data)
1749 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1750
1751 orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1752 data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1753 if (orig != data)
1754 WREG32_PCIE(ixPCIE_P_CNTL, data);
1755
1756 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1757 data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1758 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1759 data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1760 if (!disable_l0s)
1761 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1762
1763 if (!disable_l1) {
1764 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1765 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1766 if (orig != data)
1767 WREG32_PCIE(ixPCIE_LC_CNTL, data);
1768
1769 if (!disable_plloff_in_l1) {
1770 bool clk_req_support;
1771
1772 orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1773 data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1774 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1775 data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1776 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1777 if (orig != data)
1778 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1779
1780 orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1781 data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1782 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1783 data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1784 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1785 if (orig != data)
1786 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1787
1788 orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1789 data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1790 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1791 data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1792 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1793 if (orig != data)
1794 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1795
1796 orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1797 data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1798 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1799 data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1800 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1801 if (orig != data)
1802 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1803
1804 orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1805 data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1806 data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1807 if (orig != data)
1808 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1809
1810 if (!disable_clkreq) {
1811 struct pci_dev *root = adev->pdev->bus->self;
1812 u32 lnkcap;
1813
1814 clk_req_support = false;
1815 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1816 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1817 clk_req_support = true;
1818 } else {
1819 clk_req_support = false;
1820 }
1821
1822 if (clk_req_support) {
1823 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1824 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1825 PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1826 if (orig != data)
1827 WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1828
1829 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1830 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1831 THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1832 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1833 (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1834 if (orig != data)
1835 WREG32_SMC(ixTHM_CLK_CNTL, data);
1836
1837 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1838 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1839 MISC_CLK_CTRL__ZCLK_SEL_MASK);
1840 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1841 (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1842 if (orig != data)
1843 WREG32_SMC(ixMISC_CLK_CTRL, data);
1844
1845 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1846 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1847 if (orig != data)
1848 WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1849
1850 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1851 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1852 if (orig != data)
1853 WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1854
1855 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1856 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1857 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1858 if (orig != data)
1859 WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1860 }
1861 }
1862 } else {
1863 if (orig != data)
1864 WREG32_PCIE(ixPCIE_LC_CNTL, data);
1865 }
1866
1867 orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1868 data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1869 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1870 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1871 if (orig != data)
1872 WREG32_PCIE(ixPCIE_CNTL2, data);
1873
1874 if (!disable_l0s) {
1875 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1876 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1877 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1878 data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1879 if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1880 (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1881 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1882 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1883 if (orig != data)
1884 WREG32_PCIE(ixPCIE_LC_CNTL, data);
1885 }
1886 }
1887 }
1888}
1889
1890static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1891{
1892 return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1893 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1894}
1895
1896static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1897{
1898
1899 {
1900 .type = AMD_IP_BLOCK_TYPE_COMMON,
1901 .major = 1,
1902 .minor = 0,
1903 .rev = 0,
1904 .funcs = &cik_common_ip_funcs,
1905 },
1906 {
1907 .type = AMD_IP_BLOCK_TYPE_GMC,
1908 .major = 7,
1909 .minor = 0,
1910 .rev = 0,
1911 .funcs = &gmc_v7_0_ip_funcs,
1912 },
1913 {
1914 .type = AMD_IP_BLOCK_TYPE_IH,
1915 .major = 2,
1916 .minor = 0,
1917 .rev = 0,
1918 .funcs = &cik_ih_ip_funcs,
1919 },
1920 {
1921 .type = AMD_IP_BLOCK_TYPE_SMC,
1922 .major = 7,
1923 .minor = 0,
1924 .rev = 0,
1925 .funcs = &ci_dpm_ip_funcs,
1926 },
1927 {
1928 .type = AMD_IP_BLOCK_TYPE_DCE,
1929 .major = 8,
1930 .minor = 2,
1931 .rev = 0,
1932 .funcs = &dce_v8_0_ip_funcs,
1933 },
1934 {
1935 .type = AMD_IP_BLOCK_TYPE_GFX,
1936 .major = 7,
1937 .minor = 2,
1938 .rev = 0,
1939 .funcs = &gfx_v7_0_ip_funcs,
1940 },
1941 {
1942 .type = AMD_IP_BLOCK_TYPE_SDMA,
1943 .major = 2,
1944 .minor = 0,
1945 .rev = 0,
1946 .funcs = &cik_sdma_ip_funcs,
1947 },
1948 {
1949 .type = AMD_IP_BLOCK_TYPE_UVD,
1950 .major = 4,
1951 .minor = 2,
1952 .rev = 0,
1953 .funcs = &uvd_v4_2_ip_funcs,
1954 },
1955 {
1956 .type = AMD_IP_BLOCK_TYPE_VCE,
1957 .major = 2,
1958 .minor = 0,
1959 .rev = 0,
1960 .funcs = &vce_v2_0_ip_funcs,
1961 },
1962};
1963
1964static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1965{
1966
1967 {
1968 .type = AMD_IP_BLOCK_TYPE_COMMON,
1969 .major = 1,
1970 .minor = 0,
1971 .rev = 0,
1972 .funcs = &cik_common_ip_funcs,
1973 },
1974 {
1975 .type = AMD_IP_BLOCK_TYPE_GMC,
1976 .major = 7,
1977 .minor = 0,
1978 .rev = 0,
1979 .funcs = &gmc_v7_0_ip_funcs,
1980 },
1981 {
1982 .type = AMD_IP_BLOCK_TYPE_IH,
1983 .major = 2,
1984 .minor = 0,
1985 .rev = 0,
1986 .funcs = &cik_ih_ip_funcs,
1987 },
1988 {
1989 .type = AMD_IP_BLOCK_TYPE_SMC,
1990 .major = 7,
1991 .minor = 0,
1992 .rev = 0,
1993 .funcs = &ci_dpm_ip_funcs,
1994 },
1995 {
1996 .type = AMD_IP_BLOCK_TYPE_DCE,
1997 .major = 8,
1998 .minor = 5,
1999 .rev = 0,
2000 .funcs = &dce_v8_0_ip_funcs,
2001 },
2002 {
2003 .type = AMD_IP_BLOCK_TYPE_GFX,
2004 .major = 7,
2005 .minor = 3,
2006 .rev = 0,
2007 .funcs = &gfx_v7_0_ip_funcs,
2008 },
2009 {
2010 .type = AMD_IP_BLOCK_TYPE_SDMA,
2011 .major = 2,
2012 .minor = 0,
2013 .rev = 0,
2014 .funcs = &cik_sdma_ip_funcs,
2015 },
2016 {
2017 .type = AMD_IP_BLOCK_TYPE_UVD,
2018 .major = 4,
2019 .minor = 2,
2020 .rev = 0,
2021 .funcs = &uvd_v4_2_ip_funcs,
2022 },
2023 {
2024 .type = AMD_IP_BLOCK_TYPE_VCE,
2025 .major = 2,
2026 .minor = 0,
2027 .rev = 0,
2028 .funcs = &vce_v2_0_ip_funcs,
2029 },
2030};
2031
2032static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
2033{
2034
2035 {
2036 .type = AMD_IP_BLOCK_TYPE_COMMON,
2037 .major = 1,
2038 .minor = 0,
2039 .rev = 0,
2040 .funcs = &cik_common_ip_funcs,
2041 },
2042 {
2043 .type = AMD_IP_BLOCK_TYPE_GMC,
2044 .major = 7,
2045 .minor = 0,
2046 .rev = 0,
2047 .funcs = &gmc_v7_0_ip_funcs,
2048 },
2049 {
2050 .type = AMD_IP_BLOCK_TYPE_IH,
2051 .major = 2,
2052 .minor = 0,
2053 .rev = 0,
2054 .funcs = &cik_ih_ip_funcs,
2055 },
2056 {
2057 .type = AMD_IP_BLOCK_TYPE_SMC,
2058 .major = 7,
2059 .minor = 0,
2060 .rev = 0,
2061 .funcs = &kv_dpm_ip_funcs,
2062 },
2063 {
2064 .type = AMD_IP_BLOCK_TYPE_DCE,
2065 .major = 8,
2066 .minor = 3,
2067 .rev = 0,
2068 .funcs = &dce_v8_0_ip_funcs,
2069 },
2070 {
2071 .type = AMD_IP_BLOCK_TYPE_GFX,
2072 .major = 7,
2073 .minor = 2,
2074 .rev = 0,
2075 .funcs = &gfx_v7_0_ip_funcs,
2076 },
2077 {
2078 .type = AMD_IP_BLOCK_TYPE_SDMA,
2079 .major = 2,
2080 .minor = 0,
2081 .rev = 0,
2082 .funcs = &cik_sdma_ip_funcs,
2083 },
2084 {
2085 .type = AMD_IP_BLOCK_TYPE_UVD,
2086 .major = 4,
2087 .minor = 2,
2088 .rev = 0,
2089 .funcs = &uvd_v4_2_ip_funcs,
2090 },
2091 {
2092 .type = AMD_IP_BLOCK_TYPE_VCE,
2093 .major = 2,
2094 .minor = 0,
2095 .rev = 0,
2096 .funcs = &vce_v2_0_ip_funcs,
2097 },
2098};
2099
2100static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
2101{
2102
2103 {
2104 .type = AMD_IP_BLOCK_TYPE_COMMON,
2105 .major = 1,
2106 .minor = 0,
2107 .rev = 0,
2108 .funcs = &cik_common_ip_funcs,
2109 },
2110 {
2111 .type = AMD_IP_BLOCK_TYPE_GMC,
2112 .major = 7,
2113 .minor = 0,
2114 .rev = 0,
2115 .funcs = &gmc_v7_0_ip_funcs,
2116 },
2117 {
2118 .type = AMD_IP_BLOCK_TYPE_IH,
2119 .major = 2,
2120 .minor = 0,
2121 .rev = 0,
2122 .funcs = &cik_ih_ip_funcs,
2123 },
2124 {
2125 .type = AMD_IP_BLOCK_TYPE_SMC,
2126 .major = 7,
2127 .minor = 0,
2128 .rev = 0,
2129 .funcs = &kv_dpm_ip_funcs,
2130 },
2131 {
2132 .type = AMD_IP_BLOCK_TYPE_DCE,
2133 .major = 8,
2134 .minor = 3,
2135 .rev = 0,
2136 .funcs = &dce_v8_0_ip_funcs,
2137 },
2138 {
2139 .type = AMD_IP_BLOCK_TYPE_GFX,
2140 .major = 7,
2141 .minor = 2,
2142 .rev = 0,
2143 .funcs = &gfx_v7_0_ip_funcs,
2144 },
2145 {
2146 .type = AMD_IP_BLOCK_TYPE_SDMA,
2147 .major = 2,
2148 .minor = 0,
2149 .rev = 0,
2150 .funcs = &cik_sdma_ip_funcs,
2151 },
2152 {
2153 .type = AMD_IP_BLOCK_TYPE_UVD,
2154 .major = 4,
2155 .minor = 2,
2156 .rev = 0,
2157 .funcs = &uvd_v4_2_ip_funcs,
2158 },
2159 {
2160 .type = AMD_IP_BLOCK_TYPE_VCE,
2161 .major = 2,
2162 .minor = 0,
2163 .rev = 0,
2164 .funcs = &vce_v2_0_ip_funcs,
2165 },
2166};
2167
2168static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
2169{
2170
2171 {
2172 .type = AMD_IP_BLOCK_TYPE_COMMON,
2173 .major = 1,
2174 .minor = 0,
2175 .rev = 0,
2176 .funcs = &cik_common_ip_funcs,
2177 },
2178 {
2179 .type = AMD_IP_BLOCK_TYPE_GMC,
2180 .major = 7,
2181 .minor = 0,
2182 .rev = 0,
2183 .funcs = &gmc_v7_0_ip_funcs,
2184 },
2185 {
2186 .type = AMD_IP_BLOCK_TYPE_IH,
2187 .major = 2,
2188 .minor = 0,
2189 .rev = 0,
2190 .funcs = &cik_ih_ip_funcs,
2191 },
2192 {
2193 .type = AMD_IP_BLOCK_TYPE_SMC,
2194 .major = 7,
2195 .minor = 0,
2196 .rev = 0,
2197 .funcs = &kv_dpm_ip_funcs,
2198 },
2199 {
2200 .type = AMD_IP_BLOCK_TYPE_DCE,
2201 .major = 8,
2202 .minor = 1,
2203 .rev = 0,
2204 .funcs = &dce_v8_0_ip_funcs,
2205 },
2206 {
2207 .type = AMD_IP_BLOCK_TYPE_GFX,
2208 .major = 7,
2209 .minor = 1,
2210 .rev = 0,
2211 .funcs = &gfx_v7_0_ip_funcs,
2212 },
2213 {
2214 .type = AMD_IP_BLOCK_TYPE_SDMA,
2215 .major = 2,
2216 .minor = 0,
2217 .rev = 0,
2218 .funcs = &cik_sdma_ip_funcs,
2219 },
2220 {
2221 .type = AMD_IP_BLOCK_TYPE_UVD,
2222 .major = 4,
2223 .minor = 2,
2224 .rev = 0,
2225 .funcs = &uvd_v4_2_ip_funcs,
2226 },
2227 {
2228 .type = AMD_IP_BLOCK_TYPE_VCE,
2229 .major = 2,
2230 .minor = 0,
2231 .rev = 0,
2232 .funcs = &vce_v2_0_ip_funcs,
2233 },
2234};
2235
2236int cik_set_ip_blocks(struct amdgpu_device *adev)
2237{
2238 switch (adev->asic_type) {
2239 case CHIP_BONAIRE:
2240 adev->ip_blocks = bonaire_ip_blocks;
2241 adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
2242 break;
2243 case CHIP_HAWAII:
2244 adev->ip_blocks = hawaii_ip_blocks;
2245 adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
2246 break;
2247 case CHIP_KAVERI:
2248 adev->ip_blocks = kaveri_ip_blocks;
2249 adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
2250 break;
2251 case CHIP_KABINI:
2252 adev->ip_blocks = kabini_ip_blocks;
2253 adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
2254 break;
2255 case CHIP_MULLINS:
2256 adev->ip_blocks = mullins_ip_blocks;
2257 adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
2258 break;
2259 default:
2260
2261 return -EINVAL;
2262 }
2263
2264 return 0;
2265}
2266
2267static const struct amdgpu_asic_funcs cik_asic_funcs =
2268{
2269 .read_disabled_bios = &cik_read_disabled_bios,
2270 .read_register = &cik_read_register,
2271 .reset = &cik_asic_reset,
2272 .set_vga_state = &cik_vga_set_state,
2273 .get_xclk = &cik_get_xclk,
2274 .set_uvd_clocks = &cik_set_uvd_clocks,
2275 .set_vce_clocks = &cik_set_vce_clocks,
2276 .get_cu_info = &gfx_v7_0_get_cu_info,
2277
2278 .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2279 .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2280};
2281
2282static int cik_common_early_init(void *handle)
2283{
2284 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2285
2286 adev->smc_rreg = &cik_smc_rreg;
2287 adev->smc_wreg = &cik_smc_wreg;
2288 adev->pcie_rreg = &cik_pcie_rreg;
2289 adev->pcie_wreg = &cik_pcie_wreg;
2290 adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2291 adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2292 adev->didt_rreg = &cik_didt_rreg;
2293 adev->didt_wreg = &cik_didt_wreg;
2294
2295 adev->asic_funcs = &cik_asic_funcs;
2296
2297 adev->has_uvd = true;
2298
2299 adev->rev_id = cik_get_rev_id(adev);
2300 adev->external_rev_id = 0xFF;
2301 switch (adev->asic_type) {
2302 case CHIP_BONAIRE:
2303 adev->cg_flags =
2304 AMDGPU_CG_SUPPORT_GFX_MGCG |
2305 AMDGPU_CG_SUPPORT_GFX_MGLS |
2306
2307 AMDGPU_CG_SUPPORT_GFX_CGLS |
2308 AMDGPU_CG_SUPPORT_GFX_CGTS |
2309 AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2310 AMDGPU_CG_SUPPORT_GFX_CP_LS |
2311 AMDGPU_CG_SUPPORT_MC_LS |
2312 AMDGPU_CG_SUPPORT_MC_MGCG |
2313 AMDGPU_CG_SUPPORT_SDMA_MGCG |
2314 AMDGPU_CG_SUPPORT_SDMA_LS |
2315 AMDGPU_CG_SUPPORT_BIF_LS |
2316 AMDGPU_CG_SUPPORT_VCE_MGCG |
2317 AMDGPU_CG_SUPPORT_UVD_MGCG |
2318 AMDGPU_CG_SUPPORT_HDP_LS |
2319 AMDGPU_CG_SUPPORT_HDP_MGCG;
2320 adev->pg_flags = 0;
2321 adev->external_rev_id = adev->rev_id + 0x14;
2322 break;
2323 case CHIP_HAWAII:
2324 adev->cg_flags =
2325 AMDGPU_CG_SUPPORT_GFX_MGCG |
2326 AMDGPU_CG_SUPPORT_GFX_MGLS |
2327
2328 AMDGPU_CG_SUPPORT_GFX_CGLS |
2329 AMDGPU_CG_SUPPORT_GFX_CGTS |
2330 AMDGPU_CG_SUPPORT_GFX_CP_LS |
2331 AMDGPU_CG_SUPPORT_MC_LS |
2332 AMDGPU_CG_SUPPORT_MC_MGCG |
2333 AMDGPU_CG_SUPPORT_SDMA_MGCG |
2334 AMDGPU_CG_SUPPORT_SDMA_LS |
2335 AMDGPU_CG_SUPPORT_BIF_LS |
2336 AMDGPU_CG_SUPPORT_VCE_MGCG |
2337 AMDGPU_CG_SUPPORT_UVD_MGCG |
2338 AMDGPU_CG_SUPPORT_HDP_LS |
2339 AMDGPU_CG_SUPPORT_HDP_MGCG;
2340 adev->pg_flags = 0;
2341 adev->external_rev_id = 0x28;
2342 break;
2343 case CHIP_KAVERI:
2344 adev->cg_flags =
2345 AMDGPU_CG_SUPPORT_GFX_MGCG |
2346 AMDGPU_CG_SUPPORT_GFX_MGLS |
2347
2348 AMDGPU_CG_SUPPORT_GFX_CGLS |
2349 AMDGPU_CG_SUPPORT_GFX_CGTS |
2350 AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2351 AMDGPU_CG_SUPPORT_GFX_CP_LS |
2352 AMDGPU_CG_SUPPORT_SDMA_MGCG |
2353 AMDGPU_CG_SUPPORT_SDMA_LS |
2354 AMDGPU_CG_SUPPORT_BIF_LS |
2355 AMDGPU_CG_SUPPORT_VCE_MGCG |
2356 AMDGPU_CG_SUPPORT_UVD_MGCG |
2357 AMDGPU_CG_SUPPORT_HDP_LS |
2358 AMDGPU_CG_SUPPORT_HDP_MGCG;
2359 adev->pg_flags =
2360
2361
2362
2363 AMDGPU_PG_SUPPORT_UVD |
2364
2365
2366
2367
2368
2369
2370 0;
2371 if (adev->pdev->device == 0x1312 ||
2372 adev->pdev->device == 0x1316 ||
2373 adev->pdev->device == 0x1317)
2374 adev->external_rev_id = 0x41;
2375 else
2376 adev->external_rev_id = 0x1;
2377 break;
2378 case CHIP_KABINI:
2379 case CHIP_MULLINS:
2380 adev->cg_flags =
2381 AMDGPU_CG_SUPPORT_GFX_MGCG |
2382 AMDGPU_CG_SUPPORT_GFX_MGLS |
2383
2384 AMDGPU_CG_SUPPORT_GFX_CGLS |
2385 AMDGPU_CG_SUPPORT_GFX_CGTS |
2386 AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2387 AMDGPU_CG_SUPPORT_GFX_CP_LS |
2388 AMDGPU_CG_SUPPORT_SDMA_MGCG |
2389 AMDGPU_CG_SUPPORT_SDMA_LS |
2390 AMDGPU_CG_SUPPORT_BIF_LS |
2391 AMDGPU_CG_SUPPORT_VCE_MGCG |
2392 AMDGPU_CG_SUPPORT_UVD_MGCG |
2393 AMDGPU_CG_SUPPORT_HDP_LS |
2394 AMDGPU_CG_SUPPORT_HDP_MGCG;
2395 adev->pg_flags =
2396
2397
2398 AMDGPU_PG_SUPPORT_UVD |
2399
2400
2401
2402
2403
2404 0;
2405 if (adev->asic_type == CHIP_KABINI) {
2406 if (adev->rev_id == 0)
2407 adev->external_rev_id = 0x81;
2408 else if (adev->rev_id == 1)
2409 adev->external_rev_id = 0x82;
2410 else if (adev->rev_id == 2)
2411 adev->external_rev_id = 0x85;
2412 } else
2413 adev->external_rev_id = adev->rev_id + 0xa1;
2414 break;
2415 default:
2416
2417 return -EINVAL;
2418 }
2419
2420 return 0;
2421}
2422
2423static int cik_common_sw_init(void *handle)
2424{
2425 return 0;
2426}
2427
2428static int cik_common_sw_fini(void *handle)
2429{
2430 return 0;
2431}
2432
2433static int cik_common_hw_init(void *handle)
2434{
2435 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2436
2437
2438 cik_init_golden_registers(adev);
2439
2440 cik_pcie_gen3_enable(adev);
2441
2442 cik_program_aspm(adev);
2443
2444 return 0;
2445}
2446
2447static int cik_common_hw_fini(void *handle)
2448{
2449 return 0;
2450}
2451
2452static int cik_common_suspend(void *handle)
2453{
2454 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2455
2456 amdgpu_amdkfd_suspend(adev);
2457
2458 return cik_common_hw_fini(adev);
2459}
2460
2461static int cik_common_resume(void *handle)
2462{
2463 int r;
2464 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2465
2466 r = cik_common_hw_init(adev);
2467 if (r)
2468 return r;
2469
2470 return amdgpu_amdkfd_resume(adev);
2471}
2472
2473static bool cik_common_is_idle(void *handle)
2474{
2475 return true;
2476}
2477
2478static int cik_common_wait_for_idle(void *handle)
2479{
2480 return 0;
2481}
2482
2483static void cik_common_print_status(void *handle)
2484{
2485
2486}
2487
2488static int cik_common_soft_reset(void *handle)
2489{
2490
2491 return 0;
2492}
2493
2494static int cik_common_set_clockgating_state(void *handle,
2495 enum amd_clockgating_state state)
2496{
2497 return 0;
2498}
2499
2500static int cik_common_set_powergating_state(void *handle,
2501 enum amd_powergating_state state)
2502{
2503 return 0;
2504}
2505
2506const struct amd_ip_funcs cik_common_ip_funcs = {
2507 .early_init = cik_common_early_init,
2508 .late_init = NULL,
2509 .sw_init = cik_common_sw_init,
2510 .sw_fini = cik_common_sw_fini,
2511 .hw_init = cik_common_hw_init,
2512 .hw_fini = cik_common_hw_fini,
2513 .suspend = cik_common_suspend,
2514 .resume = cik_common_resume,
2515 .is_idle = cik_common_is_idle,
2516 .wait_for_idle = cik_common_wait_for_idle,
2517 .soft_reset = cik_common_soft_reset,
2518 .print_status = cik_common_print_status,
2519 .set_clockgating_state = cik_common_set_clockgating_state,
2520 .set_powergating_state = cik_common_set_powergating_state,
2521};
2522
