1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/delay.h>
25#include <linux/kernel.h>
26#include <linux/firmware.h>
27#include <linux/module.h>
28#include <linux/pci.h>
29
30#include "amdgpu.h"
31#include "amdgpu_gfx.h"
32#include "soc15.h"
33#include "soc15d.h"
34#include "amdgpu_atomfirmware.h"
35#include "amdgpu_pm.h"
36
37#include "gc/gc_9_0_offset.h"
38#include "gc/gc_9_0_sh_mask.h"
39
40#include "vega10_enum.h"
41
42#include "soc15_common.h"
43#include "clearstate_gfx9.h"
44#include "v9_structs.h"
45
46#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48#include "amdgpu_ras.h"
49
50#include "gfx_v9_4.h"
51#include "gfx_v9_0.h"
52
53#include "asic_reg/pwr/pwr_10_0_offset.h"
54#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
55#include "asic_reg/gc/gc_9_0_default.h"
56
57#define GFX9_NUM_GFX_RINGS 1
58#define GFX9_MEC_HPD_SIZE 4096
59#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
60#define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
61
62#define mmGCEA_PROBE_MAP 0x070c
63#define mmGCEA_PROBE_MAP_BASE_IDX 0
64
65MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
66MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
67MODULE_FIRMWARE("amdgpu/vega10_me.bin");
68MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
69MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
70MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
71
72MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
73MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
74MODULE_FIRMWARE("amdgpu/vega12_me.bin");
75MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
76MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
77MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
78
79MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
80MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
81MODULE_FIRMWARE("amdgpu/vega20_me.bin");
82MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
83MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
84MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
85
86MODULE_FIRMWARE("amdgpu/raven_ce.bin");
87MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
88MODULE_FIRMWARE("amdgpu/raven_me.bin");
89MODULE_FIRMWARE("amdgpu/raven_mec.bin");
90MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
91MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
92
93MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
94MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
95MODULE_FIRMWARE("amdgpu/picasso_me.bin");
96MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
97MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
98MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
99MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
100
101MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
102MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
103MODULE_FIRMWARE("amdgpu/raven2_me.bin");
104MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
105MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
106MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
107MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
108
109MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
110MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin");
111MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
112
113MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
114MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
115MODULE_FIRMWARE("amdgpu/renoir_me.bin");
116MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
117MODULE_FIRMWARE("amdgpu/renoir_mec2.bin");
118MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
119
120MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
121MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
122MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
123MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
124MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
125MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
126
127#define mmTCP_CHAN_STEER_0_ARCT 0x0b03
128#define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
129#define mmTCP_CHAN_STEER_1_ARCT 0x0b04
130#define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
131#define mmTCP_CHAN_STEER_2_ARCT 0x0b09
132#define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
133#define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
134#define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
135#define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
136#define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
137#define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
138#define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
139
140enum ta_ras_gfx_subblock {
141
142 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
143 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
144 TA_RAS_BLOCK__GFX_CPC_UCODE,
145 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
146 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
147 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
148 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
151 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
152
153 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
154 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
155 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
156 TA_RAS_BLOCK__GFX_CPF_TAG,
157 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
158
159 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
160 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
161 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
162 TA_RAS_BLOCK__GFX_CPG_TAG,
163 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
164
165 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
166 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
167 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
168 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
169 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
170 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
171 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
172
173 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
174
175 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
176 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
177 TA_RAS_BLOCK__GFX_SQ_LDS_D,
178 TA_RAS_BLOCK__GFX_SQ_LDS_I,
179 TA_RAS_BLOCK__GFX_SQ_VGPR,
180 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
181
182 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
183
184 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
185 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
186 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
187 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
188 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
189 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
190 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
191 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
192 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
193 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
195
196 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
197 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
198 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
199 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
201 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
202 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
203 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
204 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
207 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
209
210 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
211 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
212 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
213 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
215 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
217 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
218 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
221 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
223 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
224
225 TA_RAS_BLOCK__GFX_TA_INDEX_START,
226 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
227 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
228 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
229 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
230 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
231 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
232
233 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
234 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
235 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
236 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
237
238 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
239
240 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
241 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
242 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
243 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
245 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
246 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
247 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
248 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
249 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
250
251 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
252 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
253 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
254 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
255 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
256
257 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
258 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
259 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
260 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
261 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
262 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
263 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
264 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
265 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
266 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
267 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
268
269 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
270 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
271 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
272 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
274
275 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
276 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
277 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
278 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
279 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
280 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
281 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
282
283 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
284
285 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
286 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
287 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
288 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
289 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
290 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
291 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
292 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
293 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
294
295 TA_RAS_BLOCK__GFX_TD_INDEX_START,
296 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
297 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
298 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
299 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
300
301 TA_RAS_BLOCK__GFX_EA_INDEX_START,
302
303 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
304 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
305 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
306 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
307 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
308 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
309 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
310 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
311 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
312 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
313
314 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
315 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
316 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
317 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
318 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
319 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
320 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
321 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
322 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
323
324 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
325 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
326 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
327 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
328 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
329 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
330 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
331
332 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
333
334 TA_RAS_BLOCK__UTC_VML2_WALKER,
335
336 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
337
338 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
339 TA_RAS_BLOCK__GFX_MAX
340};
341
342struct ras_gfx_subblock {
343 unsigned char *name;
344 int ta_subblock;
345 int hw_supported_error_type;
346 int sw_supported_error_type;
347};
348
349#define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
350 [AMDGPU_RAS_BLOCK__##subblock] = { \
351 #subblock, \
352 TA_RAS_BLOCK__##subblock, \
353 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
354 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
355 }
356
357static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
358 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
359 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
360 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
361 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
362 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
366 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
367 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
368 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
369 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
370 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
371 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
372 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
373 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
374 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
375 0),
376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
377 0),
378 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
379 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
380 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
381 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
382 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
384 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
386 0, 0),
387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
388 0),
389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
390 0, 0),
391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
392 0),
393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
394 0, 0),
395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
396 0),
397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
398 1),
399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
400 0, 0, 0),
401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
402 0),
403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
404 0),
405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
406 0),
407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
408 0),
409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
410 0),
411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
412 0, 0),
413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
414 0),
415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
416 0),
417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
418 0, 0, 0),
419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
420 0),
421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
422 0),
423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
424 0),
425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
426 0),
427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
428 0),
429 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
430 0, 0),
431 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
432 0),
433 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
434 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
435 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
438 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
439 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
440 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
442 1),
443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
444 1),
445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
446 1),
447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
448 0),
449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
450 0),
451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
463 0),
464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
466 0),
467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
468 0, 0),
469 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
470 0),
471 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
472 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
473 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
474 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
479 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
480 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
481 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
501 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
502 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
503 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
504 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
505};
506
507static const struct soc15_reg_golden golden_settings_gc_9_0[] =
508{
509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
529};
530
531static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
532{
533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
551};
552
553static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
554{
555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
566};
567
568static const struct soc15_reg_golden golden_settings_gc_9_1[] =
569{
570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
594};
595
596static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
597{
598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
605};
606
607static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
608{
609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
628};
629
630static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
631{
632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
644};
645
646static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
647{
648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
651};
652
653static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
654{
655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
671};
672
673static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
674{
675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
688};
689
690static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
691{
692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
703};
704
705static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
706 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
707 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
708};
709
710static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
711{
712 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
713 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
714 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
715 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
716 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
717 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
718 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
719 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
720};
721
722static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
723{
724 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
725 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
726 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
727 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
728 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
729 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
730 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
731 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
732};
733
734static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
735{
736 static void *scratch_reg0;
737 static void *scratch_reg1;
738 static void *scratch_reg2;
739 static void *scratch_reg3;
740 static void *spare_int;
741 static uint32_t grbm_cntl;
742 static uint32_t grbm_idx;
743
744 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
745 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
746 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
747 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
748 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
749
750 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
751 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
752
753 if (amdgpu_sriov_runtime(adev)) {
754 pr_err("shouldn't call rlcg write register during runtime\n");
755 return;
756 }
757
758 if (offset == grbm_cntl || offset == grbm_idx) {
759 if (offset == grbm_cntl)
760 writel(v, scratch_reg2);
761 else if (offset == grbm_idx)
762 writel(v, scratch_reg3);
763
764 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
765 } else {
766 uint32_t i = 0;
767 uint32_t retries = 50000;
768
769 writel(v, scratch_reg0);
770 writel(offset | 0x80000000, scratch_reg1);
771 writel(1, spare_int);
772 for (i = 0; i < retries; i++) {
773 u32 tmp;
774
775 tmp = readl(scratch_reg1);
776 if (!(tmp & 0x80000000))
777 break;
778
779 udelay(10);
780 }
781 if (i >= retries)
782 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
783 }
784
785}
786
787#define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
788#define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
789#define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
790#define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
791
792static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
793static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
794static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
795static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
796static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
797 struct amdgpu_cu_info *cu_info);
798static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
799static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
800static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
801static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
802 void *ras_error_status);
803static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
804 void *inject_if);
805static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
806
807static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
808 uint64_t queue_mask)
809{
810 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
811 amdgpu_ring_write(kiq_ring,
812 PACKET3_SET_RESOURCES_VMID_MASK(0) |
813
814 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
815 amdgpu_ring_write(kiq_ring,
816 lower_32_bits(queue_mask));
817 amdgpu_ring_write(kiq_ring,
818 upper_32_bits(queue_mask));
819 amdgpu_ring_write(kiq_ring, 0);
820 amdgpu_ring_write(kiq_ring, 0);
821 amdgpu_ring_write(kiq_ring, 0);
822 amdgpu_ring_write(kiq_ring, 0);
823}
824
825static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
826 struct amdgpu_ring *ring)
827{
828 struct amdgpu_device *adev = kiq_ring->adev;
829 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
830 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
831 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
832
833 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
834
835 amdgpu_ring_write(kiq_ring,
836 PACKET3_MAP_QUEUES_QUEUE_SEL(0) |
837 PACKET3_MAP_QUEUES_VMID(0) |
838 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
839 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
840 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
841
842 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
843
844 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
845 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
846
847 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
848 amdgpu_ring_write(kiq_ring,
849 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
850 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
851 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
852 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
853 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
854}
855
856static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
857 struct amdgpu_ring *ring,
858 enum amdgpu_unmap_queues_action action,
859 u64 gpu_addr, u64 seq)
860{
861 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
862
863 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
864 amdgpu_ring_write(kiq_ring,
865 PACKET3_UNMAP_QUEUES_ACTION(action) |
866 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
867 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
868 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
869 amdgpu_ring_write(kiq_ring,
870 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
871
872 if (action == PREEMPT_QUEUES_NO_UNMAP) {
873 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
874 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
875 amdgpu_ring_write(kiq_ring, seq);
876 } else {
877 amdgpu_ring_write(kiq_ring, 0);
878 amdgpu_ring_write(kiq_ring, 0);
879 amdgpu_ring_write(kiq_ring, 0);
880 }
881}
882
883static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
884 struct amdgpu_ring *ring,
885 u64 addr,
886 u64 seq)
887{
888 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
889
890 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
891 amdgpu_ring_write(kiq_ring,
892 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
893 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
894 PACKET3_QUERY_STATUS_COMMAND(2));
895
896 amdgpu_ring_write(kiq_ring,
897 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
898 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
899 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
900 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
901 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
902 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
903}
904
905static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
906 uint16_t pasid, uint32_t flush_type,
907 bool all_hub)
908{
909 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
910 amdgpu_ring_write(kiq_ring,
911 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
912 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
913 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
914 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
915}
916
917static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
918 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
919 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
920 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
921 .kiq_query_status = gfx_v9_0_kiq_query_status,
922 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
923 .set_resources_size = 8,
924 .map_queues_size = 7,
925 .unmap_queues_size = 6,
926 .query_status_size = 7,
927 .invalidate_tlbs_size = 2,
928};
929
930static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
931{
932 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
933}
934
935static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
936{
937 switch (adev->asic_type) {
938 case CHIP_VEGA10:
939 soc15_program_register_sequence(adev,
940 golden_settings_gc_9_0,
941 ARRAY_SIZE(golden_settings_gc_9_0));
942 soc15_program_register_sequence(adev,
943 golden_settings_gc_9_0_vg10,
944 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
945 break;
946 case CHIP_VEGA12:
947 soc15_program_register_sequence(adev,
948 golden_settings_gc_9_2_1,
949 ARRAY_SIZE(golden_settings_gc_9_2_1));
950 soc15_program_register_sequence(adev,
951 golden_settings_gc_9_2_1_vg12,
952 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
953 break;
954 case CHIP_VEGA20:
955 soc15_program_register_sequence(adev,
956 golden_settings_gc_9_0,
957 ARRAY_SIZE(golden_settings_gc_9_0));
958 soc15_program_register_sequence(adev,
959 golden_settings_gc_9_0_vg20,
960 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
961 break;
962 case CHIP_ARCTURUS:
963 soc15_program_register_sequence(adev,
964 golden_settings_gc_9_4_1_arct,
965 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
966 break;
967 case CHIP_RAVEN:
968 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
969 ARRAY_SIZE(golden_settings_gc_9_1));
970 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
971 soc15_program_register_sequence(adev,
972 golden_settings_gc_9_1_rv2,
973 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
974 else
975 soc15_program_register_sequence(adev,
976 golden_settings_gc_9_1_rv1,
977 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
978 break;
979 case CHIP_RENOIR:
980 soc15_program_register_sequence(adev,
981 golden_settings_gc_9_1_rn,
982 ARRAY_SIZE(golden_settings_gc_9_1_rn));
983 return;
984 default:
985 break;
986 }
987
988 if (adev->asic_type != CHIP_ARCTURUS)
989 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
990 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
991}
992
993static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
994{
995 adev->gfx.scratch.num_reg = 8;
996 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
997 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
998}
999
1000static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1001 bool wc, uint32_t reg, uint32_t val)
1002{
1003 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1004 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1005 WRITE_DATA_DST_SEL(0) |
1006 (wc ? WR_CONFIRM : 0));
1007 amdgpu_ring_write(ring, reg);
1008 amdgpu_ring_write(ring, 0);
1009 amdgpu_ring_write(ring, val);
1010}
1011
1012static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1013 int mem_space, int opt, uint32_t addr0,
1014 uint32_t addr1, uint32_t ref, uint32_t mask,
1015 uint32_t inv)
1016{
1017 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1018 amdgpu_ring_write(ring,
1019
1020 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1021 WAIT_REG_MEM_OPERATION(opt) |
1022 WAIT_REG_MEM_FUNCTION(3) |
1023 WAIT_REG_MEM_ENGINE(eng_sel)));
1024
1025 if (mem_space)
1026 BUG_ON(addr0 & 0x3);
1027 amdgpu_ring_write(ring, addr0);
1028 amdgpu_ring_write(ring, addr1);
1029 amdgpu_ring_write(ring, ref);
1030 amdgpu_ring_write(ring, mask);
1031 amdgpu_ring_write(ring, inv);
1032}
1033
1034static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1035{
1036 struct amdgpu_device *adev = ring->adev;
1037 uint32_t scratch;
1038 uint32_t tmp = 0;
1039 unsigned i;
1040 int r;
1041
1042 r = amdgpu_gfx_scratch_get(adev, &scratch);
1043 if (r)
1044 return r;
1045
1046 WREG32(scratch, 0xCAFEDEAD);
1047 r = amdgpu_ring_alloc(ring, 3);
1048 if (r)
1049 goto error_free_scratch;
1050
1051 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1052 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1053 amdgpu_ring_write(ring, 0xDEADBEEF);
1054 amdgpu_ring_commit(ring);
1055
1056 for (i = 0; i < adev->usec_timeout; i++) {
1057 tmp = RREG32(scratch);
1058 if (tmp == 0xDEADBEEF)
1059 break;
1060 udelay(1);
1061 }
1062
1063 if (i >= adev->usec_timeout)
1064 r = -ETIMEDOUT;
1065
1066error_free_scratch:
1067 amdgpu_gfx_scratch_free(adev, scratch);
1068 return r;
1069}
1070
1071static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1072{
1073 struct amdgpu_device *adev = ring->adev;
1074 struct amdgpu_ib ib;
1075 struct dma_fence *f = NULL;
1076
1077 unsigned index;
1078 uint64_t gpu_addr;
1079 uint32_t tmp;
1080 long r;
1081
1082 r = amdgpu_device_wb_get(adev, &index);
1083 if (r)
1084 return r;
1085
1086 gpu_addr = adev->wb.gpu_addr + (index * 4);
1087 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1088 memset(&ib, 0, sizeof(ib));
1089 r = amdgpu_ib_get(adev, NULL, 16,
1090 AMDGPU_IB_POOL_DIRECT, &ib);
1091 if (r)
1092 goto err1;
1093
1094 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1095 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1096 ib.ptr[2] = lower_32_bits(gpu_addr);
1097 ib.ptr[3] = upper_32_bits(gpu_addr);
1098 ib.ptr[4] = 0xDEADBEEF;
1099 ib.length_dw = 5;
1100
1101 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1102 if (r)
1103 goto err2;
1104
1105 r = dma_fence_wait_timeout(f, false, timeout);
1106 if (r == 0) {
1107 r = -ETIMEDOUT;
1108 goto err2;
1109 } else if (r < 0) {
1110 goto err2;
1111 }
1112
1113 tmp = adev->wb.wb[index];
1114 if (tmp == 0xDEADBEEF)
1115 r = 0;
1116 else
1117 r = -EINVAL;
1118
1119err2:
1120 amdgpu_ib_free(adev, &ib, NULL);
1121 dma_fence_put(f);
1122err1:
1123 amdgpu_device_wb_free(adev, index);
1124 return r;
1125}
1126
1127
1128static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1129{
1130 release_firmware(adev->gfx.pfp_fw);
1131 adev->gfx.pfp_fw = NULL;
1132 release_firmware(adev->gfx.me_fw);
1133 adev->gfx.me_fw = NULL;
1134 release_firmware(adev->gfx.ce_fw);
1135 adev->gfx.ce_fw = NULL;
1136 release_firmware(adev->gfx.rlc_fw);
1137 adev->gfx.rlc_fw = NULL;
1138 release_firmware(adev->gfx.mec_fw);
1139 adev->gfx.mec_fw = NULL;
1140 release_firmware(adev->gfx.mec2_fw);
1141 adev->gfx.mec2_fw = NULL;
1142
1143 kfree(adev->gfx.rlc.register_list_format);
1144}
1145
1146static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1147{
1148 const struct rlc_firmware_header_v2_1 *rlc_hdr;
1149
1150 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1151 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1152 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1153 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1154 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1155 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1156 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1157 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1158 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1159 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1160 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1161 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1162 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1163 adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1164 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1165}
1166
1167static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1168{
1169 adev->gfx.me_fw_write_wait = false;
1170 adev->gfx.mec_fw_write_wait = false;
1171
1172 if ((adev->asic_type != CHIP_ARCTURUS) &&
1173 ((adev->gfx.mec_fw_version < 0x000001a5) ||
1174 (adev->gfx.mec_feature_version < 46) ||
1175 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1176 (adev->gfx.pfp_feature_version < 46)))
1177 DRM_WARN_ONCE("CP firmware version too old, please update!");
1178
1179 switch (adev->asic_type) {
1180 case CHIP_VEGA10:
1181 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1182 (adev->gfx.me_feature_version >= 42) &&
1183 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1184 (adev->gfx.pfp_feature_version >= 42))
1185 adev->gfx.me_fw_write_wait = true;
1186
1187 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1188 (adev->gfx.mec_feature_version >= 42))
1189 adev->gfx.mec_fw_write_wait = true;
1190 break;
1191 case CHIP_VEGA12:
1192 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1193 (adev->gfx.me_feature_version >= 44) &&
1194 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1195 (adev->gfx.pfp_feature_version >= 44))
1196 adev->gfx.me_fw_write_wait = true;
1197
1198 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1199 (adev->gfx.mec_feature_version >= 44))
1200 adev->gfx.mec_fw_write_wait = true;
1201 break;
1202 case CHIP_VEGA20:
1203 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1204 (adev->gfx.me_feature_version >= 44) &&
1205 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1206 (adev->gfx.pfp_feature_version >= 44))
1207 adev->gfx.me_fw_write_wait = true;
1208
1209 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1210 (adev->gfx.mec_feature_version >= 44))
1211 adev->gfx.mec_fw_write_wait = true;
1212 break;
1213 case CHIP_RAVEN:
1214 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1215 (adev->gfx.me_feature_version >= 42) &&
1216 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1217 (adev->gfx.pfp_feature_version >= 42))
1218 adev->gfx.me_fw_write_wait = true;
1219
1220 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1221 (adev->gfx.mec_feature_version >= 42))
1222 adev->gfx.mec_fw_write_wait = true;
1223 break;
1224 default:
1225 adev->gfx.me_fw_write_wait = true;
1226 adev->gfx.mec_fw_write_wait = true;
1227 break;
1228 }
1229}
1230
1231struct amdgpu_gfxoff_quirk {
1232 u16 chip_vendor;
1233 u16 chip_device;
1234 u16 subsys_vendor;
1235 u16 subsys_device;
1236 u8 revision;
1237};
1238
1239static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1240
1241 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1242
1243 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1244
1245 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1246 { 0, 0, 0, 0, 0 },
1247};
1248
1249static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1250{
1251 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1252
1253 while (p && p->chip_device != 0) {
1254 if (pdev->vendor == p->chip_vendor &&
1255 pdev->device == p->chip_device &&
1256 pdev->subsystem_vendor == p->subsys_vendor &&
1257 pdev->subsystem_device == p->subsys_device &&
1258 pdev->revision == p->revision) {
1259 return true;
1260 }
1261 ++p;
1262 }
1263 return false;
1264}
1265
1266static bool is_raven_kicker(struct amdgpu_device *adev)
1267{
1268 if (adev->pm.fw_version >= 0x41e2b)
1269 return true;
1270 else
1271 return false;
1272}
1273
1274static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1275{
1276 if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1277 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1278
1279 switch (adev->asic_type) {
1280 case CHIP_VEGA10:
1281 case CHIP_VEGA12:
1282 case CHIP_VEGA20:
1283 break;
1284 case CHIP_RAVEN:
1285 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1286 (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1287 ((!is_raven_kicker(adev) &&
1288 adev->gfx.rlc_fw_version < 531) ||
1289 (adev->gfx.rlc_feature_version < 1) ||
1290 !adev->gfx.rlc.is_rlc_v2_1))
1291 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1292
1293 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1294 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1295 AMD_PG_SUPPORT_CP |
1296 AMD_PG_SUPPORT_RLC_SMU_HS;
1297 break;
1298 case CHIP_RENOIR:
1299 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1300 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1301 AMD_PG_SUPPORT_CP |
1302 AMD_PG_SUPPORT_RLC_SMU_HS;
1303 break;
1304 default:
1305 break;
1306 }
1307}
1308
1309static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1310 const char *chip_name)
1311{
1312 char fw_name[30];
1313 int err;
1314 struct amdgpu_firmware_info *info = NULL;
1315 const struct common_firmware_header *header = NULL;
1316 const struct gfx_firmware_header_v1_0 *cp_hdr;
1317
1318 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1319 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1320 if (err)
1321 goto out;
1322 err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1323 if (err)
1324 goto out;
1325 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1326 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1327 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1328
1329 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1330 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1331 if (err)
1332 goto out;
1333 err = amdgpu_ucode_validate(adev->gfx.me_fw);
1334 if (err)
1335 goto out;
1336 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1337 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1338 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1339
1340 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1341 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1342 if (err)
1343 goto out;
1344 err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1345 if (err)
1346 goto out;
1347 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1348 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1349 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1350
1351 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1352 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1353 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1354 info->fw = adev->gfx.pfp_fw;
1355 header = (const struct common_firmware_header *)info->fw->data;
1356 adev->firmware.fw_size +=
1357 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1358
1359 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1360 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1361 info->fw = adev->gfx.me_fw;
1362 header = (const struct common_firmware_header *)info->fw->data;
1363 adev->firmware.fw_size +=
1364 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1365
1366 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1367 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1368 info->fw = adev->gfx.ce_fw;
1369 header = (const struct common_firmware_header *)info->fw->data;
1370 adev->firmware.fw_size +=
1371 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1372 }
1373
1374out:
1375 if (err) {
1376 dev_err(adev->dev,
1377 "gfx9: Failed to load firmware \"%s\"\n",
1378 fw_name);
1379 release_firmware(adev->gfx.pfp_fw);
1380 adev->gfx.pfp_fw = NULL;
1381 release_firmware(adev->gfx.me_fw);
1382 adev->gfx.me_fw = NULL;
1383 release_firmware(adev->gfx.ce_fw);
1384 adev->gfx.ce_fw = NULL;
1385 }
1386 return err;
1387}
1388
1389static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1390 const char *chip_name)
1391{
1392 char fw_name[30];
1393 int err;
1394 struct amdgpu_firmware_info *info = NULL;
1395 const struct common_firmware_header *header = NULL;
1396 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1397 unsigned int *tmp = NULL;
1398 unsigned int i = 0;
1399 uint16_t version_major;
1400 uint16_t version_minor;
1401 uint32_t smu_version;
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411 if (!strcmp(chip_name, "picasso") &&
1412 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1413 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1414 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1415 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1416 (smu_version >= 0x41e2b))
1417
1418
1419
1420 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1421 else
1422 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1423 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1424 if (err)
1425 goto out;
1426 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1427 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1428
1429 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1430 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1431 if (version_major == 2 && version_minor == 1)
1432 adev->gfx.rlc.is_rlc_v2_1 = true;
1433
1434 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1435 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1436 adev->gfx.rlc.save_and_restore_offset =
1437 le32_to_cpu(rlc_hdr->save_and_restore_offset);
1438 adev->gfx.rlc.clear_state_descriptor_offset =
1439 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1440 adev->gfx.rlc.avail_scratch_ram_locations =
1441 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1442 adev->gfx.rlc.reg_restore_list_size =
1443 le32_to_cpu(rlc_hdr->reg_restore_list_size);
1444 adev->gfx.rlc.reg_list_format_start =
1445 le32_to_cpu(rlc_hdr->reg_list_format_start);
1446 adev->gfx.rlc.reg_list_format_separate_start =
1447 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1448 adev->gfx.rlc.starting_offsets_start =
1449 le32_to_cpu(rlc_hdr->starting_offsets_start);
1450 adev->gfx.rlc.reg_list_format_size_bytes =
1451 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1452 adev->gfx.rlc.reg_list_size_bytes =
1453 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1454 adev->gfx.rlc.register_list_format =
1455 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1456 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1457 if (!adev->gfx.rlc.register_list_format) {
1458 err = -ENOMEM;
1459 goto out;
1460 }
1461
1462 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1463 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1464 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1465 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1466
1467 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1468
1469 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1470 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1471 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1472 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1473
1474 if (adev->gfx.rlc.is_rlc_v2_1)
1475 gfx_v9_0_init_rlc_ext_microcode(adev);
1476
1477 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1478 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1479 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1480 info->fw = adev->gfx.rlc_fw;
1481 header = (const struct common_firmware_header *)info->fw->data;
1482 adev->firmware.fw_size +=
1483 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1484
1485 if (adev->gfx.rlc.is_rlc_v2_1 &&
1486 adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1487 adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1488 adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1489 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1490 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1491 info->fw = adev->gfx.rlc_fw;
1492 adev->firmware.fw_size +=
1493 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1494
1495 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1496 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1497 info->fw = adev->gfx.rlc_fw;
1498 adev->firmware.fw_size +=
1499 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1500
1501 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1502 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1503 info->fw = adev->gfx.rlc_fw;
1504 adev->firmware.fw_size +=
1505 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1506 }
1507 }
1508
1509out:
1510 if (err) {
1511 dev_err(adev->dev,
1512 "gfx9: Failed to load firmware \"%s\"\n",
1513 fw_name);
1514 release_firmware(adev->gfx.rlc_fw);
1515 adev->gfx.rlc_fw = NULL;
1516 }
1517 return err;
1518}
1519
1520static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1521 const char *chip_name)
1522{
1523 char fw_name[30];
1524 int err;
1525 struct amdgpu_firmware_info *info = NULL;
1526 const struct common_firmware_header *header = NULL;
1527 const struct gfx_firmware_header_v1_0 *cp_hdr;
1528
1529 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1530 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1531 if (err)
1532 goto out;
1533 err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1534 if (err)
1535 goto out;
1536 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1537 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1538 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1539
1540
1541 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1542 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1543 if (!err) {
1544 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1545 if (err)
1546 goto out;
1547 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1548 adev->gfx.mec2_fw->data;
1549 adev->gfx.mec2_fw_version =
1550 le32_to_cpu(cp_hdr->header.ucode_version);
1551 adev->gfx.mec2_feature_version =
1552 le32_to_cpu(cp_hdr->ucode_feature_version);
1553 } else {
1554 err = 0;
1555 adev->gfx.mec2_fw = NULL;
1556 }
1557
1558 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1559 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1560 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1561 info->fw = adev->gfx.mec_fw;
1562 header = (const struct common_firmware_header *)info->fw->data;
1563 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1564 adev->firmware.fw_size +=
1565 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1566
1567 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1568 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1569 info->fw = adev->gfx.mec_fw;
1570 adev->firmware.fw_size +=
1571 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1572
1573 if (adev->gfx.mec2_fw) {
1574 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1575 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1576 info->fw = adev->gfx.mec2_fw;
1577 header = (const struct common_firmware_header *)info->fw->data;
1578 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1579 adev->firmware.fw_size +=
1580 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1581
1582
1583
1584 if (adev->asic_type != CHIP_ARCTURUS &&
1585 adev->asic_type != CHIP_RENOIR) {
1586 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1587 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1588 info->fw = adev->gfx.mec2_fw;
1589 adev->firmware.fw_size +=
1590 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1591 PAGE_SIZE);
1592 }
1593 }
1594 }
1595
1596out:
1597 gfx_v9_0_check_if_need_gfxoff(adev);
1598 gfx_v9_0_check_fw_write_wait(adev);
1599 if (err) {
1600 dev_err(adev->dev,
1601 "gfx9: Failed to load firmware \"%s\"\n",
1602 fw_name);
1603 release_firmware(adev->gfx.mec_fw);
1604 adev->gfx.mec_fw = NULL;
1605 release_firmware(adev->gfx.mec2_fw);
1606 adev->gfx.mec2_fw = NULL;
1607 }
1608 return err;
1609}
1610
1611static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1612{
1613 const char *chip_name;
1614 int r;
1615
1616 DRM_DEBUG("\n");
1617
1618 switch (adev->asic_type) {
1619 case CHIP_VEGA10:
1620 chip_name = "vega10";
1621 break;
1622 case CHIP_VEGA12:
1623 chip_name = "vega12";
1624 break;
1625 case CHIP_VEGA20:
1626 chip_name = "vega20";
1627 break;
1628 case CHIP_RAVEN:
1629 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1630 chip_name = "raven2";
1631 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1632 chip_name = "picasso";
1633 else
1634 chip_name = "raven";
1635 break;
1636 case CHIP_ARCTURUS:
1637 chip_name = "arcturus";
1638 break;
1639 case CHIP_RENOIR:
1640 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1641 chip_name = "renoir";
1642 else
1643 chip_name = "green_sardine";
1644 break;
1645 default:
1646 BUG();
1647 }
1648
1649
1650 if (adev->gfx.num_gfx_rings) {
1651 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1652 if (r)
1653 return r;
1654 }
1655
1656 r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1657 if (r)
1658 return r;
1659
1660 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1661 if (r)
1662 return r;
1663
1664 return r;
1665}
1666
1667static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1668{
1669 u32 count = 0;
1670 const struct cs_section_def *sect = NULL;
1671 const struct cs_extent_def *ext = NULL;
1672
1673
1674 count += 2;
1675
1676 count += 3;
1677
1678 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1679 for (ext = sect->section; ext->extent != NULL; ++ext) {
1680 if (sect->id == SECT_CONTEXT)
1681 count += 2 + ext->reg_count;
1682 else
1683 return 0;
1684 }
1685 }
1686
1687
1688 count += 2;
1689
1690 count += 2;
1691
1692 return count;
1693}
1694
1695static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1696 volatile u32 *buffer)
1697{
1698 u32 count = 0, i;
1699 const struct cs_section_def *sect = NULL;
1700 const struct cs_extent_def *ext = NULL;
1701
1702 if (adev->gfx.rlc.cs_data == NULL)
1703 return;
1704 if (buffer == NULL)
1705 return;
1706
1707 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1708 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1709
1710 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1711 buffer[count++] = cpu_to_le32(0x80000000);
1712 buffer[count++] = cpu_to_le32(0x80000000);
1713
1714 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1715 for (ext = sect->section; ext->extent != NULL; ++ext) {
1716 if (sect->id == SECT_CONTEXT) {
1717 buffer[count++] =
1718 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1719 buffer[count++] = cpu_to_le32(ext->reg_index -
1720 PACKET3_SET_CONTEXT_REG_START);
1721 for (i = 0; i < ext->reg_count; i++)
1722 buffer[count++] = cpu_to_le32(ext->extent[i]);
1723 } else {
1724 return;
1725 }
1726 }
1727 }
1728
1729 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1730 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1731
1732 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1733 buffer[count++] = cpu_to_le32(0);
1734}
1735
1736static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1737{
1738 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1739 uint32_t pg_always_on_cu_num = 2;
1740 uint32_t always_on_cu_num;
1741 uint32_t i, j, k;
1742 uint32_t mask, cu_bitmap, counter;
1743
1744 if (adev->flags & AMD_IS_APU)
1745 always_on_cu_num = 4;
1746 else if (adev->asic_type == CHIP_VEGA12)
1747 always_on_cu_num = 8;
1748 else
1749 always_on_cu_num = 12;
1750
1751 mutex_lock(&adev->grbm_idx_mutex);
1752 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1753 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1754 mask = 1;
1755 cu_bitmap = 0;
1756 counter = 0;
1757 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1758
1759 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1760 if (cu_info->bitmap[i][j] & mask) {
1761 if (counter == pg_always_on_cu_num)
1762 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1763 if (counter < always_on_cu_num)
1764 cu_bitmap |= mask;
1765 else
1766 break;
1767 counter++;
1768 }
1769 mask <<= 1;
1770 }
1771
1772 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1773 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1774 }
1775 }
1776 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1777 mutex_unlock(&adev->grbm_idx_mutex);
1778}
1779
1780static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1781{
1782 uint32_t data;
1783
1784
1785 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1786 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1787 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1788 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1789
1790
1791 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1792
1793
1794 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1795
1796 mutex_lock(&adev->grbm_idx_mutex);
1797
1798 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1799 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1800
1801
1802 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1803 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1804 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1805 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1806
1807
1808 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1809 data &= 0x0000FFFF;
1810 data |= 0x00C00000;
1811 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1812
1813
1814
1815
1816
1817
1818
1819
1820 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1821 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1822 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1823 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1824 mutex_unlock(&adev->grbm_idx_mutex);
1825
1826 gfx_v9_0_init_always_on_cu_mask(adev);
1827}
1828
1829static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1830{
1831 uint32_t data;
1832
1833
1834 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1835 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1836 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1837 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1838
1839
1840 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1841
1842
1843 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1844
1845 mutex_lock(&adev->grbm_idx_mutex);
1846
1847 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1848 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1849
1850
1851 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1852 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1853 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1854 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1855
1856
1857 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1858 data &= 0x0000FFFF;
1859 data |= 0x00C00000;
1860 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1861
1862
1863
1864
1865
1866
1867
1868
1869 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1870 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1871 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1872 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1873 mutex_unlock(&adev->grbm_idx_mutex);
1874
1875 gfx_v9_0_init_always_on_cu_mask(adev);
1876}
1877
1878static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1879{
1880 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1881}
1882
1883static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1884{
1885 return 5;
1886}
1887
1888static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1889{
1890 const struct cs_section_def *cs_data;
1891 int r;
1892
1893 adev->gfx.rlc.cs_data = gfx9_cs_data;
1894
1895 cs_data = adev->gfx.rlc.cs_data;
1896
1897 if (cs_data) {
1898
1899 r = amdgpu_gfx_rlc_init_csb(adev);
1900 if (r)
1901 return r;
1902 }
1903
1904 if (adev->flags & AMD_IS_APU) {
1905
1906 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024);
1907 r = amdgpu_gfx_rlc_init_cpt(adev);
1908 if (r)
1909 return r;
1910 }
1911
1912 switch (adev->asic_type) {
1913 case CHIP_RAVEN:
1914 gfx_v9_0_init_lbpw(adev);
1915 break;
1916 case CHIP_VEGA20:
1917 gfx_v9_4_init_lbpw(adev);
1918 break;
1919 default:
1920 break;
1921 }
1922
1923
1924 if (adev->gfx.rlc.funcs->update_spm_vmid)
1925 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1926
1927 return 0;
1928}
1929
1930static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1931{
1932 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1933 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1934}
1935
1936static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1937{
1938 int r;
1939 u32 *hpd;
1940 const __le32 *fw_data;
1941 unsigned fw_size;
1942 u32 *fw;
1943 size_t mec_hpd_size;
1944
1945 const struct gfx_firmware_header_v1_0 *mec_hdr;
1946
1947 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1948
1949
1950 amdgpu_gfx_compute_queue_acquire(adev);
1951 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1952 if (mec_hpd_size) {
1953 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1954 AMDGPU_GEM_DOMAIN_VRAM,
1955 &adev->gfx.mec.hpd_eop_obj,
1956 &adev->gfx.mec.hpd_eop_gpu_addr,
1957 (void **)&hpd);
1958 if (r) {
1959 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1960 gfx_v9_0_mec_fini(adev);
1961 return r;
1962 }
1963
1964 memset(hpd, 0, mec_hpd_size);
1965
1966 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1967 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1968 }
1969
1970 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1971
1972 fw_data = (const __le32 *)
1973 (adev->gfx.mec_fw->data +
1974 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1975 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1976
1977 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1978 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1979 &adev->gfx.mec.mec_fw_obj,
1980 &adev->gfx.mec.mec_fw_gpu_addr,
1981 (void **)&fw);
1982 if (r) {
1983 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1984 gfx_v9_0_mec_fini(adev);
1985 return r;
1986 }
1987
1988 memcpy(fw, fw_data, fw_size);
1989
1990 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1991 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1992
1993 return 0;
1994}
1995
1996static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1997{
1998 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1999 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2000 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2001 (address << SQ_IND_INDEX__INDEX__SHIFT) |
2002 (SQ_IND_INDEX__FORCE_READ_MASK));
2003 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2004}
2005
2006static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2007 uint32_t wave, uint32_t thread,
2008 uint32_t regno, uint32_t num, uint32_t *out)
2009{
2010 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
2011 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2012 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2013 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2014 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2015 (SQ_IND_INDEX__FORCE_READ_MASK) |
2016 (SQ_IND_INDEX__AUTO_INCR_MASK));
2017 while (num--)
2018 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2019}
2020
2021static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2022{
2023
2024 dst[(*no_fields)++] = 1;
2025 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2026 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2027 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2028 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2029 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2030 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2031 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2032 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2033 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2034 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2035 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2036 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2037 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2038 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2039}
2040
2041static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2042 uint32_t wave, uint32_t start,
2043 uint32_t size, uint32_t *dst)
2044{
2045 wave_read_regs(
2046 adev, simd, wave, 0,
2047 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2048}
2049
2050static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2051 uint32_t wave, uint32_t thread,
2052 uint32_t start, uint32_t size,
2053 uint32_t *dst)
2054{
2055 wave_read_regs(
2056 adev, simd, wave, thread,
2057 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2058}
2059
2060static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2061 u32 me, u32 pipe, u32 q, u32 vm)
2062{
2063 soc15_grbm_select(adev, me, pipe, q, vm);
2064}
2065
2066static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2067 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2068 .select_se_sh = &gfx_v9_0_select_se_sh,
2069 .read_wave_data = &gfx_v9_0_read_wave_data,
2070 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2071 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2072 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2073 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2074 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2075 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2076};
2077
2078static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = {
2079 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2080 .select_se_sh = &gfx_v9_0_select_se_sh,
2081 .read_wave_data = &gfx_v9_0_read_wave_data,
2082 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2083 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2084 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2085 .ras_error_inject = &gfx_v9_4_ras_error_inject,
2086 .query_ras_error_count = &gfx_v9_4_query_ras_error_count,
2087 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count,
2088 .query_ras_error_status = &gfx_v9_4_query_ras_error_status,
2089};
2090
2091static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2092{
2093 u32 gb_addr_config;
2094 int err;
2095
2096 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2097
2098 switch (adev->asic_type) {
2099 case CHIP_VEGA10:
2100 adev->gfx.config.max_hw_contexts = 8;
2101 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2102 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2103 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2104 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2105 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2106 break;
2107 case CHIP_VEGA12:
2108 adev->gfx.config.max_hw_contexts = 8;
2109 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2110 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2111 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2112 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2113 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2114 DRM_INFO("fix gfx.config for vega12\n");
2115 break;
2116 case CHIP_VEGA20:
2117 adev->gfx.config.max_hw_contexts = 8;
2118 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2119 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2120 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2121 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2122 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2123 gb_addr_config &= ~0xf3e777ff;
2124 gb_addr_config |= 0x22014042;
2125
2126 err = amdgpu_atomfirmware_get_gfx_info(adev);
2127 if (err)
2128 return err;
2129 break;
2130 case CHIP_RAVEN:
2131 adev->gfx.config.max_hw_contexts = 8;
2132 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2133 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2134 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2135 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2136 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2137 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2138 else
2139 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2140 break;
2141 case CHIP_ARCTURUS:
2142 adev->gfx.funcs = &gfx_v9_4_gfx_funcs;
2143 adev->gfx.config.max_hw_contexts = 8;
2144 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2145 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2146 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2147 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2148 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2149 gb_addr_config &= ~0xf3e777ff;
2150 gb_addr_config |= 0x22014042;
2151 break;
2152 case CHIP_RENOIR:
2153 adev->gfx.config.max_hw_contexts = 8;
2154 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2155 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2156 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2157 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2158 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2159 gb_addr_config &= ~0xf3e777ff;
2160 gb_addr_config |= 0x22010042;
2161 break;
2162 default:
2163 BUG();
2164 break;
2165 }
2166
2167 adev->gfx.config.gb_addr_config = gb_addr_config;
2168
2169 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2170 REG_GET_FIELD(
2171 adev->gfx.config.gb_addr_config,
2172 GB_ADDR_CONFIG,
2173 NUM_PIPES);
2174
2175 adev->gfx.config.max_tile_pipes =
2176 adev->gfx.config.gb_addr_config_fields.num_pipes;
2177
2178 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2179 REG_GET_FIELD(
2180 adev->gfx.config.gb_addr_config,
2181 GB_ADDR_CONFIG,
2182 NUM_BANKS);
2183 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2184 REG_GET_FIELD(
2185 adev->gfx.config.gb_addr_config,
2186 GB_ADDR_CONFIG,
2187 MAX_COMPRESSED_FRAGS);
2188 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2189 REG_GET_FIELD(
2190 adev->gfx.config.gb_addr_config,
2191 GB_ADDR_CONFIG,
2192 NUM_RB_PER_SE);
2193 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2194 REG_GET_FIELD(
2195 adev->gfx.config.gb_addr_config,
2196 GB_ADDR_CONFIG,
2197 NUM_SHADER_ENGINES);
2198 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2199 REG_GET_FIELD(
2200 adev->gfx.config.gb_addr_config,
2201 GB_ADDR_CONFIG,
2202 PIPE_INTERLEAVE_SIZE));
2203
2204 return 0;
2205}
2206
2207static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2208 int mec, int pipe, int queue)
2209{
2210 unsigned irq_type;
2211 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2212 unsigned int hw_prio;
2213
2214 ring = &adev->gfx.compute_ring[ring_id];
2215
2216
2217 ring->me = mec + 1;
2218 ring->pipe = pipe;
2219 ring->queue = queue;
2220
2221 ring->ring_obj = NULL;
2222 ring->use_doorbell = true;
2223 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2224 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2225 + (ring_id * GFX9_MEC_HPD_SIZE);
2226 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2227
2228 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2229 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2230 + ring->pipe;
2231 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2232 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
2233
2234 return amdgpu_ring_init(adev, ring, 1024,
2235 &adev->gfx.eop_irq, irq_type, hw_prio);
2236}
2237
2238static int gfx_v9_0_sw_init(void *handle)
2239{
2240 int i, j, k, r, ring_id;
2241 struct amdgpu_ring *ring;
2242 struct amdgpu_kiq *kiq;
2243 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2244
2245 switch (adev->asic_type) {
2246 case CHIP_VEGA10:
2247 case CHIP_VEGA12:
2248 case CHIP_VEGA20:
2249 case CHIP_RAVEN:
2250 case CHIP_ARCTURUS:
2251 case CHIP_RENOIR:
2252 adev->gfx.mec.num_mec = 2;
2253 break;
2254 default:
2255 adev->gfx.mec.num_mec = 1;
2256 break;
2257 }
2258
2259 adev->gfx.mec.num_pipe_per_mec = 4;
2260 adev->gfx.mec.num_queue_per_pipe = 8;
2261
2262
2263 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2264 if (r)
2265 return r;
2266
2267
2268 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2269 &adev->gfx.priv_reg_irq);
2270 if (r)
2271 return r;
2272
2273
2274 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2275 &adev->gfx.priv_inst_irq);
2276 if (r)
2277 return r;
2278
2279
2280 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2281 &adev->gfx.cp_ecc_error_irq);
2282 if (r)
2283 return r;
2284
2285
2286 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2287 &adev->gfx.cp_ecc_error_irq);
2288 if (r)
2289 return r;
2290
2291 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2292
2293 gfx_v9_0_scratch_init(adev);
2294
2295 r = gfx_v9_0_init_microcode(adev);
2296 if (r) {
2297 DRM_ERROR("Failed to load gfx firmware!\n");
2298 return r;
2299 }
2300
2301 r = adev->gfx.rlc.funcs->init(adev);
2302 if (r) {
2303 DRM_ERROR("Failed to init rlc BOs!\n");
2304 return r;
2305 }
2306
2307 r = gfx_v9_0_mec_init(adev);
2308 if (r) {
2309 DRM_ERROR("Failed to init MEC BOs!\n");
2310 return r;
2311 }
2312
2313
2314 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2315 ring = &adev->gfx.gfx_ring[i];
2316 ring->ring_obj = NULL;
2317 if (!i)
2318 sprintf(ring->name, "gfx");
2319 else
2320 sprintf(ring->name, "gfx_%d", i);
2321 ring->use_doorbell = true;
2322 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2323 r = amdgpu_ring_init(adev, ring, 1024,
2324 &adev->gfx.eop_irq,
2325 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2326 AMDGPU_RING_PRIO_DEFAULT);
2327 if (r)
2328 return r;
2329 }
2330
2331
2332 ring_id = 0;
2333 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2334 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2335 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2336 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2337 continue;
2338
2339 r = gfx_v9_0_compute_ring_init(adev,
2340 ring_id,
2341 i, k, j);
2342 if (r)
2343 return r;
2344
2345 ring_id++;
2346 }
2347 }
2348 }
2349
2350 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2351 if (r) {
2352 DRM_ERROR("Failed to init KIQ BOs!\n");
2353 return r;
2354 }
2355
2356 kiq = &adev->gfx.kiq;
2357 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2358 if (r)
2359 return r;
2360
2361
2362 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2363 if (r)
2364 return r;
2365
2366 adev->gfx.ce_ram_size = 0x8000;
2367
2368 r = gfx_v9_0_gpu_early_init(adev);
2369 if (r)
2370 return r;
2371
2372 return 0;
2373}
2374
2375
2376static int gfx_v9_0_sw_fini(void *handle)
2377{
2378 int i;
2379 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2380
2381 amdgpu_gfx_ras_fini(adev);
2382
2383 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2384 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2385 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2386 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2387
2388 amdgpu_gfx_mqd_sw_fini(adev);
2389 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2390 amdgpu_gfx_kiq_fini(adev);
2391
2392 gfx_v9_0_mec_fini(adev);
2393 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2394 if (adev->flags & AMD_IS_APU) {
2395 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2396 &adev->gfx.rlc.cp_table_gpu_addr,
2397 (void **)&adev->gfx.rlc.cp_table_ptr);
2398 }
2399 gfx_v9_0_free_microcode(adev);
2400
2401 return 0;
2402}
2403
2404
2405static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2406{
2407
2408}
2409
2410void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2411 u32 instance)
2412{
2413 u32 data;
2414
2415 if (instance == 0xffffffff)
2416 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2417 else
2418 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2419
2420 if (se_num == 0xffffffff)
2421 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2422 else
2423 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2424
2425 if (sh_num == 0xffffffff)
2426 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2427 else
2428 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2429
2430 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2431}
2432
2433static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2434{
2435 u32 data, mask;
2436
2437 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2438 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2439
2440 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2441 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2442
2443 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2444 adev->gfx.config.max_sh_per_se);
2445
2446 return (~data) & mask;
2447}
2448
2449static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2450{
2451 int i, j;
2452 u32 data;
2453 u32 active_rbs = 0;
2454 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2455 adev->gfx.config.max_sh_per_se;
2456
2457 mutex_lock(&adev->grbm_idx_mutex);
2458 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2459 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2460 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2461 data = gfx_v9_0_get_rb_active_bitmap(adev);
2462 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2463 rb_bitmap_width_per_sh);
2464 }
2465 }
2466 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2467 mutex_unlock(&adev->grbm_idx_mutex);
2468
2469 adev->gfx.config.backend_enable_mask = active_rbs;
2470 adev->gfx.config.num_rbs = hweight32(active_rbs);
2471}
2472
2473#define DEFAULT_SH_MEM_BASES (0x6000)
2474static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2475{
2476 int i;
2477 uint32_t sh_mem_config;
2478 uint32_t sh_mem_bases;
2479
2480
2481
2482
2483
2484
2485
2486 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2487
2488 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2489 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2490 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2491
2492 mutex_lock(&adev->srbm_mutex);
2493 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2494 soc15_grbm_select(adev, 0, 0, 0, i);
2495
2496 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2497 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2498 }
2499 soc15_grbm_select(adev, 0, 0, 0, 0);
2500 mutex_unlock(&adev->srbm_mutex);
2501
2502
2503
2504 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2505 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2506 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2507 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2508 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2509 }
2510}
2511
2512static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2513{
2514 int vmid;
2515
2516
2517
2518
2519
2520
2521
2522 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2523 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2524 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2525 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2526 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2527 }
2528}
2529
2530static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2531{
2532 uint32_t tmp;
2533
2534 switch (adev->asic_type) {
2535 case CHIP_ARCTURUS:
2536 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2537 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2538 DISABLE_BARRIER_WAITCNT, 1);
2539 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2540 break;
2541 default:
2542 break;
2543 }
2544}
2545
2546static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2547{
2548 u32 tmp;
2549 int i;
2550
2551 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2552
2553 gfx_v9_0_tiling_mode_table_init(adev);
2554
2555 gfx_v9_0_setup_rb(adev);
2556 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2557 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2558
2559
2560
2561 mutex_lock(&adev->srbm_mutex);
2562 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2563 soc15_grbm_select(adev, 0, 0, 0, i);
2564
2565 if (i == 0) {
2566 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2567 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2568 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2569 !!adev->gmc.noretry);
2570 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2571 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2572 } else {
2573 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2574 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2575 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2576 !!adev->gmc.noretry);
2577 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2578 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2579 (adev->gmc.private_aperture_start >> 48));
2580 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2581 (adev->gmc.shared_aperture_start >> 48));
2582 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2583 }
2584 }
2585 soc15_grbm_select(adev, 0, 0, 0, 0);
2586
2587 mutex_unlock(&adev->srbm_mutex);
2588
2589 gfx_v9_0_init_compute_vmid(adev);
2590 gfx_v9_0_init_gds_vmid(adev);
2591 gfx_v9_0_init_sq_config(adev);
2592}
2593
2594static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2595{
2596 u32 i, j, k;
2597 u32 mask;
2598
2599 mutex_lock(&adev->grbm_idx_mutex);
2600 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2601 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2602 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2603 for (k = 0; k < adev->usec_timeout; k++) {
2604 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2605 break;
2606 udelay(1);
2607 }
2608 if (k == adev->usec_timeout) {
2609 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2610 0xffffffff, 0xffffffff);
2611 mutex_unlock(&adev->grbm_idx_mutex);
2612 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2613 i, j);
2614 return;
2615 }
2616 }
2617 }
2618 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2619 mutex_unlock(&adev->grbm_idx_mutex);
2620
2621 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2622 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2623 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2624 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2625 for (k = 0; k < adev->usec_timeout; k++) {
2626 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2627 break;
2628 udelay(1);
2629 }
2630}
2631
2632static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2633 bool enable)
2634{
2635 u32 tmp;
2636
2637
2638
2639 if (!adev->gfx.num_gfx_rings)
2640 return;
2641
2642 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2643
2644 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2645 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2646 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2647 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2648
2649 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2650}
2651
2652static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2653{
2654 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2655
2656 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2657 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2658 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2659 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2660 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2661 adev->gfx.rlc.clear_state_size);
2662}
2663
2664static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2665 int indirect_offset,
2666 int list_size,
2667 int *unique_indirect_regs,
2668 int unique_indirect_reg_count,
2669 int *indirect_start_offsets,
2670 int *indirect_start_offsets_count,
2671 int max_start_offsets_count)
2672{
2673 int idx;
2674
2675 for (; indirect_offset < list_size; indirect_offset++) {
2676 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2677 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2678 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2679
2680 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2681 indirect_offset += 2;
2682
2683
2684 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2685 if (unique_indirect_regs[idx] ==
2686 register_list_format[indirect_offset] ||
2687 !unique_indirect_regs[idx])
2688 break;
2689 }
2690
2691 BUG_ON(idx >= unique_indirect_reg_count);
2692
2693 if (!unique_indirect_regs[idx])
2694 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2695
2696 indirect_offset++;
2697 }
2698 }
2699}
2700
2701static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2702{
2703 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2704 int unique_indirect_reg_count = 0;
2705
2706 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2707 int indirect_start_offsets_count = 0;
2708
2709 int list_size = 0;
2710 int i = 0, j = 0;
2711 u32 tmp = 0;
2712
2713 u32 *register_list_format =
2714 kmemdup(adev->gfx.rlc.register_list_format,
2715 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2716 if (!register_list_format)
2717 return -ENOMEM;
2718
2719
2720 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2721 gfx_v9_1_parse_ind_reg_list(register_list_format,
2722 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2723 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2724 unique_indirect_regs,
2725 unique_indirect_reg_count,
2726 indirect_start_offsets,
2727 &indirect_start_offsets_count,
2728 ARRAY_SIZE(indirect_start_offsets));
2729
2730
2731 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2732 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2733 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2734
2735
2736 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2737 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2738 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2739 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2740 adev->gfx.rlc.register_restore[i]);
2741
2742
2743 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2744 adev->gfx.rlc.reg_list_format_start);
2745
2746
2747 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2748 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2749 register_list_format[i]);
2750
2751
2752 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2753 if (register_list_format[i] == 0xFFFFFFFF) {
2754 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2755 continue;
2756 }
2757
2758 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2759 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2760
2761 for (j = 0; j < unique_indirect_reg_count; j++) {
2762 if (register_list_format[i] == unique_indirect_regs[j]) {
2763 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2764 break;
2765 }
2766 }
2767
2768 BUG_ON(j >= unique_indirect_reg_count);
2769
2770 i++;
2771 }
2772
2773
2774 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2775 list_size = list_size >> 1;
2776 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2777 adev->gfx.rlc.reg_restore_list_size);
2778 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2779
2780
2781 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2782 adev->gfx.rlc.starting_offsets_start);
2783 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2784 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2785 indirect_start_offsets[i]);
2786
2787
2788 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2789 if (unique_indirect_regs[i] != 0) {
2790 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2791 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2792 unique_indirect_regs[i] & 0x3FFFF);
2793
2794 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2795 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2796 unique_indirect_regs[i] >> 20);
2797 }
2798 }
2799
2800 kfree(register_list_format);
2801 return 0;
2802}
2803
2804static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2805{
2806 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2807}
2808
2809static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2810 bool enable)
2811{
2812 uint32_t data = 0;
2813 uint32_t default_data = 0;
2814
2815 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2816 if (enable) {
2817
2818 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2819 if(default_data != data)
2820 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2821
2822
2823 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2824 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2825 if(default_data != data)
2826 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2827 } else {
2828
2829 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2830 if(default_data != data)
2831 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2832 }
2833}
2834
2835static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2836{
2837 uint32_t data = 0;
2838
2839 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2840 AMD_PG_SUPPORT_GFX_SMG |
2841 AMD_PG_SUPPORT_GFX_DMG)) {
2842
2843 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2844 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2845 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2846 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2847
2848
2849 data = 0;
2850 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2851 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2852 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2853 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2855
2856 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2857 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2858 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2859 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2860
2861 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2862 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2863 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2864 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2865
2866 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2867 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2868
2869
2870 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2871 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2872 if (adev->asic_type != CHIP_RENOIR)
2873 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2874 }
2875}
2876
2877static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2878 bool enable)
2879{
2880 uint32_t data = 0;
2881 uint32_t default_data = 0;
2882
2883 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2884 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2885 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2886 enable ? 1 : 0);
2887 if (default_data != data)
2888 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2889}
2890
2891static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2892 bool enable)
2893{
2894 uint32_t data = 0;
2895 uint32_t default_data = 0;
2896
2897 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2898 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2899 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2900 enable ? 1 : 0);
2901 if(default_data != data)
2902 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2903}
2904
2905static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2906 bool enable)
2907{
2908 uint32_t data = 0;
2909 uint32_t default_data = 0;
2910
2911 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2912 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2913 CP_PG_DISABLE,
2914 enable ? 0 : 1);
2915 if(default_data != data)
2916 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2917}
2918
2919static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2920 bool enable)
2921{
2922 uint32_t data, default_data;
2923
2924 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2925 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2926 GFX_POWER_GATING_ENABLE,
2927 enable ? 1 : 0);
2928 if(default_data != data)
2929 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2930}
2931
2932static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2933 bool enable)
2934{
2935 uint32_t data, default_data;
2936
2937 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2938 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2939 GFX_PIPELINE_PG_ENABLE,
2940 enable ? 1 : 0);
2941 if(default_data != data)
2942 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2943
2944 if (!enable)
2945
2946 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2947}
2948
2949static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2950 bool enable)
2951{
2952 uint32_t data, default_data;
2953
2954 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2955 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2956 STATIC_PER_CU_PG_ENABLE,
2957 enable ? 1 : 0);
2958 if(default_data != data)
2959 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2960}
2961
2962static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2963 bool enable)
2964{
2965 uint32_t data, default_data;
2966
2967 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2968 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2969 DYN_PER_CU_PG_ENABLE,
2970 enable ? 1 : 0);
2971 if(default_data != data)
2972 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2973}
2974
2975static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2976{
2977 gfx_v9_0_init_csb(adev);
2978
2979
2980
2981
2982
2983 if (adev->gfx.rlc.is_rlc_v2_1) {
2984 if (adev->asic_type == CHIP_VEGA12 ||
2985 (adev->apu_flags & AMD_APU_IS_RAVEN2))
2986 gfx_v9_1_init_rlc_save_restore_list(adev);
2987 gfx_v9_0_enable_save_restore_machine(adev);
2988 }
2989
2990 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2991 AMD_PG_SUPPORT_GFX_SMG |
2992 AMD_PG_SUPPORT_GFX_DMG |
2993 AMD_PG_SUPPORT_CP |
2994 AMD_PG_SUPPORT_GDS |
2995 AMD_PG_SUPPORT_RLC_SMU_HS)) {
2996 WREG32(mmRLC_JUMP_TABLE_RESTORE,
2997 adev->gfx.rlc.cp_table_gpu_addr >> 8);
2998 gfx_v9_0_init_gfx_power_gating(adev);
2999 }
3000}
3001
3002static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3003{
3004 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3005 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3006 gfx_v9_0_wait_for_rlc_serdes(adev);
3007}
3008
3009static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3010{
3011 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3012 udelay(50);
3013 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3014 udelay(50);
3015}
3016
3017static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3018{
3019#ifdef AMDGPU_RLC_DEBUG_RETRY
3020 u32 rlc_ucode_ver;
3021#endif
3022
3023 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3024 udelay(50);
3025
3026
3027 if (!(adev->flags & AMD_IS_APU)) {
3028 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3029 udelay(50);
3030 }
3031
3032#ifdef AMDGPU_RLC_DEBUG_RETRY
3033
3034 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3035 if(rlc_ucode_ver == 0x108) {
3036 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3037 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3038
3039
3040 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3041
3042
3043
3044 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3045 }
3046#endif
3047}
3048
3049static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3050{
3051 const struct rlc_firmware_header_v2_0 *hdr;
3052 const __le32 *fw_data;
3053 unsigned i, fw_size;
3054
3055 if (!adev->gfx.rlc_fw)
3056 return -EINVAL;
3057
3058 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3059 amdgpu_ucode_print_rlc_hdr(&hdr->header);
3060
3061 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3062 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3063 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3064
3065 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3066 RLCG_UCODE_LOADING_START_ADDRESS);
3067 for (i = 0; i < fw_size; i++)
3068 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3069 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3070
3071 return 0;
3072}
3073
3074static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3075{
3076 int r;
3077
3078 if (amdgpu_sriov_vf(adev)) {
3079 gfx_v9_0_init_csb(adev);
3080 return 0;
3081 }
3082
3083 adev->gfx.rlc.funcs->stop(adev);
3084
3085
3086 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3087
3088 gfx_v9_0_init_pg(adev);
3089
3090 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3091
3092 r = gfx_v9_0_rlc_load_microcode(adev);
3093 if (r)
3094 return r;
3095 }
3096
3097 switch (adev->asic_type) {
3098 case CHIP_RAVEN:
3099 if (amdgpu_lbpw == 0)
3100 gfx_v9_0_enable_lbpw(adev, false);
3101 else
3102 gfx_v9_0_enable_lbpw(adev, true);
3103 break;
3104 case CHIP_VEGA20:
3105 if (amdgpu_lbpw > 0)
3106 gfx_v9_0_enable_lbpw(adev, true);
3107 else
3108 gfx_v9_0_enable_lbpw(adev, false);
3109 break;
3110 default:
3111 break;
3112 }
3113
3114 adev->gfx.rlc.funcs->start(adev);
3115
3116 return 0;
3117}
3118
3119static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3120{
3121 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3122
3123 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3124 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3125 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3126 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3127 udelay(50);
3128}
3129
3130static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3131{
3132 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3133 const struct gfx_firmware_header_v1_0 *ce_hdr;
3134 const struct gfx_firmware_header_v1_0 *me_hdr;
3135 const __le32 *fw_data;
3136 unsigned i, fw_size;
3137
3138 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3139 return -EINVAL;
3140
3141 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3142 adev->gfx.pfp_fw->data;
3143 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3144 adev->gfx.ce_fw->data;
3145 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3146 adev->gfx.me_fw->data;
3147
3148 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3149 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3150 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3151
3152 gfx_v9_0_cp_gfx_enable(adev, false);
3153
3154
3155 fw_data = (const __le32 *)
3156 (adev->gfx.pfp_fw->data +
3157 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3158 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3159 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3160 for (i = 0; i < fw_size; i++)
3161 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3162 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3163
3164
3165 fw_data = (const __le32 *)
3166 (adev->gfx.ce_fw->data +
3167 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3168 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3169 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3170 for (i = 0; i < fw_size; i++)
3171 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3172 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3173
3174
3175 fw_data = (const __le32 *)
3176 (adev->gfx.me_fw->data +
3177 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3178 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3179 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3180 for (i = 0; i < fw_size; i++)
3181 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3182 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3183
3184 return 0;
3185}
3186
3187static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3188{
3189 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3190 const struct cs_section_def *sect = NULL;
3191 const struct cs_extent_def *ext = NULL;
3192 int r, i, tmp;
3193
3194
3195 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3196 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3197
3198 gfx_v9_0_cp_gfx_enable(adev, true);
3199
3200 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3201 if (r) {
3202 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3203 return r;
3204 }
3205
3206 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3207 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3208
3209 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3210 amdgpu_ring_write(ring, 0x80000000);
3211 amdgpu_ring_write(ring, 0x80000000);
3212
3213 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3214 for (ext = sect->section; ext->extent != NULL; ++ext) {
3215 if (sect->id == SECT_CONTEXT) {
3216 amdgpu_ring_write(ring,
3217 PACKET3(PACKET3_SET_CONTEXT_REG,
3218 ext->reg_count));
3219 amdgpu_ring_write(ring,
3220 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3221 for (i = 0; i < ext->reg_count; i++)
3222 amdgpu_ring_write(ring, ext->extent[i]);
3223 }
3224 }
3225 }
3226
3227 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3228 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3229
3230 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3231 amdgpu_ring_write(ring, 0);
3232
3233 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3234 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3235 amdgpu_ring_write(ring, 0x8000);
3236 amdgpu_ring_write(ring, 0x8000);
3237
3238 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3239 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3240 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3241 amdgpu_ring_write(ring, tmp);
3242 amdgpu_ring_write(ring, 0);
3243
3244 amdgpu_ring_commit(ring);
3245
3246 return 0;
3247}
3248
3249static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3250{
3251 struct amdgpu_ring *ring;
3252 u32 tmp;
3253 u32 rb_bufsz;
3254 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3255
3256
3257 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3258
3259
3260 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3261
3262
3263 ring = &adev->gfx.gfx_ring[0];
3264 rb_bufsz = order_base_2(ring->ring_size / 8);
3265 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3266 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3267#ifdef __BIG_ENDIAN
3268 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3269#endif
3270 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3271
3272
3273 ring->wptr = 0;
3274 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3275 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3276
3277
3278 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3279 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3280 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3281
3282 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3283 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3284 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3285
3286 mdelay(1);
3287 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3288
3289 rb_addr = ring->gpu_addr >> 8;
3290 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3291 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3292
3293 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3294 if (ring->use_doorbell) {
3295 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3296 DOORBELL_OFFSET, ring->doorbell_index);
3297 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3298 DOORBELL_EN, 1);
3299 } else {
3300 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3301 }
3302 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3303
3304 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3305 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3306 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3307
3308 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3309 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3310
3311
3312
3313 gfx_v9_0_cp_gfx_start(adev);
3314 ring->sched.ready = true;
3315
3316 return 0;
3317}
3318
3319static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3320{
3321 if (enable) {
3322 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3323 } else {
3324 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3325 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3326 adev->gfx.kiq.ring.sched.ready = false;
3327 }
3328 udelay(50);
3329}
3330
3331static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3332{
3333 const struct gfx_firmware_header_v1_0 *mec_hdr;
3334 const __le32 *fw_data;
3335 unsigned i;
3336 u32 tmp;
3337
3338 if (!adev->gfx.mec_fw)
3339 return -EINVAL;
3340
3341 gfx_v9_0_cp_compute_enable(adev, false);
3342
3343 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3344 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3345
3346 fw_data = (const __le32 *)
3347 (adev->gfx.mec_fw->data +
3348 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3349 tmp = 0;
3350 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3351 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3352 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3353
3354 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3355 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3356 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3357 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3358
3359
3360 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3361 mec_hdr->jt_offset);
3362 for (i = 0; i < mec_hdr->jt_size; i++)
3363 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3364 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3365
3366 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3367 adev->gfx.mec_fw_version);
3368
3369
3370 return 0;
3371}
3372
3373
3374static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3375{
3376 uint32_t tmp;
3377 struct amdgpu_device *adev = ring->adev;
3378
3379
3380 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3381 tmp &= 0xffffff00;
3382 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3383 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3384 tmp |= 0x80;
3385 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3386}
3387
3388static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3389{
3390 struct amdgpu_device *adev = ring->adev;
3391
3392 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3393 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3394 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3395 mqd->cp_hqd_queue_priority =
3396 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3397 }
3398 }
3399}
3400
3401static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3402{
3403 struct amdgpu_device *adev = ring->adev;
3404 struct v9_mqd *mqd = ring->mqd_ptr;
3405 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3406 uint32_t tmp;
3407
3408 mqd->header = 0xC0310800;
3409 mqd->compute_pipelinestat_enable = 0x00000001;
3410 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3411 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3412 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3413 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3414 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3415 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3416 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3417 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3418 mqd->compute_misc_reserved = 0x00000003;
3419
3420 mqd->dynamic_cu_mask_addr_lo =
3421 lower_32_bits(ring->mqd_gpu_addr
3422 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3423 mqd->dynamic_cu_mask_addr_hi =
3424 upper_32_bits(ring->mqd_gpu_addr
3425 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3426
3427 eop_base_addr = ring->eop_gpu_addr >> 8;
3428 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3429 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3430
3431
3432 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3433 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3434 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3435
3436 mqd->cp_hqd_eop_control = tmp;
3437
3438
3439 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3440
3441 if (ring->use_doorbell) {
3442 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3443 DOORBELL_OFFSET, ring->doorbell_index);
3444 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3445 DOORBELL_EN, 1);
3446 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3447 DOORBELL_SOURCE, 0);
3448 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3449 DOORBELL_HIT, 0);
3450 } else {
3451 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3452 DOORBELL_EN, 0);
3453 }
3454
3455 mqd->cp_hqd_pq_doorbell_control = tmp;
3456
3457
3458 ring->wptr = 0;
3459 mqd->cp_hqd_dequeue_request = 0;
3460 mqd->cp_hqd_pq_rptr = 0;
3461 mqd->cp_hqd_pq_wptr_lo = 0;
3462 mqd->cp_hqd_pq_wptr_hi = 0;
3463
3464
3465 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3466 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3467
3468
3469 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3470 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3471 mqd->cp_mqd_control = tmp;
3472
3473
3474 hqd_gpu_addr = ring->gpu_addr >> 8;
3475 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3476 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3477
3478
3479 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3480 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3481 (order_base_2(ring->ring_size / 4) - 1));
3482 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3483 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3484#ifdef __BIG_ENDIAN
3485 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3486#endif
3487 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3488 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3489 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3490 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3491 mqd->cp_hqd_pq_control = tmp;
3492
3493
3494 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3495 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3496 mqd->cp_hqd_pq_rptr_report_addr_hi =
3497 upper_32_bits(wb_gpu_addr) & 0xffff;
3498
3499
3500 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3501 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3502 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3503
3504 tmp = 0;
3505
3506 if (ring->use_doorbell) {
3507 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3508 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3509 DOORBELL_OFFSET, ring->doorbell_index);
3510
3511 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3512 DOORBELL_EN, 1);
3513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3514 DOORBELL_SOURCE, 0);
3515 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3516 DOORBELL_HIT, 0);
3517 }
3518
3519 mqd->cp_hqd_pq_doorbell_control = tmp;
3520
3521
3522 ring->wptr = 0;
3523 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3524
3525
3526 mqd->cp_hqd_vmid = 0;
3527
3528 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3529 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3530 mqd->cp_hqd_persistent_state = tmp;
3531
3532
3533 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3534 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3535 mqd->cp_hqd_ib_control = tmp;
3536
3537
3538 gfx_v9_0_mqd_set_priority(ring, mqd);
3539 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3540
3541
3542
3543
3544 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3545 mqd->cp_hqd_active = 1;
3546
3547 return 0;
3548}
3549
3550static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3551{
3552 struct amdgpu_device *adev = ring->adev;
3553 struct v9_mqd *mqd = ring->mqd_ptr;
3554 int j;
3555
3556
3557 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3558
3559 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3560 mqd->cp_hqd_eop_base_addr_lo);
3561 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3562 mqd->cp_hqd_eop_base_addr_hi);
3563
3564
3565 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3566 mqd->cp_hqd_eop_control);
3567
3568
3569 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3570 mqd->cp_hqd_pq_doorbell_control);
3571
3572
3573 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3574 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3575 for (j = 0; j < adev->usec_timeout; j++) {
3576 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3577 break;
3578 udelay(1);
3579 }
3580 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3581 mqd->cp_hqd_dequeue_request);
3582 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3583 mqd->cp_hqd_pq_rptr);
3584 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3585 mqd->cp_hqd_pq_wptr_lo);
3586 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3587 mqd->cp_hqd_pq_wptr_hi);
3588 }
3589
3590
3591 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3592 mqd->cp_mqd_base_addr_lo);
3593 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3594 mqd->cp_mqd_base_addr_hi);
3595
3596
3597 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3598 mqd->cp_mqd_control);
3599
3600
3601 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3602 mqd->cp_hqd_pq_base_lo);
3603 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3604 mqd->cp_hqd_pq_base_hi);
3605
3606
3607 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3608 mqd->cp_hqd_pq_control);
3609
3610
3611 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3612 mqd->cp_hqd_pq_rptr_report_addr_lo);
3613 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3614 mqd->cp_hqd_pq_rptr_report_addr_hi);
3615
3616
3617 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3618 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3620 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3621
3622
3623 if (ring->use_doorbell) {
3624 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3625 (adev->doorbell_index.kiq * 2) << 2);
3626 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3627 (adev->doorbell_index.userqueue_end * 2) << 2);
3628 }
3629
3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3631 mqd->cp_hqd_pq_doorbell_control);
3632
3633
3634 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3635 mqd->cp_hqd_pq_wptr_lo);
3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3637 mqd->cp_hqd_pq_wptr_hi);
3638
3639
3640 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3641
3642 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3643 mqd->cp_hqd_persistent_state);
3644
3645
3646 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3647 mqd->cp_hqd_active);
3648
3649 if (ring->use_doorbell)
3650 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3651
3652 return 0;
3653}
3654
3655static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3656{
3657 struct amdgpu_device *adev = ring->adev;
3658 int j;
3659
3660
3661 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3662
3663 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3664
3665 for (j = 0; j < adev->usec_timeout; j++) {
3666 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3667 break;
3668 udelay(1);
3669 }
3670
3671 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3672 DRM_DEBUG("KIQ dequeue request failed.\n");
3673
3674
3675 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3676 }
3677
3678 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3679 0);
3680 }
3681
3682 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3683 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3685 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3687 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3688 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3689 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3690
3691 return 0;
3692}
3693
3694static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3695{
3696 struct amdgpu_device *adev = ring->adev;
3697 struct v9_mqd *mqd = ring->mqd_ptr;
3698 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3699
3700 gfx_v9_0_kiq_setting(ring);
3701
3702 if (amdgpu_in_reset(adev)) {
3703
3704 if (adev->gfx.mec.mqd_backup[mqd_idx])
3705 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3706
3707
3708 ring->wptr = 0;
3709 amdgpu_ring_clear_ring(ring);
3710
3711 mutex_lock(&adev->srbm_mutex);
3712 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3713 gfx_v9_0_kiq_init_register(ring);
3714 soc15_grbm_select(adev, 0, 0, 0, 0);
3715 mutex_unlock(&adev->srbm_mutex);
3716 } else {
3717 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3718 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3719 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3720 mutex_lock(&adev->srbm_mutex);
3721 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3722 gfx_v9_0_mqd_init(ring);
3723 gfx_v9_0_kiq_init_register(ring);
3724 soc15_grbm_select(adev, 0, 0, 0, 0);
3725 mutex_unlock(&adev->srbm_mutex);
3726
3727 if (adev->gfx.mec.mqd_backup[mqd_idx])
3728 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3729 }
3730
3731 return 0;
3732}
3733
3734static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3735{
3736 struct amdgpu_device *adev = ring->adev;
3737 struct v9_mqd *mqd = ring->mqd_ptr;
3738 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3739
3740 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3741 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3742 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3743 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3744 mutex_lock(&adev->srbm_mutex);
3745 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3746 gfx_v9_0_mqd_init(ring);
3747 soc15_grbm_select(adev, 0, 0, 0, 0);
3748 mutex_unlock(&adev->srbm_mutex);
3749
3750 if (adev->gfx.mec.mqd_backup[mqd_idx])
3751 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3752 } else if (amdgpu_in_reset(adev)) {
3753
3754 if (adev->gfx.mec.mqd_backup[mqd_idx])
3755 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3756
3757
3758 ring->wptr = 0;
3759 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3760 amdgpu_ring_clear_ring(ring);
3761 } else {
3762 amdgpu_ring_clear_ring(ring);
3763 }
3764
3765 return 0;
3766}
3767
3768static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3769{
3770 struct amdgpu_ring *ring;
3771 int r;
3772
3773 ring = &adev->gfx.kiq.ring;
3774
3775 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3776 if (unlikely(r != 0))
3777 return r;
3778
3779 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3780 if (unlikely(r != 0))
3781 return r;
3782
3783 gfx_v9_0_kiq_init_queue(ring);
3784 amdgpu_bo_kunmap(ring->mqd_obj);
3785 ring->mqd_ptr = NULL;
3786 amdgpu_bo_unreserve(ring->mqd_obj);
3787 ring->sched.ready = true;
3788 return 0;
3789}
3790
3791static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3792{
3793 struct amdgpu_ring *ring = NULL;
3794 int r = 0, i;
3795
3796 gfx_v9_0_cp_compute_enable(adev, true);
3797
3798 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3799 ring = &adev->gfx.compute_ring[i];
3800
3801 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3802 if (unlikely(r != 0))
3803 goto done;
3804 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3805 if (!r) {
3806 r = gfx_v9_0_kcq_init_queue(ring);
3807 amdgpu_bo_kunmap(ring->mqd_obj);
3808 ring->mqd_ptr = NULL;
3809 }
3810 amdgpu_bo_unreserve(ring->mqd_obj);
3811 if (r)
3812 goto done;
3813 }
3814
3815 r = amdgpu_gfx_enable_kcq(adev);
3816done:
3817 return r;
3818}
3819
3820static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3821{
3822 int r, i;
3823 struct amdgpu_ring *ring;
3824
3825 if (!(adev->flags & AMD_IS_APU))
3826 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3827
3828 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3829 if (adev->gfx.num_gfx_rings) {
3830
3831 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3832 if (r)
3833 return r;
3834 }
3835
3836 r = gfx_v9_0_cp_compute_load_microcode(adev);
3837 if (r)
3838 return r;
3839 }
3840
3841 r = gfx_v9_0_kiq_resume(adev);
3842 if (r)
3843 return r;
3844
3845 if (adev->gfx.num_gfx_rings) {
3846 r = gfx_v9_0_cp_gfx_resume(adev);
3847 if (r)
3848 return r;
3849 }
3850
3851 r = gfx_v9_0_kcq_resume(adev);
3852 if (r)
3853 return r;
3854
3855 if (adev->gfx.num_gfx_rings) {
3856 ring = &adev->gfx.gfx_ring[0];
3857 r = amdgpu_ring_test_helper(ring);
3858 if (r)
3859 return r;
3860 }
3861
3862 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3863 ring = &adev->gfx.compute_ring[i];
3864 amdgpu_ring_test_helper(ring);
3865 }
3866
3867 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3868
3869 return 0;
3870}
3871
3872static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3873{
3874 u32 tmp;
3875
3876 if (adev->asic_type != CHIP_ARCTURUS)
3877 return;
3878
3879 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3880 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3881 adev->df.hash_status.hash_64k);
3882 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3883 adev->df.hash_status.hash_2m);
3884 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3885 adev->df.hash_status.hash_1g);
3886 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3887}
3888
3889static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3890{
3891 if (adev->gfx.num_gfx_rings)
3892 gfx_v9_0_cp_gfx_enable(adev, enable);
3893 gfx_v9_0_cp_compute_enable(adev, enable);
3894}
3895
3896static int gfx_v9_0_hw_init(void *handle)
3897{
3898 int r;
3899 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3900
3901 if (!amdgpu_sriov_vf(adev))
3902 gfx_v9_0_init_golden_registers(adev);
3903
3904 gfx_v9_0_constants_init(adev);
3905
3906 gfx_v9_0_init_tcp_config(adev);
3907
3908 r = adev->gfx.rlc.funcs->resume(adev);
3909 if (r)
3910 return r;
3911
3912 r = gfx_v9_0_cp_resume(adev);
3913 if (r)
3914 return r;
3915
3916 return r;
3917}
3918
3919static int gfx_v9_0_hw_fini(void *handle)
3920{
3921 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3922
3923 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3924 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3925 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3926
3927
3928 if (!amdgpu_ras_intr_triggered())
3929
3930 amdgpu_gfx_disable_kcq(adev);
3931
3932 if (amdgpu_sriov_vf(adev)) {
3933 gfx_v9_0_cp_gfx_enable(adev, false);
3934
3935
3936
3937
3938
3939 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3940 return 0;
3941 }
3942
3943
3944
3945
3946 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3947 mutex_lock(&adev->srbm_mutex);
3948 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3949 adev->gfx.kiq.ring.pipe,
3950 adev->gfx.kiq.ring.queue, 0);
3951 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3952 soc15_grbm_select(adev, 0, 0, 0, 0);
3953 mutex_unlock(&adev->srbm_mutex);
3954 }
3955
3956 gfx_v9_0_cp_enable(adev, false);
3957 adev->gfx.rlc.funcs->stop(adev);
3958
3959 return 0;
3960}
3961
3962static int gfx_v9_0_suspend(void *handle)
3963{
3964 return gfx_v9_0_hw_fini(handle);
3965}
3966
3967static int gfx_v9_0_resume(void *handle)
3968{
3969 return gfx_v9_0_hw_init(handle);
3970}
3971
3972static bool gfx_v9_0_is_idle(void *handle)
3973{
3974 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3975
3976 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3977 GRBM_STATUS, GUI_ACTIVE))
3978 return false;
3979 else
3980 return true;
3981}
3982
3983static int gfx_v9_0_wait_for_idle(void *handle)
3984{
3985 unsigned i;
3986 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3987
3988 for (i = 0; i < adev->usec_timeout; i++) {
3989 if (gfx_v9_0_is_idle(handle))
3990 return 0;
3991 udelay(1);
3992 }
3993 return -ETIMEDOUT;
3994}
3995
3996static int gfx_v9_0_soft_reset(void *handle)
3997{
3998 u32 grbm_soft_reset = 0;
3999 u32 tmp;
4000 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4001
4002
4003 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4004 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4005 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4006 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4007 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4008 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4009 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4010 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4011 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4012 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4013 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4014 }
4015
4016 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4017 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4018 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4019 }
4020
4021
4022 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4023 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4024 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4025 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4026
4027
4028 if (grbm_soft_reset) {
4029
4030 adev->gfx.rlc.funcs->stop(adev);
4031
4032 if (adev->gfx.num_gfx_rings)
4033
4034 gfx_v9_0_cp_gfx_enable(adev, false);
4035
4036
4037 gfx_v9_0_cp_compute_enable(adev, false);
4038
4039 if (grbm_soft_reset) {
4040 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4041 tmp |= grbm_soft_reset;
4042 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4043 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4044 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4045
4046 udelay(50);
4047
4048 tmp &= ~grbm_soft_reset;
4049 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4050 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4051 }
4052
4053
4054 udelay(50);
4055 }
4056 return 0;
4057}
4058
4059static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4060{
4061 signed long r, cnt = 0;
4062 unsigned long flags;
4063 uint32_t seq, reg_val_offs = 0;
4064 uint64_t value = 0;
4065 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4066 struct amdgpu_ring *ring = &kiq->ring;
4067
4068 BUG_ON(!ring->funcs->emit_rreg);
4069
4070 spin_lock_irqsave(&kiq->ring_lock, flags);
4071 if (amdgpu_device_wb_get(adev, ®_val_offs)) {
4072 pr_err("critical bug! too many kiq readers\n");
4073 goto failed_unlock;
4074 }
4075 amdgpu_ring_alloc(ring, 32);
4076 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4077 amdgpu_ring_write(ring, 9 |
4078 (5 << 8) |
4079 (1 << 16) |
4080 (1 << 20));
4081 amdgpu_ring_write(ring, 0);
4082 amdgpu_ring_write(ring, 0);
4083 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4084 reg_val_offs * 4));
4085 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4086 reg_val_offs * 4));
4087 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4088 if (r)
4089 goto failed_undo;
4090
4091 amdgpu_ring_commit(ring);
4092 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4093
4094 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
4105 goto failed_kiq_read;
4106
4107 might_sleep();
4108 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4109 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4110 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4111 }
4112
4113 if (cnt > MAX_KIQ_REG_TRY)
4114 goto failed_kiq_read;
4115
4116 mb();
4117 value = (uint64_t)adev->wb.wb[reg_val_offs] |
4118 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4119 amdgpu_device_wb_free(adev, reg_val_offs);
4120 return value;
4121
4122failed_undo:
4123 amdgpu_ring_undo(ring);
4124failed_unlock:
4125 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4126failed_kiq_read:
4127 if (reg_val_offs)
4128 amdgpu_device_wb_free(adev, reg_val_offs);
4129 pr_err("failed to read gpu clock\n");
4130 return ~0;
4131}
4132
4133static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4134{
4135 uint64_t clock;
4136
4137 amdgpu_gfx_off_ctrl(adev, false);
4138 mutex_lock(&adev->gfx.gpu_clock_mutex);
4139 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4140 clock = gfx_v9_0_kiq_read_clock(adev);
4141 } else {
4142 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4143 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4144 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4145 }
4146 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4147 amdgpu_gfx_off_ctrl(adev, true);
4148 return clock;
4149}
4150
4151static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4152 uint32_t vmid,
4153 uint32_t gds_base, uint32_t gds_size,
4154 uint32_t gws_base, uint32_t gws_size,
4155 uint32_t oa_base, uint32_t oa_size)
4156{
4157 struct amdgpu_device *adev = ring->adev;
4158
4159
4160 gfx_v9_0_write_data_to_reg(ring, 0, false,
4161 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4162 gds_base);
4163
4164
4165 gfx_v9_0_write_data_to_reg(ring, 0, false,
4166 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4167 gds_size);
4168
4169
4170 gfx_v9_0_write_data_to_reg(ring, 0, false,
4171 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4172 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4173
4174
4175 gfx_v9_0_write_data_to_reg(ring, 0, false,
4176 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4177 (1 << (oa_size + oa_base)) - (1 << oa_base));
4178}
4179
4180static const u32 vgpr_init_compute_shader[] =
4181{
4182 0xb07c0000, 0xbe8000ff,
4183 0x000000f8, 0xbf110800,
4184 0x7e000280, 0x7e020280,
4185 0x7e040280, 0x7e060280,
4186 0x7e080280, 0x7e0a0280,
4187 0x7e0c0280, 0x7e0e0280,
4188 0x80808800, 0xbe803200,
4189 0xbf84fff5, 0xbf9c0000,
4190 0xd28c0001, 0x0001007f,
4191 0xd28d0001, 0x0002027e,
4192 0x10020288, 0xb8810904,
4193 0xb7814000, 0xd1196a01,
4194 0x00000301, 0xbe800087,
4195 0xbefc00c1, 0xd89c4000,
4196 0x00020201, 0xd89cc080,
4197 0x00040401, 0x320202ff,
4198 0x00000800, 0x80808100,
4199 0xbf84fff8, 0x7e020280,
4200 0xbf810000, 0x00000000,
4201};
4202
4203static const u32 sgpr_init_compute_shader[] =
4204{
4205 0xb07c0000, 0xbe8000ff,
4206 0x0000005f, 0xbee50080,
4207 0xbe812c65, 0xbe822c65,
4208 0xbe832c65, 0xbe842c65,
4209 0xbe852c65, 0xb77c0005,
4210 0x80808500, 0xbf84fff8,
4211 0xbe800080, 0xbf810000,
4212};
4213
4214static const u32 vgpr_init_compute_shader_arcturus[] = {
4215 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4216 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4217 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4218 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4219 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4220 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4221 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4222 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4223 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4224 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4225 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4226 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4227 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4228 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4229 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4230 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4231 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4232 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4233 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4234 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4235 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4236 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4237 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4238 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4239 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4240 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4241 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4242 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4243 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4244 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4245 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4246 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4247 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4248 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4249 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4250 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4251 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4252 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4253 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4254 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4255 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4256 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4257 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4258 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4259 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4260 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4261 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4262 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4263 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4264 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4265 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4266 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4267 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4268 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4269 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4270 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4271 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4272 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4273 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4274 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4275 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4276 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4277 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4278 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4279 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4280 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4281 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4282 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4283 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4284 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4285 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4286 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4287 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4288 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4289 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4290 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4291 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4292 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4293 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4294 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4295 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4296 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4297 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4298 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4299 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4300 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4301 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4302 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4303 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4304 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4305 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4306 0xbf84fff8, 0xbf810000,
4307};
4308
4309
4310
4311
4312static const struct soc15_reg_entry vgpr_init_regs[] = {
4313 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4314 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4315 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4316 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4317 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4318 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },
4319 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4320 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4321 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4322 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4323 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4324 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4325 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4326 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4327};
4328
4329static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4330 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4331 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },
4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4340 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4341 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4342 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4343 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4344};
4345
4346static const struct soc15_reg_entry sgpr1_init_regs[] = {
4347 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4348 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4349 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4350 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4351 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 },
4352 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4353 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4354 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4355 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4356 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4357 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4358 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4359 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4360 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4361};
4362
4363static const struct soc15_reg_entry sgpr2_init_regs[] = {
4364 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4365 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4366 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4367 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4368 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 },
4369 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4370 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4371 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4372 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4373 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4374 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4375 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4376 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4377 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4378};
4379
4380static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4381 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4382 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4383 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4384 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4385 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4386 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4387 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4388 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4389 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4390 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4391 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4392 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4393 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4394 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4395 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4396 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4397 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4398 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4399 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4400 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4401 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4402 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4403 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4404 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4405 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4406 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4407 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4408 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4409 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4410 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4411 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4412 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4413 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4414};
4415
4416static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4417{
4418 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4419 int i, r;
4420
4421
4422 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4423 return 0;
4424
4425 r = amdgpu_ring_alloc(ring, 7);
4426 if (r) {
4427 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4428 ring->name, r);
4429 return r;
4430 }
4431
4432 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4433 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4434
4435 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4436 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4437 PACKET3_DMA_DATA_DST_SEL(1) |
4438 PACKET3_DMA_DATA_SRC_SEL(2) |
4439 PACKET3_DMA_DATA_ENGINE(0)));
4440 amdgpu_ring_write(ring, 0);
4441 amdgpu_ring_write(ring, 0);
4442 amdgpu_ring_write(ring, 0);
4443 amdgpu_ring_write(ring, 0);
4444 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4445 adev->gds.gds_size);
4446
4447 amdgpu_ring_commit(ring);
4448
4449 for (i = 0; i < adev->usec_timeout; i++) {
4450 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4451 break;
4452 udelay(1);
4453 }
4454
4455 if (i >= adev->usec_timeout)
4456 r = -ETIMEDOUT;
4457
4458 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4459
4460 return r;
4461}
4462
4463static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4464{
4465 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4466 struct amdgpu_ib ib;
4467 struct dma_fence *f = NULL;
4468 int r, i;
4469 unsigned total_size, vgpr_offset, sgpr_offset;
4470 u64 gpu_addr;
4471
4472 int compute_dim_x = adev->gfx.config.max_shader_engines *
4473 adev->gfx.config.max_cu_per_sh *
4474 adev->gfx.config.max_sh_per_se;
4475 int sgpr_work_group_size = 5;
4476 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4477 int vgpr_init_shader_size;
4478 const u32 *vgpr_init_shader_ptr;
4479 const struct soc15_reg_entry *vgpr_init_regs_ptr;
4480
4481
4482 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4483 return 0;
4484
4485
4486 if (!ring->sched.ready)
4487 return 0;
4488
4489 if (adev->asic_type == CHIP_ARCTURUS) {
4490 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4491 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4492 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4493 } else {
4494 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4495 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4496 vgpr_init_regs_ptr = vgpr_init_regs;
4497 }
4498
4499 total_size =
4500 (gpr_reg_size * 3 + 4 + 5 + 2) * 4;
4501 total_size +=
4502 (gpr_reg_size * 3 + 4 + 5 + 2) * 4;
4503 total_size +=
4504 (gpr_reg_size * 3 + 4 + 5 + 2) * 4;
4505 total_size = ALIGN(total_size, 256);
4506 vgpr_offset = total_size;
4507 total_size += ALIGN(vgpr_init_shader_size, 256);
4508 sgpr_offset = total_size;
4509 total_size += sizeof(sgpr_init_compute_shader);
4510
4511
4512 memset(&ib, 0, sizeof(ib));
4513 r = amdgpu_ib_get(adev, NULL, total_size,
4514 AMDGPU_IB_POOL_DIRECT, &ib);
4515 if (r) {
4516 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4517 return r;
4518 }
4519
4520
4521 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4522 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4523
4524 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4525 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4526
4527
4528 ib.length_dw = 0;
4529
4530
4531
4532 for (i = 0; i < gpr_reg_size; i++) {
4533 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4534 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4535 - PACKET3_SET_SH_REG_START;
4536 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4537 }
4538
4539 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4540 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4541 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4542 - PACKET3_SET_SH_REG_START;
4543 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4544 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4545
4546
4547 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4548 ib.ptr[ib.length_dw++] = compute_dim_x * 2;
4549 ib.ptr[ib.length_dw++] = 1;
4550 ib.ptr[ib.length_dw++] = 1;
4551 ib.ptr[ib.length_dw++] =
4552 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4553
4554
4555 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4556 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4557
4558
4559
4560 for (i = 0; i < gpr_reg_size; i++) {
4561 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4562 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4563 - PACKET3_SET_SH_REG_START;
4564 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4565 }
4566
4567 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4568 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4569 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4570 - PACKET3_SET_SH_REG_START;
4571 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4572 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4573
4574
4575 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4576 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size;
4577 ib.ptr[ib.length_dw++] = 1;
4578 ib.ptr[ib.length_dw++] = 1;
4579 ib.ptr[ib.length_dw++] =
4580 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4581
4582
4583 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4584 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4585
4586
4587
4588 for (i = 0; i < gpr_reg_size; i++) {
4589 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4590 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4591 - PACKET3_SET_SH_REG_START;
4592 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4593 }
4594
4595 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4596 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4597 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4598 - PACKET3_SET_SH_REG_START;
4599 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4600 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4601
4602
4603 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4604 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size;
4605 ib.ptr[ib.length_dw++] = 1;
4606 ib.ptr[ib.length_dw++] = 1;
4607 ib.ptr[ib.length_dw++] =
4608 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4609
4610
4611 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4612 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4613
4614
4615 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4616 if (r) {
4617 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4618 goto fail;
4619 }
4620
4621
4622 r = dma_fence_wait(f, false);
4623 if (r) {
4624 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4625 goto fail;
4626 }
4627
4628fail:
4629 amdgpu_ib_free(adev, &ib, NULL);
4630 dma_fence_put(f);
4631
4632 return r;
4633}
4634
4635static int gfx_v9_0_early_init(void *handle)
4636{
4637 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4638
4639 if (adev->asic_type == CHIP_ARCTURUS)
4640 adev->gfx.num_gfx_rings = 0;
4641 else
4642 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4643 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4644 AMDGPU_MAX_COMPUTE_RINGS);
4645 gfx_v9_0_set_kiq_pm4_funcs(adev);
4646 gfx_v9_0_set_ring_funcs(adev);
4647 gfx_v9_0_set_irq_funcs(adev);
4648 gfx_v9_0_set_gds_init(adev);
4649 gfx_v9_0_set_rlc_funcs(adev);
4650
4651 return 0;
4652}
4653
4654static int gfx_v9_0_ecc_late_init(void *handle)
4655{
4656 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4657 int r;
4658
4659
4660
4661
4662
4663
4664
4665 if (!adev->in_suspend) {
4666 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4667 if (r)
4668 return r;
4669 }
4670
4671
4672 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4673 if (r)
4674 return r;
4675
4676 if (adev->gfx.funcs &&
4677 adev->gfx.funcs->reset_ras_error_count)
4678 adev->gfx.funcs->reset_ras_error_count(adev);
4679
4680 r = amdgpu_gfx_ras_late_init(adev);
4681 if (r)
4682 return r;
4683
4684 return 0;
4685}
4686
4687static int gfx_v9_0_late_init(void *handle)
4688{
4689 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4690 int r;
4691
4692 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4693 if (r)
4694 return r;
4695
4696 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4697 if (r)
4698 return r;
4699
4700 r = gfx_v9_0_ecc_late_init(handle);
4701 if (r)
4702 return r;
4703
4704 return 0;
4705}
4706
4707static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4708{
4709 uint32_t rlc_setting;
4710
4711
4712 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4713 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4714 return false;
4715
4716 return true;
4717}
4718
4719static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4720{
4721 uint32_t data;
4722 unsigned i;
4723
4724 data = RLC_SAFE_MODE__CMD_MASK;
4725 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4726 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4727
4728
4729 for (i = 0; i < adev->usec_timeout; i++) {
4730 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4731 break;
4732 udelay(1);
4733 }
4734}
4735
4736static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4737{
4738 uint32_t data;
4739
4740 data = RLC_SAFE_MODE__CMD_MASK;
4741 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4742}
4743
4744static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4745 bool enable)
4746{
4747 amdgpu_gfx_rlc_enter_safe_mode(adev);
4748
4749 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4750 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4751 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4752 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4753 } else {
4754 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4755 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4756 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4757 }
4758
4759 amdgpu_gfx_rlc_exit_safe_mode(adev);
4760}
4761
4762static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4763 bool enable)
4764{
4765
4766
4767
4768 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4769 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4770 else
4771 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4772
4773 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4774 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4775 else
4776 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4777
4778
4779}
4780
4781static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4782 bool enable)
4783{
4784 uint32_t data, def;
4785
4786 amdgpu_gfx_rlc_enter_safe_mode(adev);
4787
4788
4789 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4790
4791 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4792
4793 if (adev->asic_type != CHIP_VEGA12)
4794 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4795
4796 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4797 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4798 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4799
4800
4801 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4802
4803 if (def != data)
4804 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4805
4806
4807 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4808
4809 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4810 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4811 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4812 if (def != data)
4813 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4814 }
4815
4816 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4817 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4818 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4819 if (def != data)
4820 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4821 }
4822 }
4823 } else {
4824
4825 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4826
4827 if (adev->asic_type != CHIP_VEGA12)
4828 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4829
4830 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4831 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4832 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4833 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4834
4835 if (def != data)
4836 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4837
4838
4839 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4840 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4841 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4842 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4843 }
4844
4845
4846 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4847 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4848 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4849 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4850 }
4851 }
4852
4853 amdgpu_gfx_rlc_exit_safe_mode(adev);
4854}
4855
4856static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4857 bool enable)
4858{
4859 uint32_t data, def;
4860
4861 if (adev->asic_type == CHIP_ARCTURUS)
4862 return;
4863
4864 amdgpu_gfx_rlc_enter_safe_mode(adev);
4865
4866
4867 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4868
4869 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4870
4871 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4872
4873 if (def != data)
4874 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4875
4876
4877 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4878
4879 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4880 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4881 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4882 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4883 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4884 if (def != data)
4885 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4886
4887
4888 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4889 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4890 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4891 if (def != data)
4892 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4893 } else {
4894
4895 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4896
4897 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4898 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4899
4900 if (def != data)
4901 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4902 }
4903
4904 amdgpu_gfx_rlc_exit_safe_mode(adev);
4905}
4906
4907static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4908 bool enable)
4909{
4910 uint32_t def, data;
4911
4912 amdgpu_gfx_rlc_enter_safe_mode(adev);
4913
4914 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4915 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4916
4917 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4918 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4919 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4920 else
4921 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4922
4923 if (def != data)
4924 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4925
4926
4927 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4928
4929 if (adev->asic_type == CHIP_ARCTURUS)
4930 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4931 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4932 else
4933 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4934 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4935 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4936 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4937 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4938 if (def != data)
4939 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4940
4941
4942 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4943 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4944 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4945 if (def != data)
4946 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4947 } else {
4948 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4949
4950 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4951
4952 if (def != data)
4953 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4954 }
4955
4956 amdgpu_gfx_rlc_exit_safe_mode(adev);
4957}
4958
4959static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4960 bool enable)
4961{
4962 if (enable) {
4963
4964
4965
4966 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4967
4968 gfx_v9_0_update_3d_clock_gating(adev, enable);
4969
4970 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4971 } else {
4972
4973
4974
4975 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4976
4977 gfx_v9_0_update_3d_clock_gating(adev, enable);
4978
4979 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4980 }
4981 return 0;
4982}
4983
4984static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
4985{
4986 u32 reg, data;
4987
4988 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
4989 if (amdgpu_sriov_is_pp_one_vf(adev))
4990 data = RREG32_NO_KIQ(reg);
4991 else
4992 data = RREG32(reg);
4993
4994 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4995 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4996
4997 if (amdgpu_sriov_is_pp_one_vf(adev))
4998 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
4999 else
5000 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5001}
5002
5003static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5004 uint32_t offset,
5005 struct soc15_reg_rlcg *entries, int arr_size)
5006{
5007 int i;
5008 uint32_t reg;
5009
5010 if (!entries)
5011 return false;
5012
5013 for (i = 0; i < arr_size; i++) {
5014 const struct soc15_reg_rlcg *entry;
5015
5016 entry = &entries[i];
5017 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5018 if (offset == reg)
5019 return true;
5020 }
5021
5022 return false;
5023}
5024
5025static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5026{
5027 return gfx_v9_0_check_rlcg_range(adev, offset,
5028 (void *)rlcg_access_gc_9_0,
5029 ARRAY_SIZE(rlcg_access_gc_9_0));
5030}
5031
5032static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5033 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5034 .set_safe_mode = gfx_v9_0_set_safe_mode,
5035 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5036 .init = gfx_v9_0_rlc_init,
5037 .get_csb_size = gfx_v9_0_get_csb_size,
5038 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5039 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5040 .resume = gfx_v9_0_rlc_resume,
5041 .stop = gfx_v9_0_rlc_stop,
5042 .reset = gfx_v9_0_rlc_reset,
5043 .start = gfx_v9_0_rlc_start,
5044 .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5045 .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5046 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5047};
5048
5049static int gfx_v9_0_set_powergating_state(void *handle,
5050 enum amd_powergating_state state)
5051{
5052 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5053 bool enable = (state == AMD_PG_STATE_GATE);
5054
5055 switch (adev->asic_type) {
5056 case CHIP_RAVEN:
5057 case CHIP_RENOIR:
5058 if (!enable)
5059 amdgpu_gfx_off_ctrl(adev, false);
5060
5061 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5062 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5063 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5064 } else {
5065 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5066 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5067 }
5068
5069 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5070 gfx_v9_0_enable_cp_power_gating(adev, true);
5071 else
5072 gfx_v9_0_enable_cp_power_gating(adev, false);
5073
5074
5075 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5076
5077
5078 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5079
5080 if (enable)
5081 amdgpu_gfx_off_ctrl(adev, true);
5082 break;
5083 case CHIP_VEGA12:
5084 amdgpu_gfx_off_ctrl(adev, enable);
5085 break;
5086 default:
5087 break;
5088 }
5089
5090 return 0;
5091}
5092
5093static int gfx_v9_0_set_clockgating_state(void *handle,
5094 enum amd_clockgating_state state)
5095{
5096 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5097
5098 if (amdgpu_sriov_vf(adev))
5099 return 0;
5100
5101 switch (adev->asic_type) {
5102 case CHIP_VEGA10:
5103 case CHIP_VEGA12:
5104 case CHIP_VEGA20:
5105 case CHIP_RAVEN:
5106 case CHIP_ARCTURUS:
5107 case CHIP_RENOIR:
5108 gfx_v9_0_update_gfx_clock_gating(adev,
5109 state == AMD_CG_STATE_GATE);
5110 break;
5111 default:
5112 break;
5113 }
5114 return 0;
5115}
5116
5117static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5118{
5119 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5120 int data;
5121
5122 if (amdgpu_sriov_vf(adev))
5123 *flags = 0;
5124
5125
5126 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5127 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5128 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5129
5130
5131 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5132 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5133 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5134
5135
5136 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5137 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5138
5139
5140 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5141 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5142 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5143
5144
5145 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5146 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5147 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5148
5149 if (adev->asic_type != CHIP_ARCTURUS) {
5150
5151 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5152 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5153 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5154
5155
5156 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5157 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5158 }
5159}
5160
5161static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5162{
5163 return ring->adev->wb.wb[ring->rptr_offs];
5164}
5165
5166static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5167{
5168 struct amdgpu_device *adev = ring->adev;
5169 u64 wptr;
5170
5171
5172 if (ring->use_doorbell) {
5173 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5174 } else {
5175 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5176 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5177 }
5178
5179 return wptr;
5180}
5181
5182static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5183{
5184 struct amdgpu_device *adev = ring->adev;
5185
5186 if (ring->use_doorbell) {
5187
5188 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5189 WDOORBELL64(ring->doorbell_index, ring->wptr);
5190 } else {
5191 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5192 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5193 }
5194}
5195
5196static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5197{
5198 struct amdgpu_device *adev = ring->adev;
5199 u32 ref_and_mask, reg_mem_engine;
5200 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5201
5202 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5203 switch (ring->me) {
5204 case 1:
5205 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5206 break;
5207 case 2:
5208 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5209 break;
5210 default:
5211 return;
5212 }
5213 reg_mem_engine = 0;
5214 } else {
5215 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5216 reg_mem_engine = 1;
5217 }
5218
5219 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5220 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5221 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5222 ref_and_mask, ref_and_mask, 0x20);
5223}
5224
5225static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5226 struct amdgpu_job *job,
5227 struct amdgpu_ib *ib,
5228 uint32_t flags)
5229{
5230 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5231 u32 header, control = 0;
5232
5233 if (ib->flags & AMDGPU_IB_FLAG_CE)
5234 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5235 else
5236 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5237
5238 control |= ib->length_dw | (vmid << 24);
5239
5240 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5241 control |= INDIRECT_BUFFER_PRE_ENB(1);
5242
5243 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5244 gfx_v9_0_ring_emit_de_meta(ring);
5245 }
5246
5247 amdgpu_ring_write(ring, header);
5248 BUG_ON(ib->gpu_addr & 0x3);
5249 amdgpu_ring_write(ring,
5250#ifdef __BIG_ENDIAN
5251 (2 << 0) |
5252#endif
5253 lower_32_bits(ib->gpu_addr));
5254 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5255 amdgpu_ring_write(ring, control);
5256}
5257
5258static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5259 struct amdgpu_job *job,
5260 struct amdgpu_ib *ib,
5261 uint32_t flags)
5262{
5263 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5264 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5277 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5278 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5279 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5280 }
5281
5282 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5283 BUG_ON(ib->gpu_addr & 0x3);
5284 amdgpu_ring_write(ring,
5285#ifdef __BIG_ENDIAN
5286 (2 << 0) |
5287#endif
5288 lower_32_bits(ib->gpu_addr));
5289 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5290 amdgpu_ring_write(ring, control);
5291}
5292
5293static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5294 u64 seq, unsigned flags)
5295{
5296 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5297 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5298 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5299
5300
5301 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5302 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5303 EOP_TC_NC_ACTION_EN) :
5304 (EOP_TCL1_ACTION_EN |
5305 EOP_TC_ACTION_EN |
5306 EOP_TC_WB_ACTION_EN |
5307 EOP_TC_MD_ACTION_EN)) |
5308 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5309 EVENT_INDEX(5)));
5310 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5311
5312
5313
5314
5315
5316 if (write64bit)
5317 BUG_ON(addr & 0x7);
5318 else
5319 BUG_ON(addr & 0x3);
5320 amdgpu_ring_write(ring, lower_32_bits(addr));
5321 amdgpu_ring_write(ring, upper_32_bits(addr));
5322 amdgpu_ring_write(ring, lower_32_bits(seq));
5323 amdgpu_ring_write(ring, upper_32_bits(seq));
5324 amdgpu_ring_write(ring, 0);
5325}
5326
5327static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5328{
5329 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5330 uint32_t seq = ring->fence_drv.sync_seq;
5331 uint64_t addr = ring->fence_drv.gpu_addr;
5332
5333 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5334 lower_32_bits(addr), upper_32_bits(addr),
5335 seq, 0xffffffff, 4);
5336}
5337
5338static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5339 unsigned vmid, uint64_t pd_addr)
5340{
5341 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5342
5343
5344 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5345
5346 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5347 amdgpu_ring_write(ring, 0x0);
5348 }
5349}
5350
5351static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5352{
5353 return ring->adev->wb.wb[ring->rptr_offs];
5354}
5355
5356static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5357{
5358 u64 wptr;
5359
5360
5361 if (ring->use_doorbell)
5362 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5363 else
5364 BUG();
5365 return wptr;
5366}
5367
5368static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5369{
5370 struct amdgpu_device *adev = ring->adev;
5371
5372
5373 if (ring->use_doorbell) {
5374 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5375 WDOORBELL64(ring->doorbell_index, ring->wptr);
5376 } else{
5377 BUG();
5378 }
5379}
5380
5381static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5382 u64 seq, unsigned int flags)
5383{
5384 struct amdgpu_device *adev = ring->adev;
5385
5386
5387 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5388
5389
5390 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5391 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5392 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5393 amdgpu_ring_write(ring, lower_32_bits(addr));
5394 amdgpu_ring_write(ring, upper_32_bits(addr));
5395 amdgpu_ring_write(ring, lower_32_bits(seq));
5396
5397 if (flags & AMDGPU_FENCE_FLAG_INT) {
5398
5399 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5400 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5401 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5402 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5403 amdgpu_ring_write(ring, 0);
5404 amdgpu_ring_write(ring, 0x20000000);
5405 }
5406}
5407
5408static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5409{
5410 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5411 amdgpu_ring_write(ring, 0);
5412}
5413
5414static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5415{
5416 struct v9_ce_ib_state ce_payload = {0};
5417 uint64_t csa_addr;
5418 int cnt;
5419
5420 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5421 csa_addr = amdgpu_csa_vaddr(ring->adev);
5422
5423 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5424 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5425 WRITE_DATA_DST_SEL(8) |
5426 WR_CONFIRM) |
5427 WRITE_DATA_CACHE_POLICY(0));
5428 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5429 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5430 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5431}
5432
5433static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5434{
5435 struct v9_de_ib_state de_payload = {0};
5436 uint64_t csa_addr, gds_addr;
5437 int cnt;
5438
5439 csa_addr = amdgpu_csa_vaddr(ring->adev);
5440 gds_addr = csa_addr + 4096;
5441 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5442 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5443
5444 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5445 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5446 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5447 WRITE_DATA_DST_SEL(8) |
5448 WR_CONFIRM) |
5449 WRITE_DATA_CACHE_POLICY(0));
5450 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5451 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5452 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5453}
5454
5455static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5456 bool secure)
5457{
5458 uint32_t v = secure ? FRAME_TMZ : 0;
5459
5460 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5461 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5462}
5463
5464static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5465{
5466 uint32_t dw2 = 0;
5467
5468 if (amdgpu_sriov_vf(ring->adev))
5469 gfx_v9_0_ring_emit_ce_meta(ring);
5470
5471 dw2 |= 0x80000000;
5472 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5473
5474 dw2 |= 0x8001;
5475
5476 dw2 |= 0x01000000;
5477
5478 dw2 |= 0x10002;
5479
5480
5481 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5482 dw2 |= 0x10000000;
5483 } else {
5484
5485
5486
5487 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5488 dw2 |= 0x10000000;
5489 }
5490
5491 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5492 amdgpu_ring_write(ring, dw2);
5493 amdgpu_ring_write(ring, 0);
5494}
5495
5496static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5497{
5498 unsigned ret;
5499 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5500 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5501 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5502 amdgpu_ring_write(ring, 0);
5503 ret = ring->wptr & ring->buf_mask;
5504 amdgpu_ring_write(ring, 0x55aa55aa);
5505 return ret;
5506}
5507
5508static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5509{
5510 unsigned cur;
5511 BUG_ON(offset > ring->buf_mask);
5512 BUG_ON(ring->ring[offset] != 0x55aa55aa);
5513
5514 cur = (ring->wptr & ring->buf_mask) - 1;
5515 if (likely(cur > offset))
5516 ring->ring[offset] = cur - offset;
5517 else
5518 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5519}
5520
5521static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5522 uint32_t reg_val_offs)
5523{
5524 struct amdgpu_device *adev = ring->adev;
5525
5526 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5527 amdgpu_ring_write(ring, 0 |
5528 (5 << 8) |
5529 (1 << 20));
5530 amdgpu_ring_write(ring, reg);
5531 amdgpu_ring_write(ring, 0);
5532 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5533 reg_val_offs * 4));
5534 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5535 reg_val_offs * 4));
5536}
5537
5538static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5539 uint32_t val)
5540{
5541 uint32_t cmd = 0;
5542
5543 switch (ring->funcs->type) {
5544 case AMDGPU_RING_TYPE_GFX:
5545 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5546 break;
5547 case AMDGPU_RING_TYPE_KIQ:
5548 cmd = (1 << 16);
5549 break;
5550 default:
5551 cmd = WR_CONFIRM;
5552 break;
5553 }
5554 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5555 amdgpu_ring_write(ring, cmd);
5556 amdgpu_ring_write(ring, reg);
5557 amdgpu_ring_write(ring, 0);
5558 amdgpu_ring_write(ring, val);
5559}
5560
5561static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5562 uint32_t val, uint32_t mask)
5563{
5564 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5565}
5566
5567static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5568 uint32_t reg0, uint32_t reg1,
5569 uint32_t ref, uint32_t mask)
5570{
5571 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5572 struct amdgpu_device *adev = ring->adev;
5573 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5574 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5575
5576 if (fw_version_ok)
5577 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5578 ref, mask, 0x20);
5579 else
5580 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5581 ref, mask);
5582}
5583
5584static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5585{
5586 struct amdgpu_device *adev = ring->adev;
5587 uint32_t value = 0;
5588
5589 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5590 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5591 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5592 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5593 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5594}
5595
5596static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5597 enum amdgpu_interrupt_state state)
5598{
5599 switch (state) {
5600 case AMDGPU_IRQ_STATE_DISABLE:
5601 case AMDGPU_IRQ_STATE_ENABLE:
5602 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5603 TIME_STAMP_INT_ENABLE,
5604 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5605 break;
5606 default:
5607 break;
5608 }
5609}
5610
5611static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5612 int me, int pipe,
5613 enum amdgpu_interrupt_state state)
5614{
5615 u32 mec_int_cntl, mec_int_cntl_reg;
5616
5617
5618
5619
5620
5621
5622
5623 if (me == 1) {
5624 switch (pipe) {
5625 case 0:
5626 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5627 break;
5628 case 1:
5629 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5630 break;
5631 case 2:
5632 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5633 break;
5634 case 3:
5635 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5636 break;
5637 default:
5638 DRM_DEBUG("invalid pipe %d\n", pipe);
5639 return;
5640 }
5641 } else {
5642 DRM_DEBUG("invalid me %d\n", me);
5643 return;
5644 }
5645
5646 switch (state) {
5647 case AMDGPU_IRQ_STATE_DISABLE:
5648 mec_int_cntl = RREG32(mec_int_cntl_reg);
5649 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5650 TIME_STAMP_INT_ENABLE, 0);
5651 WREG32(mec_int_cntl_reg, mec_int_cntl);
5652 break;
5653 case AMDGPU_IRQ_STATE_ENABLE:
5654 mec_int_cntl = RREG32(mec_int_cntl_reg);
5655 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5656 TIME_STAMP_INT_ENABLE, 1);
5657 WREG32(mec_int_cntl_reg, mec_int_cntl);
5658 break;
5659 default:
5660 break;
5661 }
5662}
5663
5664static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5665 struct amdgpu_irq_src *source,
5666 unsigned type,
5667 enum amdgpu_interrupt_state state)
5668{
5669 switch (state) {
5670 case AMDGPU_IRQ_STATE_DISABLE:
5671 case AMDGPU_IRQ_STATE_ENABLE:
5672 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5673 PRIV_REG_INT_ENABLE,
5674 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5675 break;
5676 default:
5677 break;
5678 }
5679
5680 return 0;
5681}
5682
5683static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5684 struct amdgpu_irq_src *source,
5685 unsigned type,
5686 enum amdgpu_interrupt_state state)
5687{
5688 switch (state) {
5689 case AMDGPU_IRQ_STATE_DISABLE:
5690 case AMDGPU_IRQ_STATE_ENABLE:
5691 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5692 PRIV_INSTR_INT_ENABLE,
5693 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5694 break;
5695 default:
5696 break;
5697 }
5698
5699 return 0;
5700}
5701
5702#define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
5703 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5704 CP_ECC_ERROR_INT_ENABLE, 1)
5705
5706#define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
5707 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5708 CP_ECC_ERROR_INT_ENABLE, 0)
5709
5710static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5711 struct amdgpu_irq_src *source,
5712 unsigned type,
5713 enum amdgpu_interrupt_state state)
5714{
5715 switch (state) {
5716 case AMDGPU_IRQ_STATE_DISABLE:
5717 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5718 CP_ECC_ERROR_INT_ENABLE, 0);
5719 DISABLE_ECC_ON_ME_PIPE(1, 0);
5720 DISABLE_ECC_ON_ME_PIPE(1, 1);
5721 DISABLE_ECC_ON_ME_PIPE(1, 2);
5722 DISABLE_ECC_ON_ME_PIPE(1, 3);
5723 break;
5724
5725 case AMDGPU_IRQ_STATE_ENABLE:
5726 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5727 CP_ECC_ERROR_INT_ENABLE, 1);
5728 ENABLE_ECC_ON_ME_PIPE(1, 0);
5729 ENABLE_ECC_ON_ME_PIPE(1, 1);
5730 ENABLE_ECC_ON_ME_PIPE(1, 2);
5731 ENABLE_ECC_ON_ME_PIPE(1, 3);
5732 break;
5733 default:
5734 break;
5735 }
5736
5737 return 0;
5738}
5739
5740
5741static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5742 struct amdgpu_irq_src *src,
5743 unsigned type,
5744 enum amdgpu_interrupt_state state)
5745{
5746 switch (type) {
5747 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5748 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5749 break;
5750 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5751 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5752 break;
5753 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5754 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5755 break;
5756 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5757 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5758 break;
5759 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5760 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5761 break;
5762 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5763 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5764 break;
5765 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5766 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5767 break;
5768 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5769 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5770 break;
5771 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5772 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5773 break;
5774 default:
5775 break;
5776 }
5777 return 0;
5778}
5779
5780static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5781 struct amdgpu_irq_src *source,
5782 struct amdgpu_iv_entry *entry)
5783{
5784 int i;
5785 u8 me_id, pipe_id, queue_id;
5786 struct amdgpu_ring *ring;
5787
5788 DRM_DEBUG("IH: CP EOP\n");
5789 me_id = (entry->ring_id & 0x0c) >> 2;
5790 pipe_id = (entry->ring_id & 0x03) >> 0;
5791 queue_id = (entry->ring_id & 0x70) >> 4;
5792
5793 switch (me_id) {
5794 case 0:
5795 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5796 break;
5797 case 1:
5798 case 2:
5799 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5800 ring = &adev->gfx.compute_ring[i];
5801
5802
5803
5804 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5805 amdgpu_fence_process(ring);
5806 }
5807 break;
5808 }
5809 return 0;
5810}
5811
5812static void gfx_v9_0_fault(struct amdgpu_device *adev,
5813 struct amdgpu_iv_entry *entry)
5814{
5815 u8 me_id, pipe_id, queue_id;
5816 struct amdgpu_ring *ring;
5817 int i;
5818
5819 me_id = (entry->ring_id & 0x0c) >> 2;
5820 pipe_id = (entry->ring_id & 0x03) >> 0;
5821 queue_id = (entry->ring_id & 0x70) >> 4;
5822
5823 switch (me_id) {
5824 case 0:
5825 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5826 break;
5827 case 1:
5828 case 2:
5829 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5830 ring = &adev->gfx.compute_ring[i];
5831 if (ring->me == me_id && ring->pipe == pipe_id &&
5832 ring->queue == queue_id)
5833 drm_sched_fault(&ring->sched);
5834 }
5835 break;
5836 }
5837}
5838
5839static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5840 struct amdgpu_irq_src *source,
5841 struct amdgpu_iv_entry *entry)
5842{
5843 DRM_ERROR("Illegal register access in command stream\n");
5844 gfx_v9_0_fault(adev, entry);
5845 return 0;
5846}
5847
5848static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5849 struct amdgpu_irq_src *source,
5850 struct amdgpu_iv_entry *entry)
5851{
5852 DRM_ERROR("Illegal instruction in command stream\n");
5853 gfx_v9_0_fault(adev, entry);
5854 return 0;
5855}
5856
5857
5858static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5859 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5860 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5861 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5862 },
5863 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5864 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5865 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5866 },
5867 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5868 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5869 0, 0
5870 },
5871 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5872 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5873 0, 0
5874 },
5875 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5876 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5877 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5878 },
5879 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5880 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5881 0, 0
5882 },
5883 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5884 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5885 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5886 },
5887 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5888 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5889 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5890 },
5891 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5892 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5893 0, 0
5894 },
5895 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5896 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5897 0, 0
5898 },
5899 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5900 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5901 0, 0
5902 },
5903 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5904 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5905 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5906 },
5907 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5908 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5909 0, 0
5910 },
5911 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5912 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5913 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5914 },
5915 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5916 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5917 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5918 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5919 },
5920 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5921 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5922 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5923 0, 0
5924 },
5925 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
5926 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5927 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
5928 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
5929 },
5930 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
5931 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5932 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
5933 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
5934 },
5935 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
5936 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5937 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
5938 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
5939 },
5940 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
5941 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5942 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
5943 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
5944 },
5945 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
5946 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
5947 0, 0
5948 },
5949 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5950 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
5951 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
5952 },
5953 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5954 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
5955 0, 0
5956 },
5957 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5958 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
5959 0, 0
5960 },
5961 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5962 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
5963 0, 0
5964 },
5965 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5966 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
5967 0, 0
5968 },
5969 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5970 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
5971 0, 0
5972 },
5973 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5974 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
5975 0, 0
5976 },
5977 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5978 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
5979 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
5980 },
5981 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5982 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
5983 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
5984 },
5985 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5986 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
5987 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
5988 },
5989 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5990 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
5991 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
5992 },
5993 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5994 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
5995 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
5996 },
5997 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5998 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
5999 0, 0
6000 },
6001 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6002 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6003 0, 0
6004 },
6005 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6006 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6007 0, 0
6008 },
6009 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6010 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6011 0, 0
6012 },
6013 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6014 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6015 0, 0
6016 },
6017 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6018 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6019 0, 0
6020 },
6021 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6022 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6023 0, 0
6024 },
6025 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6026 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6027 0, 0
6028 },
6029 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6030 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6031 0, 0
6032 },
6033 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6034 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6035 0, 0
6036 },
6037 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6038 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6039 0, 0
6040 },
6041 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6042 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6043 0, 0
6044 },
6045 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6046 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6047 0, 0
6048 },
6049 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6050 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6051 0, 0
6052 },
6053 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6054 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6055 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6056 },
6057 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6058 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6059 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6060 },
6061 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6062 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6063 0, 0
6064 },
6065 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6066 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6067 0, 0
6068 },
6069 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6070 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6071 0, 0
6072 },
6073 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6074 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6075 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6076 },
6077 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6078 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6079 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6080 },
6081 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6082 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6083 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6084 },
6085 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6086 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6087 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6088 },
6089 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6090 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6091 0, 0
6092 },
6093 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6094 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6095 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6096 },
6097 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6098 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6099 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6100 },
6101 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6102 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6103 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6104 },
6105 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6106 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6107 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6108 },
6109 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6110 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6111 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6112 },
6113 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6114 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6115 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6116 },
6117 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6118 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6119 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6120 },
6121 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6122 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6123 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6124 },
6125 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6126 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6127 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6128 },
6129 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6130 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6131 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6132 },
6133 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6134 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6135 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6136 },
6137 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6138 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6139 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6140 },
6141 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6142 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6143 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6144 },
6145 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6146 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6147 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6148 },
6149 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6150 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6151 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6152 },
6153 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6154 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6155 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6156 },
6157 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6158 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6159 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6160 },
6161 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6162 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6163 0, 0
6164 },
6165 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6166 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6167 0, 0
6168 },
6169 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6170 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6171 0, 0
6172 },
6173 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6174 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6175 0, 0
6176 },
6177 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6178 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6179 0, 0
6180 },
6181 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6182 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6183 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6184 },
6185 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6186 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6187 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6188 },
6189 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6190 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6191 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6192 },
6193 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6194 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6195 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6196 },
6197 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6198 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6199 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6200 },
6201 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6202 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6203 0, 0
6204 },
6205 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6206 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6207 0, 0
6208 },
6209 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6210 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6211 0, 0
6212 },
6213 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6214 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6215 0, 0
6216 },
6217 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6218 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6219 0, 0
6220 },
6221 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6222 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6223 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6224 },
6225 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6226 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6227 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6228 },
6229 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6230 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6231 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6232 },
6233 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6234 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6235 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6236 },
6237 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6238 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6239 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6240 },
6241 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6242 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6243 0, 0
6244 },
6245 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6246 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6247 0, 0
6248 },
6249 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6250 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6251 0, 0
6252 },
6253 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6254 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6255 0, 0
6256 },
6257 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6258 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6259 0, 0
6260 },
6261 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6262 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6263 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6264 },
6265 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6266 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6267 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6268 },
6269 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6270 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6271 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6272 },
6273 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6274 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6275 0, 0
6276 },
6277 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6278 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6279 0, 0
6280 },
6281 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6282 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6283 0, 0
6284 },
6285 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6286 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6287 0, 0
6288 },
6289 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6290 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6291 0, 0
6292 },
6293 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6294 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6295 0, 0
6296 }
6297};
6298
6299static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6300 void *inject_if)
6301{
6302 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6303 int ret;
6304 struct ta_ras_trigger_error_input block_info = { 0 };
6305
6306 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6307 return -EINVAL;
6308
6309 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6310 return -EINVAL;
6311
6312 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6313 return -EPERM;
6314
6315 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6316 info->head.type)) {
6317 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6318 ras_gfx_subblocks[info->head.sub_block_index].name,
6319 info->head.type);
6320 return -EPERM;
6321 }
6322
6323 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6324 info->head.type)) {
6325 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6326 ras_gfx_subblocks[info->head.sub_block_index].name,
6327 info->head.type);
6328 return -EPERM;
6329 }
6330
6331 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6332 block_info.sub_block_index =
6333 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6334 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6335 block_info.address = info->address;
6336 block_info.value = info->value;
6337
6338 mutex_lock(&adev->grbm_idx_mutex);
6339 ret = psp_ras_trigger_error(&adev->psp, &block_info);
6340 mutex_unlock(&adev->grbm_idx_mutex);
6341
6342 return ret;
6343}
6344
6345static const char *vml2_mems[] = {
6346 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6347 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6348 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6349 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6350 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6351 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6352 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6353 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6354 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6355 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6356 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6357 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6358 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6359 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6360 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6361 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6362};
6363
6364static const char *vml2_walker_mems[] = {
6365 "UTC_VML2_CACHE_PDE0_MEM0",
6366 "UTC_VML2_CACHE_PDE0_MEM1",
6367 "UTC_VML2_CACHE_PDE1_MEM0",
6368 "UTC_VML2_CACHE_PDE1_MEM1",
6369 "UTC_VML2_CACHE_PDE2_MEM0",
6370 "UTC_VML2_CACHE_PDE2_MEM1",
6371 "UTC_VML2_RDIF_LOG_FIFO",
6372};
6373
6374static const char *atc_l2_cache_2m_mems[] = {
6375 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6376 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6377 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6378 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6379};
6380
6381static const char *atc_l2_cache_4k_mems[] = {
6382 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6383 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6384 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6385 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6386 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6387 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6388 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6389 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6390 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6391 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6392 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6393 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6394 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6395 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6396 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6397 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6398 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6399 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6400 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6401 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6402 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6403 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6404 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6405 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6406 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6407 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6408 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6409 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6410 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6411 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6412 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6413 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6414};
6415
6416static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6417 struct ras_err_data *err_data)
6418{
6419 uint32_t i, data;
6420 uint32_t sec_count, ded_count;
6421
6422 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6423 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6424 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6425 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6426 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6427 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6428 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6429 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6430
6431 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6432 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6433 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6434
6435 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6436 if (sec_count) {
6437 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6438 "SEC %d\n", i, vml2_mems[i], sec_count);
6439 err_data->ce_count += sec_count;
6440 }
6441
6442 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6443 if (ded_count) {
6444 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6445 "DED %d\n", i, vml2_mems[i], ded_count);
6446 err_data->ue_count += ded_count;
6447 }
6448 }
6449
6450 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6451 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6452 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6453
6454 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6455 SEC_COUNT);
6456 if (sec_count) {
6457 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6458 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6459 err_data->ce_count += sec_count;
6460 }
6461
6462 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6463 DED_COUNT);
6464 if (ded_count) {
6465 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6466 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6467 err_data->ue_count += ded_count;
6468 }
6469 }
6470
6471 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6472 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6473 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6474
6475 sec_count = (data & 0x00006000L) >> 0xd;
6476 if (sec_count) {
6477 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6478 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6479 sec_count);
6480 err_data->ce_count += sec_count;
6481 }
6482 }
6483
6484 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6485 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6486 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6487
6488 sec_count = (data & 0x00006000L) >> 0xd;
6489 if (sec_count) {
6490 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6491 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6492 sec_count);
6493 err_data->ce_count += sec_count;
6494 }
6495
6496 ded_count = (data & 0x00018000L) >> 0xf;
6497 if (ded_count) {
6498 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6499 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6500 ded_count);
6501 err_data->ue_count += ded_count;
6502 }
6503 }
6504
6505 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6506 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6507 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6508 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6509
6510 return 0;
6511}
6512
6513static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6514 const struct soc15_reg_entry *reg,
6515 uint32_t se_id, uint32_t inst_id, uint32_t value,
6516 uint32_t *sec_count, uint32_t *ded_count)
6517{
6518 uint32_t i;
6519 uint32_t sec_cnt, ded_cnt;
6520
6521 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6522 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6523 gfx_v9_0_ras_fields[i].seg != reg->seg ||
6524 gfx_v9_0_ras_fields[i].inst != reg->inst)
6525 continue;
6526
6527 sec_cnt = (value &
6528 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6529 gfx_v9_0_ras_fields[i].sec_count_shift;
6530 if (sec_cnt) {
6531 dev_info(adev->dev, "GFX SubBlock %s, "
6532 "Instance[%d][%d], SEC %d\n",
6533 gfx_v9_0_ras_fields[i].name,
6534 se_id, inst_id,
6535 sec_cnt);
6536 *sec_count += sec_cnt;
6537 }
6538
6539 ded_cnt = (value &
6540 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6541 gfx_v9_0_ras_fields[i].ded_count_shift;
6542 if (ded_cnt) {
6543 dev_info(adev->dev, "GFX SubBlock %s, "
6544 "Instance[%d][%d], DED %d\n",
6545 gfx_v9_0_ras_fields[i].name,
6546 se_id, inst_id,
6547 ded_cnt);
6548 *ded_count += ded_cnt;
6549 }
6550 }
6551
6552 return 0;
6553}
6554
6555static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6556{
6557 int i, j, k;
6558
6559 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6560 return;
6561
6562
6563 mutex_lock(&adev->grbm_idx_mutex);
6564 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6565 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6566 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6567 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6568 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6569 }
6570 }
6571 }
6572 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6573 mutex_unlock(&adev->grbm_idx_mutex);
6574
6575 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6576 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6577 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6578 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6579 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6580 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6581 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6582 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6583
6584 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6585 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6586 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6587 }
6588
6589 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6590 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6591 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6592 }
6593
6594 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6595 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6596 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6597 }
6598
6599 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6600 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6601 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6602 }
6603
6604 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6605 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6606 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6607 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6608}
6609
6610static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6611 void *ras_error_status)
6612{
6613 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6614 uint32_t sec_count = 0, ded_count = 0;
6615 uint32_t i, j, k;
6616 uint32_t reg_value;
6617
6618 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6619 return -EINVAL;
6620
6621 err_data->ue_count = 0;
6622 err_data->ce_count = 0;
6623
6624 mutex_lock(&adev->grbm_idx_mutex);
6625
6626 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6627 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6628 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6629 gfx_v9_0_select_se_sh(adev, j, 0, k);
6630 reg_value =
6631 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6632 if (reg_value)
6633 gfx_v9_0_ras_error_count(adev,
6634 &gfx_v9_0_edc_counter_regs[i],
6635 j, k, reg_value,
6636 &sec_count, &ded_count);
6637 }
6638 }
6639 }
6640
6641 err_data->ce_count += sec_count;
6642 err_data->ue_count += ded_count;
6643
6644 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6645 mutex_unlock(&adev->grbm_idx_mutex);
6646
6647 gfx_v9_0_query_utc_edc_status(adev, err_data);
6648
6649 return 0;
6650}
6651
6652static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6653{
6654 const unsigned int cp_coher_cntl =
6655 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6656 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6657 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6658 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6659 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6660
6661
6662 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6663 amdgpu_ring_write(ring, cp_coher_cntl);
6664 amdgpu_ring_write(ring, 0xffffffff);
6665 amdgpu_ring_write(ring, 0xffffff);
6666 amdgpu_ring_write(ring, 0);
6667 amdgpu_ring_write(ring, 0);
6668 amdgpu_ring_write(ring, 0x0000000A);
6669}
6670
6671static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6672 uint32_t pipe, bool enable)
6673{
6674 struct amdgpu_device *adev = ring->adev;
6675 uint32_t val;
6676 uint32_t wcl_cs_reg;
6677
6678
6679 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6680
6681 switch (pipe) {
6682 case 0:
6683 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6684 break;
6685 case 1:
6686 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6687 break;
6688 case 2:
6689 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6690 break;
6691 case 3:
6692 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6693 break;
6694 default:
6695 DRM_DEBUG("invalid pipe %d\n", pipe);
6696 return;
6697 }
6698
6699 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6700
6701}
6702static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6703{
6704 struct amdgpu_device *adev = ring->adev;
6705 uint32_t val;
6706 int i;
6707
6708
6709
6710
6711
6712
6713 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6714 amdgpu_ring_emit_wreg(ring,
6715 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6716 val);
6717
6718
6719
6720
6721
6722
6723 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6724 if (i != ring->pipe)
6725 gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
6726
6727 }
6728}
6729
6730static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6731 .name = "gfx_v9_0",
6732 .early_init = gfx_v9_0_early_init,
6733 .late_init = gfx_v9_0_late_init,
6734 .sw_init = gfx_v9_0_sw_init,
6735 .sw_fini = gfx_v9_0_sw_fini,
6736 .hw_init = gfx_v9_0_hw_init,
6737 .hw_fini = gfx_v9_0_hw_fini,
6738 .suspend = gfx_v9_0_suspend,
6739 .resume = gfx_v9_0_resume,
6740 .is_idle = gfx_v9_0_is_idle,
6741 .wait_for_idle = gfx_v9_0_wait_for_idle,
6742 .soft_reset = gfx_v9_0_soft_reset,
6743 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6744 .set_powergating_state = gfx_v9_0_set_powergating_state,
6745 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6746};
6747
6748static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6749 .type = AMDGPU_RING_TYPE_GFX,
6750 .align_mask = 0xff,
6751 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6752 .support_64bit_ptrs = true,
6753 .vmhub = AMDGPU_GFXHUB_0,
6754 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6755 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6756 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6757 .emit_frame_size =
6758 5 +
6759 7 +
6760 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6761 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6762 2 +
6763 8 +
6764 20 +
6765 4 +
6766
6767
6768 5 +
6769 7 +
6770 4 +
6771 14 +
6772 31 +
6773 3 +
6774 5 +
6775 8 + 8 +
6776 2 +
6777 7,
6778 .emit_ib_size = 4,
6779 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6780 .emit_fence = gfx_v9_0_ring_emit_fence,
6781 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6782 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6783 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6784 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6785 .test_ring = gfx_v9_0_ring_test_ring,
6786 .test_ib = gfx_v9_0_ring_test_ib,
6787 .insert_nop = amdgpu_ring_insert_nop,
6788 .pad_ib = amdgpu_ring_generic_pad_ib,
6789 .emit_switch_buffer = gfx_v9_ring_emit_sb,
6790 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6791 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6792 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6793 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6794 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6795 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6796 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6797 .soft_recovery = gfx_v9_0_ring_soft_recovery,
6798 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6799};
6800
6801static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6802 .type = AMDGPU_RING_TYPE_COMPUTE,
6803 .align_mask = 0xff,
6804 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6805 .support_64bit_ptrs = true,
6806 .vmhub = AMDGPU_GFXHUB_0,
6807 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6808 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6809 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6810 .emit_frame_size =
6811 20 +
6812 7 +
6813 5 +
6814 7 +
6815 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6816 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6817 2 +
6818 8 + 8 + 8 +
6819 7 +
6820 5 +
6821 15,
6822 .emit_ib_size = 7,
6823 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6824 .emit_fence = gfx_v9_0_ring_emit_fence,
6825 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6826 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6827 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6828 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6829 .test_ring = gfx_v9_0_ring_test_ring,
6830 .test_ib = gfx_v9_0_ring_test_ib,
6831 .insert_nop = amdgpu_ring_insert_nop,
6832 .pad_ib = amdgpu_ring_generic_pad_ib,
6833 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6834 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6835 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6836 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
6837 .emit_wave_limit = gfx_v9_0_emit_wave_limit,
6838};
6839
6840static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6841 .type = AMDGPU_RING_TYPE_KIQ,
6842 .align_mask = 0xff,
6843 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6844 .support_64bit_ptrs = true,
6845 .vmhub = AMDGPU_GFXHUB_0,
6846 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6847 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6848 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6849 .emit_frame_size =
6850 20 +
6851 7 +
6852 5 +
6853 7 +
6854 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6855 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6856 2 +
6857 8 + 8 + 8,
6858 .emit_ib_size = 7,
6859 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6860 .test_ring = gfx_v9_0_ring_test_ring,
6861 .insert_nop = amdgpu_ring_insert_nop,
6862 .pad_ib = amdgpu_ring_generic_pad_ib,
6863 .emit_rreg = gfx_v9_0_ring_emit_rreg,
6864 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6865 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6866 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6867};
6868
6869static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6870{
6871 int i;
6872
6873 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6874
6875 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6876 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6877
6878 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6879 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6880}
6881
6882static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6883 .set = gfx_v9_0_set_eop_interrupt_state,
6884 .process = gfx_v9_0_eop_irq,
6885};
6886
6887static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6888 .set = gfx_v9_0_set_priv_reg_fault_state,
6889 .process = gfx_v9_0_priv_reg_irq,
6890};
6891
6892static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6893 .set = gfx_v9_0_set_priv_inst_fault_state,
6894 .process = gfx_v9_0_priv_inst_irq,
6895};
6896
6897static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6898 .set = gfx_v9_0_set_cp_ecc_error_state,
6899 .process = amdgpu_gfx_cp_ecc_error_irq,
6900};
6901
6902
6903static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6904{
6905 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6906 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6907
6908 adev->gfx.priv_reg_irq.num_types = 1;
6909 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6910
6911 adev->gfx.priv_inst_irq.num_types = 1;
6912 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6913
6914 adev->gfx.cp_ecc_error_irq.num_types = 2;
6915 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6916}
6917
6918static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6919{
6920 switch (adev->asic_type) {
6921 case CHIP_VEGA10:
6922 case CHIP_VEGA12:
6923 case CHIP_VEGA20:
6924 case CHIP_RAVEN:
6925 case CHIP_ARCTURUS:
6926 case CHIP_RENOIR:
6927 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
6928 break;
6929 default:
6930 break;
6931 }
6932}
6933
6934static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
6935{
6936
6937 switch (adev->asic_type) {
6938 case CHIP_VEGA10:
6939 case CHIP_VEGA12:
6940 case CHIP_VEGA20:
6941 adev->gds.gds_size = 0x10000;
6942 break;
6943 case CHIP_RAVEN:
6944 case CHIP_ARCTURUS:
6945 adev->gds.gds_size = 0x1000;
6946 break;
6947 default:
6948 adev->gds.gds_size = 0x10000;
6949 break;
6950 }
6951
6952 switch (adev->asic_type) {
6953 case CHIP_VEGA10:
6954 case CHIP_VEGA20:
6955 adev->gds.gds_compute_max_wave_id = 0x7ff;
6956 break;
6957 case CHIP_VEGA12:
6958 adev->gds.gds_compute_max_wave_id = 0x27f;
6959 break;
6960 case CHIP_RAVEN:
6961 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
6962 adev->gds.gds_compute_max_wave_id = 0x77;
6963 else
6964 adev->gds.gds_compute_max_wave_id = 0x15f;
6965 break;
6966 case CHIP_ARCTURUS:
6967 adev->gds.gds_compute_max_wave_id = 0xfff;
6968 break;
6969 default:
6970
6971 adev->gds.gds_compute_max_wave_id = 0x7ff;
6972 break;
6973 }
6974
6975 adev->gds.gws_size = 64;
6976 adev->gds.oa_size = 16;
6977}
6978
6979static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
6980 u32 bitmap)
6981{
6982 u32 data;
6983
6984 if (!bitmap)
6985 return;
6986
6987 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6988 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6989
6990 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
6991}
6992
6993static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
6994{
6995 u32 data, mask;
6996
6997 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
6998 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
6999
7000 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7001 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7002
7003 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7004
7005 return (~data) & mask;
7006}
7007
7008static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7009 struct amdgpu_cu_info *cu_info)
7010{
7011 int i, j, k, counter, active_cu_number = 0;
7012 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7013 unsigned disable_masks[4 * 4];
7014
7015 if (!adev || !cu_info)
7016 return -EINVAL;
7017
7018
7019
7020
7021 if (adev->gfx.config.max_shader_engines *
7022 adev->gfx.config.max_sh_per_se > 16)
7023 return -EINVAL;
7024
7025 amdgpu_gfx_parse_disable_cu(disable_masks,
7026 adev->gfx.config.max_shader_engines,
7027 adev->gfx.config.max_sh_per_se);
7028
7029 mutex_lock(&adev->grbm_idx_mutex);
7030 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7031 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7032 mask = 1;
7033 ao_bitmap = 0;
7034 counter = 0;
7035 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
7036 gfx_v9_0_set_user_cu_inactive_bitmap(
7037 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7038 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052 cu_info->bitmap[i % 4][j + i / 4] = bitmap;
7053
7054 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7055 if (bitmap & mask) {
7056 if (counter < adev->gfx.config.max_cu_per_sh)
7057 ao_bitmap |= mask;
7058 counter ++;
7059 }
7060 mask <<= 1;
7061 }
7062 active_cu_number += counter;
7063 if (i < 2 && j < 2)
7064 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7065 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7066 }
7067 }
7068 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7069 mutex_unlock(&adev->grbm_idx_mutex);
7070
7071 cu_info->number = active_cu_number;
7072 cu_info->ao_cu_mask = ao_cu_mask;
7073 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7074
7075 return 0;
7076}
7077
7078const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7079{
7080 .type = AMD_IP_BLOCK_TYPE_GFX,
7081 .major = 9,
7082 .minor = 0,
7083 .rev = 0,
7084 .funcs = &gfx_v9_0_ip_funcs,
7085};
7086
