1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/firmware.h>
25#include <linux/pci.h>
26
27#include <drm/drm_cache.h>
28
29#include "amdgpu.h"
30#include "gmc_v9_0.h"
31#include "amdgpu_atomfirmware.h"
32#include "amdgpu_gem.h"
33
34#include "gc/gc_9_0_sh_mask.h"
35#include "dce/dce_12_0_offset.h"
36#include "dce/dce_12_0_sh_mask.h"
37#include "vega10_enum.h"
38#include "mmhub/mmhub_1_0_offset.h"
39#include "athub/athub_1_0_sh_mask.h"
40#include "athub/athub_1_0_offset.h"
41#include "oss/osssys_4_0_offset.h"
42
43#include "soc15.h"
44#include "soc15d.h"
45#include "soc15_common.h"
46#include "umc/umc_6_0_sh_mask.h"
47
48#include "gfxhub_v1_0.h"
49#include "mmhub_v1_0.h"
50#include "athub_v1_0.h"
51#include "gfxhub_v1_1.h"
52#include "mmhub_v9_4.h"
53#include "mmhub_v1_7.h"
54#include "umc_v6_1.h"
55#include "umc_v6_0.h"
56#include "umc_v6_7.h"
57#include "hdp_v4_0.h"
58#include "mca_v3_0.h"
59
60#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
61
62#include "amdgpu_ras.h"
63#include "amdgpu_xgmi.h"
64
65
66#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d
67#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
68#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0
69#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10
70#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL
71#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L
72#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d
73#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2
74
75
76static const char *gfxhub_client_ids[] = {
77 "CB",
78 "DB",
79 "IA",
80 "WD",
81 "CPF",
82 "CPC",
83 "CPG",
84 "RLC",
85 "TCP",
86 "SQC (inst)",
87 "SQC (data)",
88 "SQG",
89 "PA",
90};
91
92static const char *mmhub_client_ids_raven[][2] = {
93 [0][0] = "MP1",
94 [1][0] = "MP0",
95 [2][0] = "VCN",
96 [3][0] = "VCNU",
97 [4][0] = "HDP",
98 [5][0] = "DCE",
99 [13][0] = "UTCL2",
100 [19][0] = "TLS",
101 [26][0] = "OSS",
102 [27][0] = "SDMA0",
103 [0][1] = "MP1",
104 [1][1] = "MP0",
105 [2][1] = "VCN",
106 [3][1] = "VCNU",
107 [4][1] = "HDP",
108 [5][1] = "XDP",
109 [6][1] = "DBGU0",
110 [7][1] = "DCE",
111 [8][1] = "DCEDWB0",
112 [9][1] = "DCEDWB1",
113 [26][1] = "OSS",
114 [27][1] = "SDMA0",
115};
116
117static const char *mmhub_client_ids_renoir[][2] = {
118 [0][0] = "MP1",
119 [1][0] = "MP0",
120 [2][0] = "HDP",
121 [4][0] = "DCEDMC",
122 [5][0] = "DCEVGA",
123 [13][0] = "UTCL2",
124 [19][0] = "TLS",
125 [26][0] = "OSS",
126 [27][0] = "SDMA0",
127 [28][0] = "VCN",
128 [29][0] = "VCNU",
129 [30][0] = "JPEG",
130 [0][1] = "MP1",
131 [1][1] = "MP0",
132 [2][1] = "HDP",
133 [3][1] = "XDP",
134 [6][1] = "DBGU0",
135 [7][1] = "DCEDMC",
136 [8][1] = "DCEVGA",
137 [9][1] = "DCEDWB",
138 [26][1] = "OSS",
139 [27][1] = "SDMA0",
140 [28][1] = "VCN",
141 [29][1] = "VCNU",
142 [30][1] = "JPEG",
143};
144
145static const char *mmhub_client_ids_vega10[][2] = {
146 [0][0] = "MP0",
147 [1][0] = "UVD",
148 [2][0] = "UVDU",
149 [3][0] = "HDP",
150 [13][0] = "UTCL2",
151 [14][0] = "OSS",
152 [15][0] = "SDMA1",
153 [32+0][0] = "VCE0",
154 [32+1][0] = "VCE0U",
155 [32+2][0] = "XDMA",
156 [32+3][0] = "DCE",
157 [32+4][0] = "MP1",
158 [32+14][0] = "SDMA0",
159 [0][1] = "MP0",
160 [1][1] = "UVD",
161 [2][1] = "UVDU",
162 [3][1] = "DBGU0",
163 [4][1] = "HDP",
164 [5][1] = "XDP",
165 [14][1] = "OSS",
166 [15][1] = "SDMA0",
167 [32+0][1] = "VCE0",
168 [32+1][1] = "VCE0U",
169 [32+2][1] = "XDMA",
170 [32+3][1] = "DCE",
171 [32+4][1] = "DCEDWB",
172 [32+5][1] = "MP1",
173 [32+6][1] = "DBGU1",
174 [32+14][1] = "SDMA1",
175};
176
177static const char *mmhub_client_ids_vega12[][2] = {
178 [0][0] = "MP0",
179 [1][0] = "VCE0",
180 [2][0] = "VCE0U",
181 [3][0] = "HDP",
182 [13][0] = "UTCL2",
183 [14][0] = "OSS",
184 [15][0] = "SDMA1",
185 [32+0][0] = "DCE",
186 [32+1][0] = "XDMA",
187 [32+2][0] = "UVD",
188 [32+3][0] = "UVDU",
189 [32+4][0] = "MP1",
190 [32+15][0] = "SDMA0",
191 [0][1] = "MP0",
192 [1][1] = "VCE0",
193 [2][1] = "VCE0U",
194 [3][1] = "DBGU0",
195 [4][1] = "HDP",
196 [5][1] = "XDP",
197 [14][1] = "OSS",
198 [15][1] = "SDMA0",
199 [32+0][1] = "DCE",
200 [32+1][1] = "DCEDWB",
201 [32+2][1] = "XDMA",
202 [32+3][1] = "UVD",
203 [32+4][1] = "UVDU",
204 [32+5][1] = "MP1",
205 [32+6][1] = "DBGU1",
206 [32+15][1] = "SDMA1",
207};
208
209static const char *mmhub_client_ids_vega20[][2] = {
210 [0][0] = "XDMA",
211 [1][0] = "DCE",
212 [2][0] = "VCE0",
213 [3][0] = "VCE0U",
214 [4][0] = "UVD",
215 [5][0] = "UVD1U",
216 [13][0] = "OSS",
217 [14][0] = "HDP",
218 [15][0] = "SDMA0",
219 [32+0][0] = "UVD",
220 [32+1][0] = "UVDU",
221 [32+2][0] = "MP1",
222 [32+3][0] = "MP0",
223 [32+12][0] = "UTCL2",
224 [32+14][0] = "SDMA1",
225 [0][1] = "XDMA",
226 [1][1] = "DCE",
227 [2][1] = "DCEDWB",
228 [3][1] = "VCE0",
229 [4][1] = "VCE0U",
230 [5][1] = "UVD1",
231 [6][1] = "UVD1U",
232 [7][1] = "DBGU0",
233 [8][1] = "XDP",
234 [13][1] = "OSS",
235 [14][1] = "HDP",
236 [15][1] = "SDMA0",
237 [32+0][1] = "UVD",
238 [32+1][1] = "UVDU",
239 [32+2][1] = "DBGU1",
240 [32+3][1] = "MP1",
241 [32+4][1] = "MP0",
242 [32+14][1] = "SDMA1",
243};
244
245static const char *mmhub_client_ids_arcturus[][2] = {
246 [0][0] = "DBGU1",
247 [1][0] = "XDP",
248 [2][0] = "MP1",
249 [14][0] = "HDP",
250 [171][0] = "JPEG",
251 [172][0] = "VCN",
252 [173][0] = "VCNU",
253 [203][0] = "JPEG1",
254 [204][0] = "VCN1",
255 [205][0] = "VCN1U",
256 [256][0] = "SDMA0",
257 [257][0] = "SDMA1",
258 [258][0] = "SDMA2",
259 [259][0] = "SDMA3",
260 [260][0] = "SDMA4",
261 [261][0] = "SDMA5",
262 [262][0] = "SDMA6",
263 [263][0] = "SDMA7",
264 [384][0] = "OSS",
265 [0][1] = "DBGU1",
266 [1][1] = "XDP",
267 [2][1] = "MP1",
268 [14][1] = "HDP",
269 [171][1] = "JPEG",
270 [172][1] = "VCN",
271 [173][1] = "VCNU",
272 [203][1] = "JPEG1",
273 [204][1] = "VCN1",
274 [205][1] = "VCN1U",
275 [256][1] = "SDMA0",
276 [257][1] = "SDMA1",
277 [258][1] = "SDMA2",
278 [259][1] = "SDMA3",
279 [260][1] = "SDMA4",
280 [261][1] = "SDMA5",
281 [262][1] = "SDMA6",
282 [263][1] = "SDMA7",
283 [384][1] = "OSS",
284};
285
286static const char *mmhub_client_ids_aldebaran[][2] = {
287 [2][0] = "MP1",
288 [3][0] = "MP0",
289 [32+1][0] = "DBGU_IO0",
290 [32+2][0] = "DBGU_IO2",
291 [32+4][0] = "MPIO",
292 [96+11][0] = "JPEG0",
293 [96+12][0] = "VCN0",
294 [96+13][0] = "VCNU0",
295 [128+11][0] = "JPEG1",
296 [128+12][0] = "VCN1",
297 [128+13][0] = "VCNU1",
298 [160+1][0] = "XDP",
299 [160+14][0] = "HDP",
300 [256+0][0] = "SDMA0",
301 [256+1][0] = "SDMA1",
302 [256+2][0] = "SDMA2",
303 [256+3][0] = "SDMA3",
304 [256+4][0] = "SDMA4",
305 [384+0][0] = "OSS",
306 [2][1] = "MP1",
307 [3][1] = "MP0",
308 [32+1][1] = "DBGU_IO0",
309 [32+2][1] = "DBGU_IO2",
310 [32+4][1] = "MPIO",
311 [96+11][1] = "JPEG0",
312 [96+12][1] = "VCN0",
313 [96+13][1] = "VCNU0",
314 [128+11][1] = "JPEG1",
315 [128+12][1] = "VCN1",
316 [128+13][1] = "VCNU1",
317 [160+1][1] = "XDP",
318 [160+14][1] = "HDP",
319 [256+0][1] = "SDMA0",
320 [256+1][1] = "SDMA1",
321 [256+2][1] = "SDMA2",
322 [256+3][1] = "SDMA3",
323 [256+4][1] = "SDMA4",
324 [384+0][1] = "OSS",
325};
326
327static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
328{
329 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
330 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
331};
332
333static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
334{
335 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
336 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
337};
338
339static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
340 (0x000143c0 + 0x00000000),
341 (0x000143c0 + 0x00000800),
342 (0x000143c0 + 0x00001000),
343 (0x000143c0 + 0x00001800),
344 (0x000543c0 + 0x00000000),
345 (0x000543c0 + 0x00000800),
346 (0x000543c0 + 0x00001000),
347 (0x000543c0 + 0x00001800),
348 (0x000943c0 + 0x00000000),
349 (0x000943c0 + 0x00000800),
350 (0x000943c0 + 0x00001000),
351 (0x000943c0 + 0x00001800),
352 (0x000d43c0 + 0x00000000),
353 (0x000d43c0 + 0x00000800),
354 (0x000d43c0 + 0x00001000),
355 (0x000d43c0 + 0x00001800),
356 (0x001143c0 + 0x00000000),
357 (0x001143c0 + 0x00000800),
358 (0x001143c0 + 0x00001000),
359 (0x001143c0 + 0x00001800),
360 (0x001543c0 + 0x00000000),
361 (0x001543c0 + 0x00000800),
362 (0x001543c0 + 0x00001000),
363 (0x001543c0 + 0x00001800),
364 (0x001943c0 + 0x00000000),
365 (0x001943c0 + 0x00000800),
366 (0x001943c0 + 0x00001000),
367 (0x001943c0 + 0x00001800),
368 (0x001d43c0 + 0x00000000),
369 (0x001d43c0 + 0x00000800),
370 (0x001d43c0 + 0x00001000),
371 (0x001d43c0 + 0x00001800),
372};
373
374static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
375 (0x000143e0 + 0x00000000),
376 (0x000143e0 + 0x00000800),
377 (0x000143e0 + 0x00001000),
378 (0x000143e0 + 0x00001800),
379 (0x000543e0 + 0x00000000),
380 (0x000543e0 + 0x00000800),
381 (0x000543e0 + 0x00001000),
382 (0x000543e0 + 0x00001800),
383 (0x000943e0 + 0x00000000),
384 (0x000943e0 + 0x00000800),
385 (0x000943e0 + 0x00001000),
386 (0x000943e0 + 0x00001800),
387 (0x000d43e0 + 0x00000000),
388 (0x000d43e0 + 0x00000800),
389 (0x000d43e0 + 0x00001000),
390 (0x000d43e0 + 0x00001800),
391 (0x001143e0 + 0x00000000),
392 (0x001143e0 + 0x00000800),
393 (0x001143e0 + 0x00001000),
394 (0x001143e0 + 0x00001800),
395 (0x001543e0 + 0x00000000),
396 (0x001543e0 + 0x00000800),
397 (0x001543e0 + 0x00001000),
398 (0x001543e0 + 0x00001800),
399 (0x001943e0 + 0x00000000),
400 (0x001943e0 + 0x00000800),
401 (0x001943e0 + 0x00001000),
402 (0x001943e0 + 0x00001800),
403 (0x001d43e0 + 0x00000000),
404 (0x001d43e0 + 0x00000800),
405 (0x001d43e0 + 0x00001000),
406 (0x001d43e0 + 0x00001800),
407};
408
409static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
410 struct amdgpu_irq_src *src,
411 unsigned type,
412 enum amdgpu_interrupt_state state)
413{
414 u32 bits, i, tmp, reg;
415
416
417
418 if (adev->asic_type >= CHIP_VEGA20)
419 return 0;
420
421 bits = 0x7f;
422
423 switch (state) {
424 case AMDGPU_IRQ_STATE_DISABLE:
425 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
426 reg = ecc_umc_mcumc_ctrl_addrs[i];
427 tmp = RREG32(reg);
428 tmp &= ~bits;
429 WREG32(reg, tmp);
430 }
431 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
432 reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
433 tmp = RREG32(reg);
434 tmp &= ~bits;
435 WREG32(reg, tmp);
436 }
437 break;
438 case AMDGPU_IRQ_STATE_ENABLE:
439 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
440 reg = ecc_umc_mcumc_ctrl_addrs[i];
441 tmp = RREG32(reg);
442 tmp |= bits;
443 WREG32(reg, tmp);
444 }
445 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
446 reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
447 tmp = RREG32(reg);
448 tmp |= bits;
449 WREG32(reg, tmp);
450 }
451 break;
452 default:
453 break;
454 }
455
456 return 0;
457}
458
459static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
460 struct amdgpu_irq_src *src,
461 unsigned type,
462 enum amdgpu_interrupt_state state)
463{
464 struct amdgpu_vmhub *hub;
465 u32 tmp, reg, bits, i, j;
466
467 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
468 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
469 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
470 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
471 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
472 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
473 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
474
475 switch (state) {
476 case AMDGPU_IRQ_STATE_DISABLE:
477 for (j = 0; j < adev->num_vmhubs; j++) {
478 hub = &adev->vmhub[j];
479 for (i = 0; i < 16; i++) {
480 reg = hub->vm_context0_cntl + i;
481 tmp = RREG32(reg);
482 tmp &= ~bits;
483 WREG32(reg, tmp);
484 }
485 }
486 break;
487 case AMDGPU_IRQ_STATE_ENABLE:
488 for (j = 0; j < adev->num_vmhubs; j++) {
489 hub = &adev->vmhub[j];
490 for (i = 0; i < 16; i++) {
491 reg = hub->vm_context0_cntl + i;
492 tmp = RREG32(reg);
493 tmp |= bits;
494 WREG32(reg, tmp);
495 }
496 }
497 break;
498 default:
499 break;
500 }
501
502 return 0;
503}
504
505static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
506 struct amdgpu_irq_src *source,
507 struct amdgpu_iv_entry *entry)
508{
509 bool retry_fault = !!(entry->src_data[1] & 0x80);
510 bool write_fault = !!(entry->src_data[1] & 0x20);
511 uint32_t status = 0, cid = 0, rw = 0;
512 struct amdgpu_task_info task_info;
513 struct amdgpu_vmhub *hub;
514 const char *mmhub_cid;
515 const char *hub_name;
516 u64 addr;
517
518 addr = (u64)entry->src_data[0] << 12;
519 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
520
521 if (retry_fault) {
522
523
524
525 if (entry->ih != &adev->irq.ih_soft &&
526 amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
527 entry->timestamp))
528 return 1;
529
530
531
532
533 if (entry->ih == &adev->irq.ih) {
534 amdgpu_irq_delegate(adev, entry, 8);
535 return 1;
536 }
537
538
539
540
541 if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault))
542 return 1;
543 }
544
545 if (!printk_ratelimit())
546 return 0;
547
548 if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
549 hub_name = "mmhub0";
550 hub = &adev->vmhub[AMDGPU_MMHUB_0];
551 } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
552 hub_name = "mmhub1";
553 hub = &adev->vmhub[AMDGPU_MMHUB_1];
554 } else {
555 hub_name = "gfxhub0";
556 hub = &adev->vmhub[AMDGPU_GFXHUB_0];
557 }
558
559 memset(&task_info, 0, sizeof(struct amdgpu_task_info));
560 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
561
562 dev_err(adev->dev,
563 "[%s] %s page fault (src_id:%u ring:%u vmid:%u "
564 "pasid:%u, for process %s pid %d thread %s pid %d)\n",
565 hub_name, retry_fault ? "retry" : "no-retry",
566 entry->src_id, entry->ring_id, entry->vmid,
567 entry->pasid, task_info.process_name, task_info.tgid,
568 task_info.task_name, task_info.pid);
569 dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
570 addr, entry->client_id,
571 soc15_ih_clientid_name[entry->client_id]);
572
573 if (amdgpu_sriov_vf(adev))
574 return 0;
575
576
577
578
579
580
581 if ((entry->vmid_src == AMDGPU_GFXHUB_0) &&
582 (adev->asic_type < CHIP_ALDEBARAN))
583 RREG32(hub->vm_l2_pro_fault_status);
584
585 status = RREG32(hub->vm_l2_pro_fault_status);
586 cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
587 rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
588 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
589
590
591 dev_err(adev->dev,
592 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
593 status);
594 if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
595 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
596 cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
597 gfxhub_client_ids[cid],
598 cid);
599 } else {
600 switch (adev->asic_type) {
601 case CHIP_VEGA10:
602 mmhub_cid = mmhub_client_ids_vega10[cid][rw];
603 break;
604 case CHIP_VEGA12:
605 mmhub_cid = mmhub_client_ids_vega12[cid][rw];
606 break;
607 case CHIP_VEGA20:
608 mmhub_cid = mmhub_client_ids_vega20[cid][rw];
609 break;
610 case CHIP_ARCTURUS:
611 mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
612 break;
613 case CHIP_RAVEN:
614 mmhub_cid = mmhub_client_ids_raven[cid][rw];
615 break;
616 case CHIP_RENOIR:
617 mmhub_cid = mmhub_client_ids_renoir[cid][rw];
618 break;
619 case CHIP_ALDEBARAN:
620 mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
621 break;
622 default:
623 mmhub_cid = NULL;
624 break;
625 }
626 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
627 mmhub_cid ? mmhub_cid : "unknown", cid);
628 }
629 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
630 REG_GET_FIELD(status,
631 VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
632 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
633 REG_GET_FIELD(status,
634 VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
635 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
636 REG_GET_FIELD(status,
637 VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
638 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
639 REG_GET_FIELD(status,
640 VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
641 dev_err(adev->dev, "\t RW: 0x%x\n", rw);
642 return 0;
643}
644
645static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
646 .set = gmc_v9_0_vm_fault_interrupt_state,
647 .process = gmc_v9_0_process_interrupt,
648};
649
650
651static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
652 .set = gmc_v9_0_ecc_interrupt_state,
653 .process = amdgpu_umc_process_ecc_irq,
654};
655
656static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
657{
658 adev->gmc.vm_fault.num_types = 1;
659 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
660
661 if (!amdgpu_sriov_vf(adev) &&
662 !adev->gmc.xgmi.connected_to_cpu) {
663 adev->gmc.ecc_irq.num_types = 1;
664 adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
665 }
666}
667
668static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
669 uint32_t flush_type)
670{
671 u32 req = 0;
672
673 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
674 PER_VMID_INVALIDATE_REQ, 1 << vmid);
675 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
676 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
677 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
678 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
679 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
680 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
681 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
682 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
683
684 return req;
685}
686
687
688
689
690
691
692
693
694static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
695 uint32_t vmhub)
696{
697 if (adev->asic_type == CHIP_ALDEBARAN)
698 return false;
699
700 return ((vmhub == AMDGPU_MMHUB_0 ||
701 vmhub == AMDGPU_MMHUB_1) &&
702 (!amdgpu_sriov_vf(adev)) &&
703 (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
704 (adev->apu_flags & AMD_APU_IS_PICASSO))));
705}
706
707static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
708 uint8_t vmid, uint16_t *p_pasid)
709{
710 uint32_t value;
711
712 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
713 + vmid);
714 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
715
716 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
717}
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
737 uint32_t vmhub, uint32_t flush_type)
738{
739 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
740 const unsigned eng = 17;
741 u32 j, inv_req, inv_req2, tmp;
742 struct amdgpu_vmhub *hub;
743
744 BUG_ON(vmhub >= adev->num_vmhubs);
745
746 hub = &adev->vmhub[vmhub];
747 if (adev->gmc.xgmi.num_physical_nodes &&
748 adev->asic_type == CHIP_VEGA20) {
749
750
751
752
753
754
755 inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
756 inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
757 } else {
758 inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
759 inv_req2 = 0;
760 }
761
762
763
764
765 if (adev->gfx.kiq.ring.sched.ready &&
766 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
767 down_read_trylock(&adev->reset_sem)) {
768 uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
769 uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
770
771 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
772 1 << vmid);
773 up_read(&adev->reset_sem);
774 return;
775 }
776
777 spin_lock(&adev->gmc.invalidate_lock);
778
779
780
781
782
783
784
785
786
787 if (use_semaphore) {
788 for (j = 0; j < adev->usec_timeout; j++) {
789
790 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem +
791 hub->eng_distance * eng);
792 if (tmp & 0x1)
793 break;
794 udelay(1);
795 }
796
797 if (j >= adev->usec_timeout)
798 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
799 }
800
801 do {
802 WREG32_NO_KIQ(hub->vm_inv_eng0_req +
803 hub->eng_distance * eng, inv_req);
804
805
806
807
808
809
810 if ((vmhub == AMDGPU_GFXHUB_0) &&
811 (adev->asic_type < CHIP_ALDEBARAN))
812 RREG32_NO_KIQ(hub->vm_inv_eng0_req +
813 hub->eng_distance * eng);
814
815 for (j = 0; j < adev->usec_timeout; j++) {
816 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack +
817 hub->eng_distance * eng);
818 if (tmp & (1 << vmid))
819 break;
820 udelay(1);
821 }
822
823 inv_req = inv_req2;
824 inv_req2 = 0;
825 } while (inv_req);
826
827
828 if (use_semaphore)
829
830
831
832
833 WREG32_NO_KIQ(hub->vm_inv_eng0_sem +
834 hub->eng_distance * eng, 0);
835
836 spin_unlock(&adev->gmc.invalidate_lock);
837
838 if (j < adev->usec_timeout)
839 return;
840
841 DRM_ERROR("Timeout waiting for VM flush ACK!\n");
842}
843
844
845
846
847
848
849
850
851
852
853
854static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
855 uint16_t pasid, uint32_t flush_type,
856 bool all_hub)
857{
858 int vmid, i;
859 signed long r;
860 uint32_t seq;
861 uint16_t queried_pasid;
862 bool ret;
863 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
864 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
865
866 if (amdgpu_in_reset(adev))
867 return -EIO;
868
869 if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) {
870
871
872
873
874
875
876 bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
877 adev->asic_type == CHIP_VEGA20);
878
879 unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
880
881 if (vega20_xgmi_wa)
882 ndw += kiq->pmf->invalidate_tlbs_size;
883
884 spin_lock(&adev->gfx.kiq.ring_lock);
885
886 amdgpu_ring_alloc(ring, ndw);
887 if (vega20_xgmi_wa)
888 kiq->pmf->kiq_invalidate_tlbs(ring,
889 pasid, 2, all_hub);
890 kiq->pmf->kiq_invalidate_tlbs(ring,
891 pasid, flush_type, all_hub);
892 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
893 if (r) {
894 amdgpu_ring_undo(ring);
895 spin_unlock(&adev->gfx.kiq.ring_lock);
896 up_read(&adev->reset_sem);
897 return -ETIME;
898 }
899
900 amdgpu_ring_commit(ring);
901 spin_unlock(&adev->gfx.kiq.ring_lock);
902 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
903 if (r < 1) {
904 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
905 up_read(&adev->reset_sem);
906 return -ETIME;
907 }
908 up_read(&adev->reset_sem);
909 return 0;
910 }
911
912 for (vmid = 1; vmid < 16; vmid++) {
913
914 ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
915 &queried_pasid);
916 if (ret && queried_pasid == pasid) {
917 if (all_hub) {
918 for (i = 0; i < adev->num_vmhubs; i++)
919 gmc_v9_0_flush_gpu_tlb(adev, vmid,
920 i, flush_type);
921 } else {
922 gmc_v9_0_flush_gpu_tlb(adev, vmid,
923 AMDGPU_GFXHUB_0, flush_type);
924 }
925 break;
926 }
927 }
928
929 return 0;
930
931}
932
933static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
934 unsigned vmid, uint64_t pd_addr)
935{
936 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
937 struct amdgpu_device *adev = ring->adev;
938 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
939 uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
940 unsigned eng = ring->vm_inv_eng;
941
942
943
944
945
946
947
948
949
950 if (use_semaphore)
951
952 amdgpu_ring_emit_reg_wait(ring,
953 hub->vm_inv_eng0_sem +
954 hub->eng_distance * eng, 0x1, 0x1);
955
956 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
957 (hub->ctx_addr_distance * vmid),
958 lower_32_bits(pd_addr));
959
960 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
961 (hub->ctx_addr_distance * vmid),
962 upper_32_bits(pd_addr));
963
964 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
965 hub->eng_distance * eng,
966 hub->vm_inv_eng0_ack +
967 hub->eng_distance * eng,
968 req, 1 << vmid);
969
970
971 if (use_semaphore)
972
973
974
975
976 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
977 hub->eng_distance * eng, 0);
978
979 return pd_addr;
980}
981
982static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
983 unsigned pasid)
984{
985 struct amdgpu_device *adev = ring->adev;
986 uint32_t reg;
987
988
989 if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
990 return;
991
992 if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
993 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
994 else
995 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
996
997 amdgpu_ring_emit_wreg(ring, reg, pasid);
998}
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1033
1034{
1035 switch (flags) {
1036 case AMDGPU_VM_MTYPE_DEFAULT:
1037 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1038 case AMDGPU_VM_MTYPE_NC:
1039 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1040 case AMDGPU_VM_MTYPE_WC:
1041 return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1042 case AMDGPU_VM_MTYPE_RW:
1043 return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1044 case AMDGPU_VM_MTYPE_CC:
1045 return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1046 case AMDGPU_VM_MTYPE_UC:
1047 return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1048 default:
1049 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1050 }
1051}
1052
1053static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1054 uint64_t *addr, uint64_t *flags)
1055{
1056 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1057 *addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
1058 BUG_ON(*addr & 0xFFFF00000000003FULL);
1059
1060 if (!adev->gmc.translate_further)
1061 return;
1062
1063 if (level == AMDGPU_VM_PDB1) {
1064
1065 if (!(*flags & AMDGPU_PDE_PTE))
1066 *flags |= AMDGPU_PDE_BFS(0x9);
1067
1068 } else if (level == AMDGPU_VM_PDB0) {
1069 if (*flags & AMDGPU_PDE_PTE)
1070 *flags &= ~AMDGPU_PDE_PTE;
1071 else
1072 *flags |= AMDGPU_PTE_TF;
1073 }
1074}
1075
1076static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1077 struct amdgpu_bo_va_mapping *mapping,
1078 uint64_t *flags)
1079{
1080 *flags &= ~AMDGPU_PTE_EXECUTABLE;
1081 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1082
1083 *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1084 *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1085
1086 if (mapping->flags & AMDGPU_PTE_PRT) {
1087 *flags |= AMDGPU_PTE_PRT;
1088 *flags &= ~AMDGPU_PTE_VALID;
1089 }
1090
1091 if ((adev->asic_type == CHIP_ARCTURUS ||
1092 adev->asic_type == CHIP_ALDEBARAN) &&
1093 !(*flags & AMDGPU_PTE_SYSTEM) &&
1094 mapping->bo_va->is_xgmi)
1095 *flags |= AMDGPU_PTE_SNOOPED;
1096
1097 if (adev->asic_type == CHIP_ALDEBARAN)
1098 *flags |= mapping->flags & AMDGPU_PTE_SNOOPED;
1099}
1100
1101static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1102{
1103 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1104 unsigned size;
1105
1106 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1107 size = AMDGPU_VBIOS_VGA_ALLOCATION;
1108 } else {
1109 u32 viewport;
1110
1111 switch (adev->asic_type) {
1112 case CHIP_RAVEN:
1113 case CHIP_RENOIR:
1114 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1115 size = (REG_GET_FIELD(viewport,
1116 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1117 REG_GET_FIELD(viewport,
1118 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1119 4);
1120 break;
1121 case CHIP_VEGA10:
1122 case CHIP_VEGA12:
1123 case CHIP_VEGA20:
1124 default:
1125 viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1126 size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1127 REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1128 4);
1129 break;
1130 }
1131 }
1132
1133 return size;
1134}
1135
1136static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1137 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1138 .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1139 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1140 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1141 .map_mtype = gmc_v9_0_map_mtype,
1142 .get_vm_pde = gmc_v9_0_get_vm_pde,
1143 .get_vm_pte = gmc_v9_0_get_vm_pte,
1144 .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1145};
1146
1147static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1148{
1149 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1150}
1151
1152static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1153{
1154 switch (adev->asic_type) {
1155 case CHIP_VEGA10:
1156 adev->umc.funcs = &umc_v6_0_funcs;
1157 break;
1158 case CHIP_VEGA20:
1159 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1160 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1161 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1162 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1163 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1164 adev->umc.ras_funcs = &umc_v6_1_ras_funcs;
1165 break;
1166 case CHIP_ARCTURUS:
1167 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1168 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1169 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1170 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1171 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1172 adev->umc.ras_funcs = &umc_v6_1_ras_funcs;
1173 break;
1174 case CHIP_ALDEBARAN:
1175 adev->umc.max_ras_err_cnt_per_query = UMC_V6_7_TOTAL_CHANNEL_NUM;
1176 adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1177 adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1178 adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1179 if (!adev->gmc.xgmi.connected_to_cpu)
1180 adev->umc.ras_funcs = &umc_v6_7_ras_funcs;
1181 if (1 & adev->smuio.funcs->get_die_id(adev))
1182 adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1183 else
1184 adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1185 break;
1186 default:
1187 break;
1188 }
1189}
1190
1191static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1192{
1193 switch (adev->asic_type) {
1194 case CHIP_ARCTURUS:
1195 adev->mmhub.funcs = &mmhub_v9_4_funcs;
1196 break;
1197 case CHIP_ALDEBARAN:
1198 adev->mmhub.funcs = &mmhub_v1_7_funcs;
1199 break;
1200 default:
1201 adev->mmhub.funcs = &mmhub_v1_0_funcs;
1202 break;
1203 }
1204}
1205
1206static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1207{
1208 switch (adev->asic_type) {
1209 case CHIP_VEGA20:
1210 adev->mmhub.ras_funcs = &mmhub_v1_0_ras_funcs;
1211 break;
1212 case CHIP_ARCTURUS:
1213 adev->mmhub.ras_funcs = &mmhub_v9_4_ras_funcs;
1214 break;
1215 case CHIP_ALDEBARAN:
1216 adev->mmhub.ras_funcs = &mmhub_v1_7_ras_funcs;
1217 break;
1218 default:
1219
1220 break;
1221 }
1222}
1223
1224static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1225{
1226 adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1227}
1228
1229static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1230{
1231 adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs;
1232}
1233
1234static void gmc_v9_0_set_mca_funcs(struct amdgpu_device *adev)
1235{
1236 switch (adev->asic_type) {
1237 case CHIP_ALDEBARAN:
1238 if (!adev->gmc.xgmi.connected_to_cpu)
1239 adev->mca.funcs = &mca_v3_0_funcs;
1240 break;
1241 default:
1242 break;
1243 }
1244}
1245
1246static int gmc_v9_0_early_init(void *handle)
1247{
1248 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1249
1250 if (adev->asic_type == CHIP_VEGA20 ||
1251 adev->asic_type == CHIP_ARCTURUS)
1252 adev->gmc.xgmi.supported = true;
1253
1254 if (adev->asic_type == CHIP_ALDEBARAN) {
1255 adev->gmc.xgmi.supported = true;
1256 adev->gmc.xgmi.connected_to_cpu =
1257 adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1258 }
1259
1260 gmc_v9_0_set_gmc_funcs(adev);
1261 gmc_v9_0_set_irq_funcs(adev);
1262 gmc_v9_0_set_umc_funcs(adev);
1263 gmc_v9_0_set_mmhub_funcs(adev);
1264 gmc_v9_0_set_mmhub_ras_funcs(adev);
1265 gmc_v9_0_set_gfxhub_funcs(adev);
1266 gmc_v9_0_set_hdp_ras_funcs(adev);
1267 gmc_v9_0_set_mca_funcs(adev);
1268
1269 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1270 adev->gmc.shared_aperture_end =
1271 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1272 adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1273 adev->gmc.private_aperture_end =
1274 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1275
1276 return 0;
1277}
1278
1279static int gmc_v9_0_late_init(void *handle)
1280{
1281 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1282 int r;
1283
1284 r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1285 if (r)
1286 return r;
1287
1288
1289
1290
1291
1292 if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
1293 if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1294 if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1295 adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1296 }
1297 }
1298
1299 if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1300 if (adev->mmhub.ras_funcs &&
1301 adev->mmhub.ras_funcs->reset_ras_error_count)
1302 adev->mmhub.ras_funcs->reset_ras_error_count(adev);
1303
1304 if (adev->hdp.ras_funcs &&
1305 adev->hdp.ras_funcs->reset_ras_error_count)
1306 adev->hdp.ras_funcs->reset_ras_error_count(adev);
1307 }
1308
1309 r = amdgpu_gmc_ras_late_init(adev);
1310 if (r)
1311 return r;
1312
1313 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1314}
1315
1316static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1317 struct amdgpu_gmc *mc)
1318{
1319 u64 base = adev->mmhub.funcs->get_fb_location(adev);
1320
1321
1322 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1323 if (adev->gmc.xgmi.connected_to_cpu) {
1324 amdgpu_gmc_sysvm_location(adev, mc);
1325 } else {
1326 amdgpu_gmc_vram_location(adev, mc, base);
1327 amdgpu_gmc_gart_location(adev, mc);
1328 amdgpu_gmc_agp_location(adev, mc);
1329 }
1330
1331 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1332
1333
1334 adev->vm_manager.vram_base_offset +=
1335 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1336}
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1348{
1349 int r;
1350
1351
1352 adev->gmc.mc_vram_size =
1353 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1354 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1355
1356 if (!(adev->flags & AMD_IS_APU) &&
1357 !adev->gmc.xgmi.connected_to_cpu) {
1358 r = amdgpu_device_resize_fb_bar(adev);
1359 if (r)
1360 return r;
1361 }
1362 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1363 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1364
1365#ifdef CONFIG_X86_64
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378 if ((adev->flags & AMD_IS_APU) ||
1379 (adev->gmc.xgmi.supported &&
1380 adev->gmc.xgmi.connected_to_cpu)) {
1381 adev->gmc.aper_base =
1382 adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1383 adev->gmc.xgmi.physical_node_id *
1384 adev->gmc.xgmi.node_segment_size;
1385 adev->gmc.aper_size = adev->gmc.real_vram_size;
1386 }
1387
1388#endif
1389
1390 adev->gmc.visible_vram_size = adev->gmc.aper_size;
1391 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1392 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
1393
1394
1395 if (amdgpu_gart_size == -1) {
1396 switch (adev->asic_type) {
1397 case CHIP_VEGA10:
1398 case CHIP_VEGA12:
1399 case CHIP_VEGA20:
1400 case CHIP_ARCTURUS:
1401 case CHIP_ALDEBARAN:
1402 default:
1403 adev->gmc.gart_size = 512ULL << 20;
1404 break;
1405 case CHIP_RAVEN:
1406 case CHIP_RENOIR:
1407 adev->gmc.gart_size = 1024ULL << 20;
1408 break;
1409 }
1410 } else {
1411 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1412 }
1413
1414 adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
1415
1416 gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1417
1418 return 0;
1419}
1420
1421static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1422{
1423 int r;
1424
1425 if (adev->gart.bo) {
1426 WARN(1, "VEGA10 PCIE GART already initialized\n");
1427 return 0;
1428 }
1429
1430 if (adev->gmc.xgmi.connected_to_cpu) {
1431 adev->gmc.vmid0_page_table_depth = 1;
1432 adev->gmc.vmid0_page_table_block_size = 12;
1433 } else {
1434 adev->gmc.vmid0_page_table_depth = 0;
1435 adev->gmc.vmid0_page_table_block_size = 0;
1436 }
1437
1438
1439 r = amdgpu_gart_init(adev);
1440 if (r)
1441 return r;
1442 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1443 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1444 AMDGPU_PTE_EXECUTABLE;
1445
1446 r = amdgpu_gart_table_vram_alloc(adev);
1447 if (r)
1448 return r;
1449
1450 if (adev->gmc.xgmi.connected_to_cpu) {
1451 r = amdgpu_gmc_pdb0_alloc(adev);
1452 }
1453
1454 return r;
1455}
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1466{
1467 if (adev->asic_type == CHIP_RAVEN)
1468 adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1469}
1470
1471static int gmc_v9_0_sw_init(void *handle)
1472{
1473 int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
1474 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1475
1476 adev->gfxhub.funcs->init(adev);
1477
1478 adev->mmhub.funcs->init(adev);
1479 if (adev->mca.funcs)
1480 adev->mca.funcs->init(adev);
1481
1482 spin_lock_init(&adev->gmc.invalidate_lock);
1483
1484 r = amdgpu_atomfirmware_get_vram_info(adev,
1485 &vram_width, &vram_type, &vram_vendor);
1486 if (amdgpu_sriov_vf(adev))
1487
1488
1489
1490
1491 adev->gmc.vram_width = 2048;
1492 else if (amdgpu_emu_mode != 1)
1493 adev->gmc.vram_width = vram_width;
1494
1495 if (!adev->gmc.vram_width) {
1496 int chansize, numchan;
1497
1498
1499 if (adev->flags & AMD_IS_APU)
1500 chansize = 64;
1501 else
1502 chansize = 128;
1503
1504 numchan = adev->df.funcs->get_hbm_channel_number(adev);
1505 adev->gmc.vram_width = numchan * chansize;
1506 }
1507
1508 adev->gmc.vram_type = vram_type;
1509 adev->gmc.vram_vendor = vram_vendor;
1510 switch (adev->asic_type) {
1511 case CHIP_RAVEN:
1512 adev->num_vmhubs = 2;
1513
1514 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1515 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1516 } else {
1517
1518 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1519 adev->gmc.translate_further =
1520 adev->vm_manager.num_level > 1;
1521 }
1522 break;
1523 case CHIP_VEGA10:
1524 case CHIP_VEGA12:
1525 case CHIP_VEGA20:
1526 case CHIP_RENOIR:
1527 case CHIP_ALDEBARAN:
1528 adev->num_vmhubs = 2;
1529
1530
1531
1532
1533
1534
1535
1536
1537 if (amdgpu_sriov_vf(adev))
1538 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1539 else
1540 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1541 break;
1542 case CHIP_ARCTURUS:
1543 adev->num_vmhubs = 3;
1544
1545
1546 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1547 break;
1548 default:
1549 break;
1550 }
1551
1552
1553 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1554 &adev->gmc.vm_fault);
1555 if (r)
1556 return r;
1557
1558 if (adev->asic_type == CHIP_ARCTURUS) {
1559 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1560 &adev->gmc.vm_fault);
1561 if (r)
1562 return r;
1563 }
1564
1565 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1566 &adev->gmc.vm_fault);
1567
1568 if (r)
1569 return r;
1570
1571 if (!amdgpu_sriov_vf(adev) &&
1572 !adev->gmc.xgmi.connected_to_cpu) {
1573
1574 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1575 &adev->gmc.ecc_irq);
1576 if (r)
1577 return r;
1578 }
1579
1580
1581
1582
1583
1584 adev->gmc.mc_mask = 0xffffffffffffULL;
1585
1586 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
1587 if (r) {
1588 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
1589 return r;
1590 }
1591 adev->need_swiotlb = drm_need_swiotlb(44);
1592
1593 if (adev->gmc.xgmi.supported) {
1594 r = adev->gfxhub.funcs->get_xgmi_info(adev);
1595 if (r)
1596 return r;
1597 }
1598
1599 r = gmc_v9_0_mc_init(adev);
1600 if (r)
1601 return r;
1602
1603 amdgpu_gmc_get_vbios_allocations(adev);
1604
1605
1606 r = amdgpu_bo_init(adev);
1607 if (r)
1608 return r;
1609
1610 r = gmc_v9_0_gart_init(adev);
1611 if (r)
1612 return r;
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624 adev->vm_manager.first_kfd_vmid =
1625 (adev->asic_type == CHIP_ARCTURUS ||
1626 adev->asic_type == CHIP_ALDEBARAN) ? 3 : 8;
1627
1628 amdgpu_vm_manager_init(adev);
1629
1630 gmc_v9_0_save_registers(adev);
1631
1632 return 0;
1633}
1634
1635static int gmc_v9_0_sw_fini(void *handle)
1636{
1637 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1638
1639 amdgpu_gmc_ras_fini(adev);
1640 amdgpu_gem_force_release(adev);
1641 amdgpu_vm_manager_fini(adev);
1642 amdgpu_gart_table_vram_free(adev);
1643 amdgpu_bo_unref(&adev->gmc.pdb0_bo);
1644 amdgpu_bo_fini(adev);
1645
1646 return 0;
1647}
1648
1649static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1650{
1651
1652 switch (adev->asic_type) {
1653 case CHIP_VEGA10:
1654 if (amdgpu_sriov_vf(adev))
1655 break;
1656 fallthrough;
1657 case CHIP_VEGA20:
1658 soc15_program_register_sequence(adev,
1659 golden_settings_mmhub_1_0_0,
1660 ARRAY_SIZE(golden_settings_mmhub_1_0_0));
1661 soc15_program_register_sequence(adev,
1662 golden_settings_athub_1_0_0,
1663 ARRAY_SIZE(golden_settings_athub_1_0_0));
1664 break;
1665 case CHIP_VEGA12:
1666 break;
1667 case CHIP_RAVEN:
1668
1669 soc15_program_register_sequence(adev,
1670 golden_settings_athub_1_0_0,
1671 ARRAY_SIZE(golden_settings_athub_1_0_0));
1672 break;
1673 default:
1674 break;
1675 }
1676}
1677
1678
1679
1680
1681
1682
1683
1684
1685void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
1686{
1687 if (adev->asic_type == CHIP_RAVEN) {
1688 WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
1689 WARN_ON(adev->gmc.sdpif_register !=
1690 RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
1691 }
1692}
1693
1694
1695
1696
1697
1698
1699static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1700{
1701 int r;
1702
1703 if (adev->gmc.xgmi.connected_to_cpu)
1704 amdgpu_gmc_init_pdb0(adev);
1705
1706 if (adev->gart.bo == NULL) {
1707 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1708 return -EINVAL;
1709 }
1710
1711 r = amdgpu_gart_table_vram_pin(adev);
1712 if (r)
1713 return r;
1714
1715 r = adev->gfxhub.funcs->gart_enable(adev);
1716 if (r)
1717 return r;
1718
1719 r = adev->mmhub.funcs->gart_enable(adev);
1720 if (r)
1721 return r;
1722
1723 DRM_INFO("PCIE GART of %uM enabled.\n",
1724 (unsigned)(adev->gmc.gart_size >> 20));
1725 if (adev->gmc.pdb0_bo)
1726 DRM_INFO("PDB0 located at 0x%016llX\n",
1727 (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
1728 DRM_INFO("PTB located at 0x%016llX\n",
1729 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1730
1731 adev->gart.ready = true;
1732 return 0;
1733}
1734
1735static int gmc_v9_0_hw_init(void *handle)
1736{
1737 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1738 bool value;
1739 int r, i;
1740
1741
1742 gmc_v9_0_init_golden_registers(adev);
1743
1744 if (adev->mode_info.num_crtc) {
1745
1746 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1747
1748 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1749 }
1750
1751 if (adev->mmhub.funcs->update_power_gating)
1752 adev->mmhub.funcs->update_power_gating(adev, true);
1753
1754 adev->hdp.funcs->init_registers(adev);
1755
1756
1757 adev->hdp.funcs->flush_hdp(adev, NULL);
1758
1759 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1760 value = false;
1761 else
1762 value = true;
1763
1764 if (!amdgpu_sriov_vf(adev)) {
1765 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
1766 adev->mmhub.funcs->set_fault_enable_default(adev, value);
1767 }
1768 for (i = 0; i < adev->num_vmhubs; ++i)
1769 gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
1770
1771 if (adev->umc.funcs && adev->umc.funcs->init_registers)
1772 adev->umc.funcs->init_registers(adev);
1773
1774 r = gmc_v9_0_gart_enable(adev);
1775
1776 return r;
1777}
1778
1779
1780
1781
1782
1783
1784
1785
1786static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1787{
1788 adev->gfxhub.funcs->gart_disable(adev);
1789 adev->mmhub.funcs->gart_disable(adev);
1790 amdgpu_gart_table_vram_unpin(adev);
1791}
1792
1793static int gmc_v9_0_hw_fini(void *handle)
1794{
1795 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1796
1797 gmc_v9_0_gart_disable(adev);
1798
1799 if (amdgpu_sriov_vf(adev)) {
1800
1801 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1802 return 0;
1803 }
1804
1805 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
1806 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1807
1808 return 0;
1809}
1810
1811static int gmc_v9_0_suspend(void *handle)
1812{
1813 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1814
1815 return gmc_v9_0_hw_fini(adev);
1816}
1817
1818static int gmc_v9_0_resume(void *handle)
1819{
1820 int r;
1821 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1822
1823 r = gmc_v9_0_hw_init(adev);
1824 if (r)
1825 return r;
1826
1827 amdgpu_vmid_reset_all(adev);
1828
1829 return 0;
1830}
1831
1832static bool gmc_v9_0_is_idle(void *handle)
1833{
1834
1835 return true;
1836}
1837
1838static int gmc_v9_0_wait_for_idle(void *handle)
1839{
1840
1841 return 0;
1842}
1843
1844static int gmc_v9_0_soft_reset(void *handle)
1845{
1846
1847 return 0;
1848}
1849
1850static int gmc_v9_0_set_clockgating_state(void *handle,
1851 enum amd_clockgating_state state)
1852{
1853 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1854
1855 adev->mmhub.funcs->set_clockgating(adev, state);
1856
1857 athub_v1_0_set_clockgating(adev, state);
1858
1859 return 0;
1860}
1861
1862static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1863{
1864 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1865
1866 adev->mmhub.funcs->get_clockgating(adev, flags);
1867
1868 athub_v1_0_get_clockgating(adev, flags);
1869}
1870
1871static int gmc_v9_0_set_powergating_state(void *handle,
1872 enum amd_powergating_state state)
1873{
1874 return 0;
1875}
1876
1877const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1878 .name = "gmc_v9_0",
1879 .early_init = gmc_v9_0_early_init,
1880 .late_init = gmc_v9_0_late_init,
1881 .sw_init = gmc_v9_0_sw_init,
1882 .sw_fini = gmc_v9_0_sw_fini,
1883 .hw_init = gmc_v9_0_hw_init,
1884 .hw_fini = gmc_v9_0_hw_fini,
1885 .suspend = gmc_v9_0_suspend,
1886 .resume = gmc_v9_0_resume,
1887 .is_idle = gmc_v9_0_is_idle,
1888 .wait_for_idle = gmc_v9_0_wait_for_idle,
1889 .soft_reset = gmc_v9_0_soft_reset,
1890 .set_clockgating_state = gmc_v9_0_set_clockgating_state,
1891 .set_powergating_state = gmc_v9_0_set_powergating_state,
1892 .get_clockgating_state = gmc_v9_0_get_clockgating_state,
1893};
1894
1895const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1896{
1897 .type = AMD_IP_BLOCK_TYPE_GMC,
1898 .major = 9,
1899 .minor = 0,
1900 .rev = 0,
1901 .funcs = &gmc_v9_0_ip_funcs,
1902};
1903
