1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/firmware.h>
24#include <drm/drm_cache.h>
25#include "amdgpu.h"
26#include "gmc_v9_0.h"
27#include "amdgpu_atomfirmware.h"
28
29#include "hdp/hdp_4_0_offset.h"
30#include "hdp/hdp_4_0_sh_mask.h"
31#include "gc/gc_9_0_sh_mask.h"
32#include "dce/dce_12_0_offset.h"
33#include "dce/dce_12_0_sh_mask.h"
34#include "vega10_enum.h"
35#include "mmhub/mmhub_1_0_offset.h"
36#include "athub/athub_1_0_offset.h"
37#include "oss/osssys_4_0_offset.h"
38
39#include "soc15.h"
40#include "soc15_common.h"
41#include "umc/umc_6_0_sh_mask.h"
42
43#include "gfxhub_v1_0.h"
44#include "mmhub_v1_0.h"
45
46#define mmDF_CS_AON0_DramBaseAddress0 0x0044
47#define mmDF_CS_AON0_DramBaseAddress0_BASE_IDX 0
48
49#define DF_CS_AON0_DramBaseAddress0__AddrRngVal__SHIFT 0x0
50#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn__SHIFT 0x1
51#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT 0x4
52#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel__SHIFT 0x8
53#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr__SHIFT 0xc
54#define DF_CS_AON0_DramBaseAddress0__AddrRngVal_MASK 0x00000001L
55#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK 0x00000002L
56#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK 0x000000F0L
57#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel_MASK 0x00000700L
58#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr_MASK 0xFFFFF000L
59
60
61#define AMDGPU_NUM_OF_VMIDS 8
62
63static const u32 golden_settings_vega10_hdp[] =
64{
65 0xf64, 0x0fffffff, 0x00000000,
66 0xf65, 0x0fffffff, 0x00000000,
67 0xf66, 0x0fffffff, 0x00000000,
68 0xf67, 0x0fffffff, 0x00000000,
69 0xf68, 0x0fffffff, 0x00000000,
70 0xf6a, 0x0fffffff, 0x00000000,
71 0xf6b, 0x0fffffff, 0x00000000,
72 0xf6c, 0x0fffffff, 0x00000000,
73 0xf6d, 0x0fffffff, 0x00000000,
74 0xf6e, 0x0fffffff, 0x00000000,
75};
76
77static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
78{
79 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
80 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
81};
82
83static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
84{
85 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
86 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
87};
88
89
90
91
92#define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000)
93#define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800)
94#define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000)
95#define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800)
96#define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000)
97#define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800)
98#define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000)
99#define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800)
100#define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000)
101#define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800)
102#define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000)
103#define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800)
104#define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000)
105#define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800)
106#define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000)
107#define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800)
108
109
110
111#define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000)
112#define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800)
113#define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000)
114#define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800)
115#define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000)
116#define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800)
117#define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000)
118#define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800)
119#define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000)
120#define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800)
121#define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000)
122#define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800)
123#define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000)
124#define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800)
125#define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000)
126#define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800)
127
128
129
130#define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000)
131#define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800)
132#define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000)
133#define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800)
134#define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000)
135#define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800)
136#define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000)
137#define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800)
138#define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000)
139#define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800)
140#define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000)
141#define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800)
142#define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000)
143#define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800)
144#define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000)
145#define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800)
146
147static const uint32_t ecc_umclocalcap_addrs[] = {
148 UMCLOCALCAPS_ADDR0,
149 UMCLOCALCAPS_ADDR1,
150 UMCLOCALCAPS_ADDR2,
151 UMCLOCALCAPS_ADDR3,
152 UMCLOCALCAPS_ADDR4,
153 UMCLOCALCAPS_ADDR5,
154 UMCLOCALCAPS_ADDR6,
155 UMCLOCALCAPS_ADDR7,
156 UMCLOCALCAPS_ADDR8,
157 UMCLOCALCAPS_ADDR9,
158 UMCLOCALCAPS_ADDR10,
159 UMCLOCALCAPS_ADDR11,
160 UMCLOCALCAPS_ADDR12,
161 UMCLOCALCAPS_ADDR13,
162 UMCLOCALCAPS_ADDR14,
163 UMCLOCALCAPS_ADDR15,
164};
165
166static const uint32_t ecc_umcch_umc_config_addrs[] = {
167 UMCCH_UMC_CONFIG_ADDR0,
168 UMCCH_UMC_CONFIG_ADDR1,
169 UMCCH_UMC_CONFIG_ADDR2,
170 UMCCH_UMC_CONFIG_ADDR3,
171 UMCCH_UMC_CONFIG_ADDR4,
172 UMCCH_UMC_CONFIG_ADDR5,
173 UMCCH_UMC_CONFIG_ADDR6,
174 UMCCH_UMC_CONFIG_ADDR7,
175 UMCCH_UMC_CONFIG_ADDR8,
176 UMCCH_UMC_CONFIG_ADDR9,
177 UMCCH_UMC_CONFIG_ADDR10,
178 UMCCH_UMC_CONFIG_ADDR11,
179 UMCCH_UMC_CONFIG_ADDR12,
180 UMCCH_UMC_CONFIG_ADDR13,
181 UMCCH_UMC_CONFIG_ADDR14,
182 UMCCH_UMC_CONFIG_ADDR15,
183};
184
185static const uint32_t ecc_umcch_eccctrl_addrs[] = {
186 UMCCH_ECCCTRL_ADDR0,
187 UMCCH_ECCCTRL_ADDR1,
188 UMCCH_ECCCTRL_ADDR2,
189 UMCCH_ECCCTRL_ADDR3,
190 UMCCH_ECCCTRL_ADDR4,
191 UMCCH_ECCCTRL_ADDR5,
192 UMCCH_ECCCTRL_ADDR6,
193 UMCCH_ECCCTRL_ADDR7,
194 UMCCH_ECCCTRL_ADDR8,
195 UMCCH_ECCCTRL_ADDR9,
196 UMCCH_ECCCTRL_ADDR10,
197 UMCCH_ECCCTRL_ADDR11,
198 UMCCH_ECCCTRL_ADDR12,
199 UMCCH_ECCCTRL_ADDR13,
200 UMCCH_ECCCTRL_ADDR14,
201 UMCCH_ECCCTRL_ADDR15,
202};
203
204static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
205 struct amdgpu_irq_src *src,
206 unsigned type,
207 enum amdgpu_interrupt_state state)
208{
209 struct amdgpu_vmhub *hub;
210 u32 tmp, reg, bits, i, j;
211
212 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
213 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
214 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
215 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
216 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
217 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
218 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
219
220 switch (state) {
221 case AMDGPU_IRQ_STATE_DISABLE:
222 for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
223 hub = &adev->vmhub[j];
224 for (i = 0; i < 16; i++) {
225 reg = hub->vm_context0_cntl + i;
226 tmp = RREG32(reg);
227 tmp &= ~bits;
228 WREG32(reg, tmp);
229 }
230 }
231 break;
232 case AMDGPU_IRQ_STATE_ENABLE:
233 for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
234 hub = &adev->vmhub[j];
235 for (i = 0; i < 16; i++) {
236 reg = hub->vm_context0_cntl + i;
237 tmp = RREG32(reg);
238 tmp |= bits;
239 WREG32(reg, tmp);
240 }
241 }
242 default:
243 break;
244 }
245
246 return 0;
247}
248
249static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
250 struct amdgpu_irq_src *source,
251 struct amdgpu_iv_entry *entry)
252{
253 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
254 uint32_t status = 0;
255 u64 addr;
256
257 addr = (u64)entry->src_data[0] << 12;
258 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
259
260 if (!amdgpu_sriov_vf(adev)) {
261 status = RREG32(hub->vm_l2_pro_fault_status);
262 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
263 }
264
265 if (printk_ratelimit()) {
266 dev_err(adev->dev,
267 "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",
268 entry->vmid_src ? "mmhub" : "gfxhub",
269 entry->src_id, entry->ring_id, entry->vmid,
270 entry->pasid);
271 dev_err(adev->dev, " at page 0x%016llx from %d\n",
272 addr, entry->client_id);
273 if (!amdgpu_sriov_vf(adev))
274 dev_err(adev->dev,
275 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
276 status);
277 }
278
279 return 0;
280}
281
282static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
283 .set = gmc_v9_0_vm_fault_interrupt_state,
284 .process = gmc_v9_0_process_interrupt,
285};
286
287static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
288{
289 adev->gmc.vm_fault.num_types = 1;
290 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
291}
292
293static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
294{
295 u32 req = 0;
296
297
298 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
299 PER_VMID_INVALIDATE_REQ, 1 << vmid);
300 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
301 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
302 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
303 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
304 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
305 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
306 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
307 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
308
309 return req;
310}
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
328 uint32_t vmid)
329{
330
331 const unsigned eng = 17;
332 unsigned i, j;
333
334 spin_lock(&adev->gmc.invalidate_lock);
335
336 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
337 struct amdgpu_vmhub *hub = &adev->vmhub[i];
338 u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
339
340 WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
341
342
343 for (j = 0; j < 100; j++) {
344 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
345 tmp &= 1 << vmid;
346 if (tmp)
347 break;
348 cpu_relax();
349 }
350 if (j < 100)
351 continue;
352
353
354 for (j = 0; j < adev->usec_timeout; j++) {
355 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
356 tmp &= 1 << vmid;
357 if (tmp)
358 break;
359 udelay(1);
360 }
361 if (j < adev->usec_timeout)
362 continue;
363
364 DRM_ERROR("Timeout waiting for VM flush ACK!\n");
365 }
366
367 spin_unlock(&adev->gmc.invalidate_lock);
368}
369
370static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
371 unsigned vmid, uint64_t pd_addr)
372{
373 struct amdgpu_device *adev = ring->adev;
374 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
375 uint32_t req = gmc_v9_0_get_invalidate_req(vmid);
376 uint64_t flags = AMDGPU_PTE_VALID;
377 unsigned eng = ring->vm_inv_eng;
378
379 amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags);
380 pd_addr |= flags;
381
382 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
383 lower_32_bits(pd_addr));
384
385 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
386 upper_32_bits(pd_addr));
387
388 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req);
389
390
391 amdgpu_ring_emit_reg_wait(ring, hub->vm_inv_eng0_ack + eng,
392 1 << vmid, 1 << vmid);
393
394 return pd_addr;
395}
396
397static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
398 unsigned pasid)
399{
400 struct amdgpu_device *adev = ring->adev;
401 uint32_t reg;
402
403 if (ring->funcs->vmhub == AMDGPU_GFXHUB)
404 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
405 else
406 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
407
408 amdgpu_ring_emit_wreg(ring, reg, pasid);
409}
410
411
412
413
414
415
416
417
418
419
420
421
422static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
423 uint32_t gpu_page_idx, uint64_t addr,
424 uint64_t flags)
425{
426 void __iomem *ptr = (void *)cpu_pt_addr;
427 uint64_t value;
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464 value = addr & 0x0000FFFFFFFFF000ULL;
465 value |= flags;
466 writeq(value, ptr + (gpu_page_idx * 8));
467 return 0;
468}
469
470static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
471 uint32_t flags)
472
473{
474 uint64_t pte_flag = 0;
475
476 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
477 pte_flag |= AMDGPU_PTE_EXECUTABLE;
478 if (flags & AMDGPU_VM_PAGE_READABLE)
479 pte_flag |= AMDGPU_PTE_READABLE;
480 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
481 pte_flag |= AMDGPU_PTE_WRITEABLE;
482
483 switch (flags & AMDGPU_VM_MTYPE_MASK) {
484 case AMDGPU_VM_MTYPE_DEFAULT:
485 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
486 break;
487 case AMDGPU_VM_MTYPE_NC:
488 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
489 break;
490 case AMDGPU_VM_MTYPE_WC:
491 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC);
492 break;
493 case AMDGPU_VM_MTYPE_CC:
494 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC);
495 break;
496 case AMDGPU_VM_MTYPE_UC:
497 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC);
498 break;
499 default:
500 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
501 break;
502 }
503
504 if (flags & AMDGPU_VM_PAGE_PRT)
505 pte_flag |= AMDGPU_PTE_PRT;
506
507 return pte_flag;
508}
509
510static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
511 uint64_t *addr, uint64_t *flags)
512{
513 if (!(*flags & AMDGPU_PDE_PTE))
514 *addr = adev->vm_manager.vram_base_offset + *addr -
515 adev->gmc.vram_start;
516 BUG_ON(*addr & 0xFFFF00000000003FULL);
517
518 if (!adev->gmc.translate_further)
519 return;
520
521 if (level == AMDGPU_VM_PDB1) {
522
523 if (!(*flags & AMDGPU_PDE_PTE))
524 *flags |= AMDGPU_PDE_BFS(0x9);
525
526 } else if (level == AMDGPU_VM_PDB0) {
527 if (*flags & AMDGPU_PDE_PTE)
528 *flags &= ~AMDGPU_PDE_PTE;
529 else
530 *flags |= AMDGPU_PTE_TF;
531 }
532}
533
534static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
535 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
536 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
537 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
538 .set_pte_pde = gmc_v9_0_set_pte_pde,
539 .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
540 .get_vm_pde = gmc_v9_0_get_vm_pde
541};
542
543static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
544{
545 if (adev->gmc.gmc_funcs == NULL)
546 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
547}
548
549static int gmc_v9_0_early_init(void *handle)
550{
551 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
552
553 gmc_v9_0_set_gmc_funcs(adev);
554 gmc_v9_0_set_irq_funcs(adev);
555
556 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
557 adev->gmc.shared_aperture_end =
558 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
559 adev->gmc.private_aperture_start =
560 adev->gmc.shared_aperture_end + 1;
561 adev->gmc.private_aperture_end =
562 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
563
564 return 0;
565}
566
567static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)
568{
569 uint32_t reg_val;
570 uint32_t reg_addr;
571 uint32_t field_val;
572 size_t i;
573 uint32_t fv2;
574 size_t lost_sheep;
575
576 DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n");
577
578 lost_sheep = 0;
579 for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) {
580 reg_addr = ecc_umclocalcap_addrs[i];
581 DRM_DEBUG("ecc: "
582 "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n",
583 i, reg_addr);
584 reg_val = RREG32(reg_addr);
585 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap,
586 EccDis);
587 DRM_DEBUG("ecc: "
588 "reg_val: 0x%08x, "
589 "EccDis: 0x%08x, ",
590 reg_val, field_val);
591 if (field_val) {
592 DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n");
593 ++lost_sheep;
594 }
595 }
596
597 for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) {
598 reg_addr = ecc_umcch_umc_config_addrs[i];
599 DRM_DEBUG("ecc: "
600 "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x",
601 i, reg_addr);
602 reg_val = RREG32(reg_addr);
603 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG,
604 DramReady);
605 DRM_DEBUG("ecc: "
606 "reg_val: 0x%08x, "
607 "DramReady: 0x%08x\n",
608 reg_val, field_val);
609
610 if (!field_val) {
611 DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n");
612 ++lost_sheep;
613 }
614 }
615
616 for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) {
617 reg_addr = ecc_umcch_eccctrl_addrs[i];
618 DRM_DEBUG("ecc: "
619 "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ",
620 i, reg_addr);
621 reg_val = RREG32(reg_addr);
622 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
623 WrEccEn);
624 fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
625 RdEccEn);
626 DRM_DEBUG("ecc: "
627 "reg_val: 0x%08x, "
628 "WrEccEn: 0x%08x, "
629 "RdEccEn: 0x%08x\n",
630 reg_val, field_val, fv2);
631
632 if (!field_val) {
633 DRM_DEBUG("ecc: WrEccEn is not set\n");
634 ++lost_sheep;
635 }
636 if (!fv2) {
637 DRM_DEBUG("ecc: RdEccEn is not set\n");
638 ++lost_sheep;
639 }
640 }
641
642 DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep);
643 return lost_sheep == 0;
644}
645
646static int gmc_v9_0_late_init(void *handle)
647{
648 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
649
650
651
652
653
654
655
656
657
658 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
659 unsigned i;
660 int r;
661
662 for(i = 0; i < adev->num_rings; ++i) {
663 struct amdgpu_ring *ring = adev->rings[i];
664 unsigned vmhub = ring->funcs->vmhub;
665
666 ring->vm_inv_eng = vm_inv_eng[vmhub]++;
667 dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",
668 ring->idx, ring->name, ring->vm_inv_eng,
669 ring->funcs->vmhub);
670 }
671
672
673 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
674 BUG_ON(vm_inv_eng[i] > 16);
675
676 if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) {
677 r = gmc_v9_0_ecc_available(adev);
678 if (r == 1) {
679 DRM_INFO("ECC is active.\n");
680 } else if (r == 0) {
681 DRM_INFO("ECC is not present.\n");
682 } else {
683 DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
684 return r;
685 }
686 }
687
688 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
689}
690
691static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
692 struct amdgpu_gmc *mc)
693{
694 u64 base = 0;
695 if (!amdgpu_sriov_vf(adev))
696 base = mmhub_v1_0_get_fb_location(adev);
697 amdgpu_device_vram_location(adev, &adev->gmc, base);
698 amdgpu_device_gart_location(adev, mc);
699
700 if (adev->flags & AMD_IS_APU)
701 adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
702 else
703 adev->vm_manager.vram_base_offset = 0;
704}
705
706
707
708
709
710
711
712
713
714
715static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
716{
717 u32 tmp;
718 int chansize, numchan;
719 int r;
720
721 if (amdgpu_emu_mode != 1)
722 adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
723 if (!adev->gmc.vram_width) {
724
725 if (adev->flags & AMD_IS_APU)
726 chansize = 64;
727 else
728 chansize = 128;
729
730 tmp = RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0);
731 tmp &= DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK;
732 tmp >>= DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
733 switch (tmp) {
734 case 0:
735 default:
736 numchan = 1;
737 break;
738 case 1:
739 numchan = 2;
740 break;
741 case 2:
742 numchan = 0;
743 break;
744 case 3:
745 numchan = 4;
746 break;
747 case 4:
748 numchan = 0;
749 break;
750 case 5:
751 numchan = 8;
752 break;
753 case 6:
754 numchan = 0;
755 break;
756 case 7:
757 numchan = 16;
758 break;
759 case 8:
760 numchan = 2;
761 break;
762 }
763 adev->gmc.vram_width = numchan * chansize;
764 }
765
766
767 adev->gmc.mc_vram_size =
768 adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;
769 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
770
771 if (!(adev->flags & AMD_IS_APU)) {
772 r = amdgpu_device_resize_fb_bar(adev);
773 if (r)
774 return r;
775 }
776 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
777 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
778
779#ifdef CONFIG_X86_64
780 if (adev->flags & AMD_IS_APU) {
781 adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev);
782 adev->gmc.aper_size = adev->gmc.real_vram_size;
783 }
784#endif
785
786 adev->gmc.visible_vram_size = adev->gmc.aper_size;
787 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
788 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
789
790
791 if (amdgpu_gart_size == -1) {
792 switch (adev->asic_type) {
793 case CHIP_VEGA10:
794 case CHIP_VEGA12:
795 default:
796 adev->gmc.gart_size = 512ULL << 20;
797 break;
798 case CHIP_RAVEN:
799 adev->gmc.gart_size = 1024ULL << 20;
800 break;
801 }
802 } else {
803 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
804 }
805
806 gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
807
808 return 0;
809}
810
811static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
812{
813 int r;
814
815 if (adev->gart.robj) {
816 WARN(1, "VEGA10 PCIE GART already initialized\n");
817 return 0;
818 }
819
820 r = amdgpu_gart_init(adev);
821 if (r)
822 return r;
823 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
824 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) |
825 AMDGPU_PTE_EXECUTABLE;
826 return amdgpu_gart_table_vram_alloc(adev);
827}
828
829static int gmc_v9_0_sw_init(void *handle)
830{
831 int r;
832 int dma_bits;
833 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
834
835 gfxhub_v1_0_init(adev);
836 mmhub_v1_0_init(adev);
837
838 spin_lock_init(&adev->gmc.invalidate_lock);
839
840 adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev);
841 switch (adev->asic_type) {
842 case CHIP_RAVEN:
843 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
844 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
845 } else {
846
847 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
848 adev->gmc.translate_further =
849 adev->vm_manager.num_level > 1;
850 }
851 break;
852 case CHIP_VEGA10:
853 case CHIP_VEGA12:
854
855
856
857
858
859 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
860 break;
861 default:
862 break;
863 }
864
865
866 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, 0,
867 &adev->gmc.vm_fault);
868 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, 0,
869 &adev->gmc.vm_fault);
870
871 if (r)
872 return r;
873
874
875
876
877
878 adev->gmc.mc_mask = 0xffffffffffffULL;
879
880
881
882
883
884 adev->gmc.stolen_size = 8 * 1024 * 1024;
885
886
887
888
889
890
891 adev->need_dma32 = false;
892 dma_bits = adev->need_dma32 ? 32 : 44;
893 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
894 if (r) {
895 adev->need_dma32 = true;
896 dma_bits = 32;
897 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
898 }
899 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
900 if (r) {
901 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
902 printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
903 }
904 adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
905
906 r = gmc_v9_0_mc_init(adev);
907 if (r)
908 return r;
909
910
911 r = amdgpu_bo_init(adev);
912 if (r)
913 return r;
914
915 r = gmc_v9_0_gart_init(adev);
916 if (r)
917 return r;
918
919
920
921
922
923
924
925 adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
926 adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
927
928 amdgpu_vm_manager_init(adev);
929
930 return 0;
931}
932
933
934
935
936
937
938
939
940static void gmc_v9_0_gart_fini(struct amdgpu_device *adev)
941{
942 amdgpu_gart_table_vram_free(adev);
943 amdgpu_gart_fini(adev);
944}
945
946static int gmc_v9_0_sw_fini(void *handle)
947{
948 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
949
950 amdgpu_gem_force_release(adev);
951 amdgpu_vm_manager_fini(adev);
952 gmc_v9_0_gart_fini(adev);
953 amdgpu_bo_fini(adev);
954
955 return 0;
956}
957
958static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
959{
960
961 switch (adev->asic_type) {
962 case CHIP_VEGA10:
963 soc15_program_register_sequence(adev,
964 golden_settings_mmhub_1_0_0,
965 ARRAY_SIZE(golden_settings_mmhub_1_0_0));
966 soc15_program_register_sequence(adev,
967 golden_settings_athub_1_0_0,
968 ARRAY_SIZE(golden_settings_athub_1_0_0));
969 break;
970 case CHIP_VEGA12:
971 break;
972 case CHIP_RAVEN:
973 soc15_program_register_sequence(adev,
974 golden_settings_athub_1_0_0,
975 ARRAY_SIZE(golden_settings_athub_1_0_0));
976 break;
977 default:
978 break;
979 }
980}
981
982
983
984
985
986
987static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
988{
989 int r;
990 bool value;
991 u32 tmp;
992
993 amdgpu_device_program_register_sequence(adev,
994 golden_settings_vega10_hdp,
995 ARRAY_SIZE(golden_settings_vega10_hdp));
996
997 if (adev->gart.robj == NULL) {
998 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
999 return -EINVAL;
1000 }
1001 r = amdgpu_gart_table_vram_pin(adev);
1002 if (r)
1003 return r;
1004
1005 switch (adev->asic_type) {
1006 case CHIP_RAVEN:
1007 mmhub_v1_0_initialize_power_gating(adev);
1008 mmhub_v1_0_update_power_gating(adev, true);
1009 break;
1010 default:
1011 break;
1012 }
1013
1014 r = gfxhub_v1_0_gart_enable(adev);
1015 if (r)
1016 return r;
1017
1018 r = mmhub_v1_0_gart_enable(adev);
1019 if (r)
1020 return r;
1021
1022 WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
1023
1024 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1025 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
1026
1027
1028 adev->nbio_funcs->hdp_flush(adev, NULL);
1029
1030 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1031 value = false;
1032 else
1033 value = true;
1034
1035 gfxhub_v1_0_set_fault_enable_default(adev, value);
1036 mmhub_v1_0_set_fault_enable_default(adev, value);
1037 gmc_v9_0_flush_gpu_tlb(adev, 0);
1038
1039 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1040 (unsigned)(adev->gmc.gart_size >> 20),
1041 (unsigned long long)adev->gart.table_addr);
1042 adev->gart.ready = true;
1043 return 0;
1044}
1045
1046static int gmc_v9_0_hw_init(void *handle)
1047{
1048 int r;
1049 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1050
1051
1052 gmc_v9_0_init_golden_registers(adev);
1053
1054 if (adev->mode_info.num_crtc) {
1055
1056 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1057
1058
1059 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1060 }
1061
1062 r = gmc_v9_0_gart_enable(adev);
1063
1064 return r;
1065}
1066
1067
1068
1069
1070
1071
1072
1073
1074static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1075{
1076 gfxhub_v1_0_gart_disable(adev);
1077 mmhub_v1_0_gart_disable(adev);
1078 amdgpu_gart_table_vram_unpin(adev);
1079}
1080
1081static int gmc_v9_0_hw_fini(void *handle)
1082{
1083 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1084
1085 if (amdgpu_sriov_vf(adev)) {
1086
1087 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1088 return 0;
1089 }
1090
1091 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1092 gmc_v9_0_gart_disable(adev);
1093
1094 return 0;
1095}
1096
1097static int gmc_v9_0_suspend(void *handle)
1098{
1099 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1100
1101 return gmc_v9_0_hw_fini(adev);
1102}
1103
1104static int gmc_v9_0_resume(void *handle)
1105{
1106 int r;
1107 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1108
1109 r = gmc_v9_0_hw_init(adev);
1110 if (r)
1111 return r;
1112
1113 amdgpu_vmid_reset_all(adev);
1114
1115 return 0;
1116}
1117
1118static bool gmc_v9_0_is_idle(void *handle)
1119{
1120
1121 return true;
1122}
1123
1124static int gmc_v9_0_wait_for_idle(void *handle)
1125{
1126
1127 return 0;
1128}
1129
1130static int gmc_v9_0_soft_reset(void *handle)
1131{
1132
1133 return 0;
1134}
1135
1136static int gmc_v9_0_set_clockgating_state(void *handle,
1137 enum amd_clockgating_state state)
1138{
1139 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1140
1141 return mmhub_v1_0_set_clockgating(adev, state);
1142}
1143
1144static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1145{
1146 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1147
1148 mmhub_v1_0_get_clockgating(adev, flags);
1149}
1150
1151static int gmc_v9_0_set_powergating_state(void *handle,
1152 enum amd_powergating_state state)
1153{
1154 return 0;
1155}
1156
1157const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1158 .name = "gmc_v9_0",
1159 .early_init = gmc_v9_0_early_init,
1160 .late_init = gmc_v9_0_late_init,
1161 .sw_init = gmc_v9_0_sw_init,
1162 .sw_fini = gmc_v9_0_sw_fini,
1163 .hw_init = gmc_v9_0_hw_init,
1164 .hw_fini = gmc_v9_0_hw_fini,
1165 .suspend = gmc_v9_0_suspend,
1166 .resume = gmc_v9_0_resume,
1167 .is_idle = gmc_v9_0_is_idle,
1168 .wait_for_idle = gmc_v9_0_wait_for_idle,
1169 .soft_reset = gmc_v9_0_soft_reset,
1170 .set_clockgating_state = gmc_v9_0_set_clockgating_state,
1171 .set_powergating_state = gmc_v9_0_set_powergating_state,
1172 .get_clockgating_state = gmc_v9_0_get_clockgating_state,
1173};
1174
1175const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1176{
1177 .type = AMD_IP_BLOCK_TYPE_GMC,
1178 .major = 9,
1179 .minor = 0,
1180 .rev = 0,
1181 .funcs = &gmc_v9_0_ip_funcs,
1182};
1183
