1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/firmware.h>
24#include <drm/drmP.h>
25#include "amdgpu.h"
26#include "gmc_v6_0.h"
27#include "amdgpu_ucode.h"
28
29#include "bif/bif_3_0_d.h"
30#include "bif/bif_3_0_sh_mask.h"
31#include "oss/oss_1_0_d.h"
32#include "oss/oss_1_0_sh_mask.h"
33#include "gmc/gmc_6_0_d.h"
34#include "gmc/gmc_6_0_sh_mask.h"
35#include "dce/dce_6_0_d.h"
36#include "dce/dce_6_0_sh_mask.h"
37#include "si_enums.h"
38
39static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev);
40static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
41static int gmc_v6_0_wait_for_idle(void *handle);
42
43MODULE_FIRMWARE("radeon/tahiti_mc.bin");
44MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
45MODULE_FIRMWARE("radeon/verde_mc.bin");
46MODULE_FIRMWARE("radeon/oland_mc.bin");
47MODULE_FIRMWARE("radeon/si58_mc.bin");
48
49#define MC_SEQ_MISC0__MT__MASK 0xf0000000
50#define MC_SEQ_MISC0__MT__GDDR1 0x10000000
51#define MC_SEQ_MISC0__MT__DDR2 0x20000000
52#define MC_SEQ_MISC0__MT__GDDR3 0x30000000
53#define MC_SEQ_MISC0__MT__GDDR4 0x40000000
54#define MC_SEQ_MISC0__MT__GDDR5 0x50000000
55#define MC_SEQ_MISC0__MT__HBM 0x60000000
56#define MC_SEQ_MISC0__MT__DDR3 0xB0000000
57
58
59static const u32 crtc_offsets[6] =
60{
61 SI_CRTC0_REGISTER_OFFSET,
62 SI_CRTC1_REGISTER_OFFSET,
63 SI_CRTC2_REGISTER_OFFSET,
64 SI_CRTC3_REGISTER_OFFSET,
65 SI_CRTC4_REGISTER_OFFSET,
66 SI_CRTC5_REGISTER_OFFSET
67};
68
69static void gmc_v6_0_mc_stop(struct amdgpu_device *adev,
70 struct amdgpu_mode_mc_save *save)
71{
72 u32 blackout;
73
74 if (adev->mode_info.num_crtc)
75 amdgpu_display_stop_mc_access(adev, save);
76
77 gmc_v6_0_wait_for_idle((void *)adev);
78
79 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
80 if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
81
82 WREG32(mmBIF_FB_EN, 0);
83
84 blackout = REG_SET_FIELD(blackout,
85 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
86 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
87 }
88
89 udelay(100);
90
91}
92
93static void gmc_v6_0_mc_resume(struct amdgpu_device *adev,
94 struct amdgpu_mode_mc_save *save)
95{
96 u32 tmp;
97
98
99 tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
100 tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
101 WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
102
103 tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
104 tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
105 WREG32(mmBIF_FB_EN, tmp);
106
107 if (adev->mode_info.num_crtc)
108 amdgpu_display_resume_mc_access(adev, save);
109
110}
111
112static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
113{
114 const char *chip_name;
115 char fw_name[30];
116 int err;
117 bool is_58_fw = false;
118
119 DRM_DEBUG("\n");
120
121 switch (adev->asic_type) {
122 case CHIP_TAHITI:
123 chip_name = "tahiti";
124 break;
125 case CHIP_PITCAIRN:
126 chip_name = "pitcairn";
127 break;
128 case CHIP_VERDE:
129 chip_name = "verde";
130 break;
131 case CHIP_OLAND:
132 chip_name = "oland";
133 break;
134 case CHIP_HAINAN:
135 chip_name = "hainan";
136 break;
137 default: BUG();
138 }
139
140
141 if (((RREG32(mmMC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
142 is_58_fw = true;
143
144 if (is_58_fw)
145 snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
146 else
147 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
148 err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
149 if (err)
150 goto out;
151
152 err = amdgpu_ucode_validate(adev->mc.fw);
153
154out:
155 if (err) {
156 dev_err(adev->dev,
157 "si_mc: Failed to load firmware \"%s\"\n",
158 fw_name);
159 release_firmware(adev->mc.fw);
160 adev->mc.fw = NULL;
161 }
162 return err;
163}
164
165static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
166{
167 const __le32 *new_fw_data = NULL;
168 u32 running;
169 const __le32 *new_io_mc_regs = NULL;
170 int i, regs_size, ucode_size;
171 const struct mc_firmware_header_v1_0 *hdr;
172
173 if (!adev->mc.fw)
174 return -EINVAL;
175
176 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
177
178 amdgpu_ucode_print_mc_hdr(&hdr->header);
179
180 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
181 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
182 new_io_mc_regs = (const __le32 *)
183 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
184 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
185 new_fw_data = (const __le32 *)
186 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
187
188 running = RREG32(mmMC_SEQ_SUP_CNTL) & MC_SEQ_SUP_CNTL__RUN_MASK;
189
190 if (running == 0) {
191
192
193 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
194 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
195
196
197 for (i = 0; i < regs_size; i++) {
198 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
199 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
200 }
201
202 for (i = 0; i < ucode_size; i++) {
203 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
204 }
205
206
207 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
208 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
209 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
210
211
212 for (i = 0; i < adev->usec_timeout; i++) {
213 if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0_MASK)
214 break;
215 udelay(1);
216 }
217 for (i = 0; i < adev->usec_timeout; i++) {
218 if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1_MASK)
219 break;
220 udelay(1);
221 }
222
223 }
224
225 return 0;
226}
227
228static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
229 struct amdgpu_mc *mc)
230{
231 if (mc->mc_vram_size > 0xFFC0000000ULL) {
232 dev_warn(adev->dev, "limiting VRAM\n");
233 mc->real_vram_size = 0xFFC0000000ULL;
234 mc->mc_vram_size = 0xFFC0000000ULL;
235 }
236 amdgpu_vram_location(adev, &adev->mc, 0);
237 adev->mc.gtt_base_align = 0;
238 amdgpu_gtt_location(adev, mc);
239}
240
241static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
242{
243 struct amdgpu_mode_mc_save save;
244 u32 tmp;
245 int i, j;
246
247
248 for (i = 0, j = 0; i < 32; i++, j += 0x6) {
249 WREG32((0xb05 + j), 0x00000000);
250 WREG32((0xb06 + j), 0x00000000);
251 WREG32((0xb07 + j), 0x00000000);
252 WREG32((0xb08 + j), 0x00000000);
253 WREG32((0xb09 + j), 0x00000000);
254 }
255 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
256
257 if (adev->mode_info.num_crtc)
258 amdgpu_display_set_vga_render_state(adev, false);
259
260 gmc_v6_0_mc_stop(adev, &save);
261
262 if (gmc_v6_0_wait_for_idle((void *)adev)) {
263 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
264 }
265
266 WREG32(mmVGA_HDP_CONTROL, VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK);
267
268 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
269 adev->mc.vram_start >> 12);
270 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
271 adev->mc.vram_end >> 12);
272 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
273 adev->vram_scratch.gpu_addr >> 12);
274 tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
275 tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
276 WREG32(mmMC_VM_FB_LOCATION, tmp);
277
278 WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
279 WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
280 WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
281 WREG32(mmMC_VM_AGP_BASE, 0);
282 WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
283 WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
284
285 if (gmc_v6_0_wait_for_idle((void *)adev)) {
286 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
287 }
288 gmc_v6_0_mc_resume(adev, &save);
289}
290
291static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
292{
293
294 u32 tmp;
295 int chansize, numchan;
296
297 tmp = RREG32(mmMC_ARB_RAMCFG);
298 if (tmp & (1 << 11)) {
299 chansize = 16;
300 } else if (tmp & MC_ARB_RAMCFG__CHANSIZE_MASK) {
301 chansize = 64;
302 } else {
303 chansize = 32;
304 }
305 tmp = RREG32(mmMC_SHARED_CHMAP);
306 switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
307 case 0:
308 default:
309 numchan = 1;
310 break;
311 case 1:
312 numchan = 2;
313 break;
314 case 2:
315 numchan = 4;
316 break;
317 case 3:
318 numchan = 8;
319 break;
320 case 4:
321 numchan = 3;
322 break;
323 case 5:
324 numchan = 6;
325 break;
326 case 6:
327 numchan = 10;
328 break;
329 case 7:
330 numchan = 12;
331 break;
332 case 8:
333 numchan = 16;
334 break;
335 }
336 adev->mc.vram_width = numchan * chansize;
337
338 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
339 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
340
341 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
342 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
343 adev->mc.visible_vram_size = adev->mc.aper_size;
344
345
346
347
348 if (amdgpu_gart_size == -1)
349 adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
350 adev->mc.mc_vram_size);
351 else
352 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
353
354 gmc_v6_0_vram_gtt_location(adev, &adev->mc);
355
356 return 0;
357}
358
359static void gmc_v6_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
360 uint32_t vmid)
361{
362 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
363
364 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
365}
366
367static int gmc_v6_0_gart_set_pte_pde(struct amdgpu_device *adev,
368 void *cpu_pt_addr,
369 uint32_t gpu_page_idx,
370 uint64_t addr,
371 uint64_t flags)
372{
373 void __iomem *ptr = (void *)cpu_pt_addr;
374 uint64_t value;
375
376 value = addr & 0xFFFFFFFFFFFFF000ULL;
377 value |= flags;
378 writeq(value, ptr + (gpu_page_idx * 8));
379
380 return 0;
381}
382
383static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev,
384 uint32_t flags)
385{
386 uint64_t pte_flag = 0;
387
388 if (flags & AMDGPU_VM_PAGE_READABLE)
389 pte_flag |= AMDGPU_PTE_READABLE;
390 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
391 pte_flag |= AMDGPU_PTE_WRITEABLE;
392 if (flags & AMDGPU_VM_PAGE_PRT)
393 pte_flag |= AMDGPU_PTE_PRT;
394
395 return pte_flag;
396}
397
398static uint64_t gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
399{
400 BUG_ON(addr & 0xFFFFFF0000000FFFULL);
401 return addr;
402}
403
404static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
405 bool value)
406{
407 u32 tmp;
408
409 tmp = RREG32(mmVM_CONTEXT1_CNTL);
410 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
411 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
412 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
413 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
414 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
415 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
416 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
417 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
418 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
419 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
420 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
421 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
422 WREG32(mmVM_CONTEXT1_CNTL, tmp);
423}
424
425
426
427
428
429
430
431static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
432{
433 u32 tmp;
434
435 if (enable && !adev->mc.prt_warning) {
436 dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
437 adev->mc.prt_warning = true;
438 }
439
440 tmp = RREG32(mmVM_PRT_CNTL);
441 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
442 CB_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
443 enable);
444 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
445 TC_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
446 enable);
447 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
448 L2_CACHE_STORE_INVALID_ENTRIES,
449 enable);
450 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
451 L1_TLB_STORE_INVALID_ENTRIES,
452 enable);
453 WREG32(mmVM_PRT_CNTL, tmp);
454
455 if (enable) {
456 uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
457 uint32_t high = adev->vm_manager.max_pfn;
458
459 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
460 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
461 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
462 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
463 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
464 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
465 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
466 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
467 } else {
468 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
469 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
470 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
471 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
472 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
473 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
474 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
475 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
476 }
477}
478
479static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
480{
481 int r, i;
482
483 if (adev->gart.robj == NULL) {
484 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
485 return -EINVAL;
486 }
487 r = amdgpu_gart_table_vram_pin(adev);
488 if (r)
489 return r;
490
491 WREG32(mmMC_VM_MX_L1_TLB_CNTL,
492 (0xA << 7) |
493 MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK |
494 MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING_MASK |
495 MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
496 MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK |
497 (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
498
499 WREG32(mmVM_L2_CNTL,
500 VM_L2_CNTL__ENABLE_L2_CACHE_MASK |
501 VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK |
502 VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
503 VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
504 (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
505 (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
506 WREG32(mmVM_L2_CNTL2,
507 VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
508 VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
509 WREG32(mmVM_L2_CNTL3,
510 VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
511 (4UL << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
512 (4UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
513
514 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
515 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
516 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
517 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
518 (u32)(adev->dummy_page.addr >> 12));
519 WREG32(mmVM_CONTEXT0_CNTL2, 0);
520 WREG32(mmVM_CONTEXT0_CNTL,
521 VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK |
522 (0UL << VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
523 VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK);
524
525 WREG32(0x575, 0);
526 WREG32(0x576, 0);
527 WREG32(0x577, 0);
528
529
530
531 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
532 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
533
534
535
536
537 for (i = 1; i < 16; i++) {
538 if (i < 8)
539 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
540 adev->gart.table_addr >> 12);
541 else
542 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
543 adev->gart.table_addr >> 12);
544 }
545
546
547 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
548 (u32)(adev->dummy_page.addr >> 12));
549 WREG32(mmVM_CONTEXT1_CNTL2, 4);
550 WREG32(mmVM_CONTEXT1_CNTL,
551 VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK |
552 (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
553 ((adev->vm_manager.block_size - 9)
554 << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT));
555 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
556 gmc_v6_0_set_fault_enable_default(adev, false);
557 else
558 gmc_v6_0_set_fault_enable_default(adev, true);
559
560 gmc_v6_0_gart_flush_gpu_tlb(adev, 0);
561 dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
562 (unsigned)(adev->mc.gtt_size >> 20),
563 (unsigned long long)adev->gart.table_addr);
564 adev->gart.ready = true;
565 return 0;
566}
567
568static int gmc_v6_0_gart_init(struct amdgpu_device *adev)
569{
570 int r;
571
572 if (adev->gart.robj) {
573 dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");
574 return 0;
575 }
576 r = amdgpu_gart_init(adev);
577 if (r)
578 return r;
579 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
580 adev->gart.gart_pte_flags = 0;
581 return amdgpu_gart_table_vram_alloc(adev);
582}
583
584static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)
585{
586
587
588
589
590
591
592
593
594
595
596
597
598 WREG32(mmVM_CONTEXT0_CNTL, 0);
599 WREG32(mmVM_CONTEXT1_CNTL, 0);
600
601 WREG32(mmMC_VM_MX_L1_TLB_CNTL,
602 MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
603 (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
604
605 WREG32(mmVM_L2_CNTL,
606 VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
607 VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
608 (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
609 (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
610 WREG32(mmVM_L2_CNTL2, 0);
611 WREG32(mmVM_L2_CNTL3,
612 VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
613 (0UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
614 amdgpu_gart_table_vram_unpin(adev);
615}
616
617static void gmc_v6_0_gart_fini(struct amdgpu_device *adev)
618{
619 amdgpu_gart_table_vram_free(adev);
620 amdgpu_gart_fini(adev);
621}
622
623static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
624 u32 status, u32 addr, u32 mc_client)
625{
626 u32 mc_id;
627 u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
628 u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
629 PROTECTIONS);
630 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
631 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
632
633 mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
634 MEMORY_CLIENT_ID);
635
636 dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
637 protections, vmid, addr,
638 REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
639 MEMORY_CLIENT_RW) ?
640 "write" : "read", block, mc_client, mc_id);
641}
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770static int gmc_v6_0_convert_vram_type(int mc_seq_vram_type)
771{
772 switch (mc_seq_vram_type) {
773 case MC_SEQ_MISC0__MT__GDDR1:
774 return AMDGPU_VRAM_TYPE_GDDR1;
775 case MC_SEQ_MISC0__MT__DDR2:
776 return AMDGPU_VRAM_TYPE_DDR2;
777 case MC_SEQ_MISC0__MT__GDDR3:
778 return AMDGPU_VRAM_TYPE_GDDR3;
779 case MC_SEQ_MISC0__MT__GDDR4:
780 return AMDGPU_VRAM_TYPE_GDDR4;
781 case MC_SEQ_MISC0__MT__GDDR5:
782 return AMDGPU_VRAM_TYPE_GDDR5;
783 case MC_SEQ_MISC0__MT__DDR3:
784 return AMDGPU_VRAM_TYPE_DDR3;
785 default:
786 return AMDGPU_VRAM_TYPE_UNKNOWN;
787 }
788}
789
790static int gmc_v6_0_early_init(void *handle)
791{
792 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
793
794 gmc_v6_0_set_gart_funcs(adev);
795 gmc_v6_0_set_irq_funcs(adev);
796
797 return 0;
798}
799
800static int gmc_v6_0_late_init(void *handle)
801{
802 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
803
804 if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
805 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
806 else
807 return 0;
808}
809
810static int gmc_v6_0_sw_init(void *handle)
811{
812 int r;
813 int dma_bits;
814 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
815
816 if (adev->flags & AMD_IS_APU) {
817 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
818 } else {
819 u32 tmp = RREG32(mmMC_SEQ_MISC0);
820 tmp &= MC_SEQ_MISC0__MT__MASK;
821 adev->mc.vram_type = gmc_v6_0_convert_vram_type(tmp);
822 }
823
824 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
825 if (r)
826 return r;
827
828 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
829 if (r)
830 return r;
831
832 amdgpu_vm_adjust_size(adev, 64);
833 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
834
835 adev->mc.mc_mask = 0xffffffffffULL;
836
837 adev->mc.stolen_size = 256 * 1024;
838
839 adev->need_dma32 = false;
840 dma_bits = adev->need_dma32 ? 32 : 40;
841 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
842 if (r) {
843 adev->need_dma32 = true;
844 dma_bits = 32;
845 dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
846 }
847 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
848 if (r) {
849 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
850 dev_warn(adev->dev, "amdgpu: No coherent DMA available.\n");
851 }
852
853 r = gmc_v6_0_init_microcode(adev);
854 if (r) {
855 dev_err(adev->dev, "Failed to load mc firmware!\n");
856 return r;
857 }
858
859 r = gmc_v6_0_mc_init(adev);
860 if (r)
861 return r;
862
863 r = amdgpu_bo_init(adev);
864 if (r)
865 return r;
866
867 r = gmc_v6_0_gart_init(adev);
868 if (r)
869 return r;
870
871
872
873
874
875
876
877 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
878 adev->vm_manager.num_level = 1;
879 amdgpu_vm_manager_init(adev);
880
881
882 if (adev->flags & AMD_IS_APU) {
883 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
884
885 tmp <<= 22;
886 adev->vm_manager.vram_base_offset = tmp;
887 } else {
888 adev->vm_manager.vram_base_offset = 0;
889 }
890
891 return 0;
892}
893
894static int gmc_v6_0_sw_fini(void *handle)
895{
896 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
897
898 amdgpu_vm_manager_fini(adev);
899 gmc_v6_0_gart_fini(adev);
900 amdgpu_gem_force_release(adev);
901 amdgpu_bo_fini(adev);
902
903 return 0;
904}
905
906static int gmc_v6_0_hw_init(void *handle)
907{
908 int r;
909 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
910
911 gmc_v6_0_mc_program(adev);
912
913 if (!(adev->flags & AMD_IS_APU)) {
914 r = gmc_v6_0_mc_load_microcode(adev);
915 if (r) {
916 dev_err(adev->dev, "Failed to load MC firmware!\n");
917 return r;
918 }
919 }
920
921 r = gmc_v6_0_gart_enable(adev);
922 if (r)
923 return r;
924
925 return r;
926}
927
928static int gmc_v6_0_hw_fini(void *handle)
929{
930 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
931
932 amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
933 gmc_v6_0_gart_disable(adev);
934
935 return 0;
936}
937
938static int gmc_v6_0_suspend(void *handle)
939{
940 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
941
942 gmc_v6_0_hw_fini(adev);
943
944 return 0;
945}
946
947static int gmc_v6_0_resume(void *handle)
948{
949 int r;
950 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
951
952 r = gmc_v6_0_hw_init(adev);
953 if (r)
954 return r;
955
956 amdgpu_vm_reset_all_ids(adev);
957
958 return 0;
959}
960
961static bool gmc_v6_0_is_idle(void *handle)
962{
963 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
964 u32 tmp = RREG32(mmSRBM_STATUS);
965
966 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
967 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
968 return false;
969
970 return true;
971}
972
973static int gmc_v6_0_wait_for_idle(void *handle)
974{
975 unsigned i;
976 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
977
978 for (i = 0; i < adev->usec_timeout; i++) {
979 if (gmc_v6_0_is_idle(handle))
980 return 0;
981 udelay(1);
982 }
983 return -ETIMEDOUT;
984
985}
986
987static int gmc_v6_0_soft_reset(void *handle)
988{
989 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
990 struct amdgpu_mode_mc_save save;
991 u32 srbm_soft_reset = 0;
992 u32 tmp = RREG32(mmSRBM_STATUS);
993
994 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
995 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
996 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
997
998 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
999 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1000 if (!(adev->flags & AMD_IS_APU))
1001 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1002 SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1003 }
1004
1005 if (srbm_soft_reset) {
1006 gmc_v6_0_mc_stop(adev, &save);
1007 if (gmc_v6_0_wait_for_idle(adev)) {
1008 dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1009 }
1010
1011
1012 tmp = RREG32(mmSRBM_SOFT_RESET);
1013 tmp |= srbm_soft_reset;
1014 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1015 WREG32(mmSRBM_SOFT_RESET, tmp);
1016 tmp = RREG32(mmSRBM_SOFT_RESET);
1017
1018 udelay(50);
1019
1020 tmp &= ~srbm_soft_reset;
1021 WREG32(mmSRBM_SOFT_RESET, tmp);
1022 tmp = RREG32(mmSRBM_SOFT_RESET);
1023
1024 udelay(50);
1025
1026 gmc_v6_0_mc_resume(adev, &save);
1027 udelay(50);
1028 }
1029
1030 return 0;
1031}
1032
1033static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1034 struct amdgpu_irq_src *src,
1035 unsigned type,
1036 enum amdgpu_interrupt_state state)
1037{
1038 u32 tmp;
1039 u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1040 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1041 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1042 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1043 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1044 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1045
1046 switch (state) {
1047 case AMDGPU_IRQ_STATE_DISABLE:
1048 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1049 tmp &= ~bits;
1050 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1051 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1052 tmp &= ~bits;
1053 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1054 break;
1055 case AMDGPU_IRQ_STATE_ENABLE:
1056 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1057 tmp |= bits;
1058 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1059 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1060 tmp |= bits;
1061 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1062 break;
1063 default:
1064 break;
1065 }
1066
1067 return 0;
1068}
1069
1070static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
1071 struct amdgpu_irq_src *source,
1072 struct amdgpu_iv_entry *entry)
1073{
1074 u32 addr, status;
1075
1076 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1077 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1078 WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1079
1080 if (!addr && !status)
1081 return 0;
1082
1083 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1084 gmc_v6_0_set_fault_enable_default(adev, false);
1085
1086 if (printk_ratelimit()) {
1087 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1088 entry->src_id, entry->src_data[0]);
1089 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1090 addr);
1091 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1092 status);
1093 gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
1094 }
1095
1096 return 0;
1097}
1098
1099static int gmc_v6_0_set_clockgating_state(void *handle,
1100 enum amd_clockgating_state state)
1101{
1102 return 0;
1103}
1104
1105static int gmc_v6_0_set_powergating_state(void *handle,
1106 enum amd_powergating_state state)
1107{
1108 return 0;
1109}
1110
1111static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
1112 .name = "gmc_v6_0",
1113 .early_init = gmc_v6_0_early_init,
1114 .late_init = gmc_v6_0_late_init,
1115 .sw_init = gmc_v6_0_sw_init,
1116 .sw_fini = gmc_v6_0_sw_fini,
1117 .hw_init = gmc_v6_0_hw_init,
1118 .hw_fini = gmc_v6_0_hw_fini,
1119 .suspend = gmc_v6_0_suspend,
1120 .resume = gmc_v6_0_resume,
1121 .is_idle = gmc_v6_0_is_idle,
1122 .wait_for_idle = gmc_v6_0_wait_for_idle,
1123 .soft_reset = gmc_v6_0_soft_reset,
1124 .set_clockgating_state = gmc_v6_0_set_clockgating_state,
1125 .set_powergating_state = gmc_v6_0_set_powergating_state,
1126};
1127
1128static const struct amdgpu_gart_funcs gmc_v6_0_gart_funcs = {
1129 .flush_gpu_tlb = gmc_v6_0_gart_flush_gpu_tlb,
1130 .set_pte_pde = gmc_v6_0_gart_set_pte_pde,
1131 .set_prt = gmc_v6_0_set_prt,
1132 .get_vm_pde = gmc_v6_0_get_vm_pde,
1133 .get_vm_pte_flags = gmc_v6_0_get_vm_pte_flags
1134};
1135
1136static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = {
1137 .set = gmc_v6_0_vm_fault_interrupt_state,
1138 .process = gmc_v6_0_process_interrupt,
1139};
1140
1141static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev)
1142{
1143 if (adev->gart.gart_funcs == NULL)
1144 adev->gart.gart_funcs = &gmc_v6_0_gart_funcs;
1145}
1146
1147static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1148{
1149 adev->mc.vm_fault.num_types = 1;
1150 adev->mc.vm_fault.funcs = &gmc_v6_0_irq_funcs;
1151}
1152
1153const struct amdgpu_ip_block_version gmc_v6_0_ip_block =
1154{
1155 .type = AMD_IP_BLOCK_TYPE_GMC,
1156 .major = 6,
1157 .minor = 0,
1158 .rev = 0,
1159 .funcs = &gmc_v6_0_ip_funcs,
1160};
1161
