1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/firmware.h>
25#include "amdgpu.h"
26#include "amdgpu_vcn.h"
27#include "amdgpu_pm.h"
28#include "soc15.h"
29#include "soc15d.h"
30#include "vcn_v2_0.h"
31#include "mmsch_v3_0.h"
32
33#include "vcn/vcn_3_0_0_offset.h"
34#include "vcn/vcn_3_0_0_sh_mask.h"
35#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
36
37#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
38#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
39#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
40#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
41#define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
42#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
43#define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
44
45#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
46#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
47#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
48#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
49
50#define VCN_INSTANCES_SIENNA_CICHLID 2
51#define DEC_SW_RING_ENABLED FALSE
52
53static int amdgpu_ih_clientid_vcns[] = {
54 SOC15_IH_CLIENTID_VCN,
55 SOC15_IH_CLIENTID_VCN1
56};
57
58static int amdgpu_ucode_id_vcns[] = {
59 AMDGPU_UCODE_ID_VCN,
60 AMDGPU_UCODE_ID_VCN1
61};
62
63static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
64static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
65static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
66static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
67static int vcn_v3_0_set_powergating_state(void *handle,
68 enum amd_powergating_state state);
69static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
70 int inst_idx, struct dpg_pause_state *new_state);
71
72static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
73static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
74
75
76
77
78
79
80
81
82static int vcn_v3_0_early_init(void *handle)
83{
84 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
85
86 if (amdgpu_sriov_vf(adev)) {
87 adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
88 adev->vcn.harvest_config = 0;
89 adev->vcn.num_enc_rings = 1;
90
91 } else {
92 if (adev->asic_type == CHIP_SIENNA_CICHLID) {
93 u32 harvest;
94 int i;
95
96 adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
97 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
98 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);
99 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)
100 adev->vcn.harvest_config |= 1 << i;
101 }
102
103 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
104 AMDGPU_VCN_HARVEST_VCN1))
105
106 return -ENOENT;
107 } else
108 adev->vcn.num_vcn_inst = 1;
109
110 adev->vcn.num_enc_rings = 2;
111 }
112
113 vcn_v3_0_set_dec_ring_funcs(adev);
114 vcn_v3_0_set_enc_ring_funcs(adev);
115 vcn_v3_0_set_irq_funcs(adev);
116
117 return 0;
118}
119
120
121
122
123
124
125
126
127static int vcn_v3_0_sw_init(void *handle)
128{
129 struct amdgpu_ring *ring;
130 int i, j, r;
131 int vcn_doorbell_index = 0;
132 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
133
134 r = amdgpu_vcn_sw_init(adev);
135 if (r)
136 return r;
137
138 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
139 const struct common_firmware_header *hdr;
140 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
141 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
142 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
143 adev->firmware.fw_size +=
144 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
145
146 if (adev->vcn.num_vcn_inst == VCN_INSTANCES_SIENNA_CICHLID) {
147 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].ucode_id = AMDGPU_UCODE_ID_VCN1;
148 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].fw = adev->vcn.fw;
149 adev->firmware.fw_size +=
150 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
151 }
152 DRM_INFO("PSP loading VCN firmware\n");
153 }
154
155 r = amdgpu_vcn_resume(adev);
156 if (r)
157 return r;
158
159
160
161
162
163
164
165
166 if (amdgpu_sriov_vf(adev)) {
167 vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
168
169 vcn_doorbell_index = vcn_doorbell_index << 1;
170 }
171
172 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
173 volatile struct amdgpu_fw_shared *fw_shared;
174 if (adev->vcn.harvest_config & (1 << i))
175 continue;
176
177 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
178 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
179 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
180 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
181 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
182 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
183
184 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
185 adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
186 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
187 adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
188 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
189 adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
190 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
191 adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
192 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
193 adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
194
195
196 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
197 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
198 if (r)
199 return r;
200
201 ring = &adev->vcn.inst[i].ring_dec;
202 ring->use_doorbell = true;
203 if (amdgpu_sriov_vf(adev)) {
204 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
205 } else {
206 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
207 }
208 if (adev->asic_type == CHIP_SIENNA_CICHLID && i != 0)
209 ring->no_scheduler = true;
210 sprintf(ring->name, "vcn_dec_%d", i);
211 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
212 AMDGPU_RING_PRIO_DEFAULT);
213 if (r)
214 return r;
215
216 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
217
218 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
219 j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
220 if (r)
221 return r;
222
223 ring = &adev->vcn.inst[i].ring_enc[j];
224 ring->use_doorbell = true;
225 if (amdgpu_sriov_vf(adev)) {
226 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
227 } else {
228 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
229 }
230 if (adev->asic_type == CHIP_SIENNA_CICHLID && i != 1)
231 ring->no_scheduler = true;
232 sprintf(ring->name, "vcn_enc_%d.%d", i, j);
233 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
234 AMDGPU_RING_PRIO_DEFAULT);
235 if (r)
236 return r;
237 }
238
239 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
240 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
241 cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
242 fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
243 }
244
245 if (amdgpu_sriov_vf(adev)) {
246 r = amdgpu_virt_alloc_mm_table(adev);
247 if (r)
248 return r;
249 }
250 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
251 adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
252
253 return 0;
254}
255
256
257
258
259
260
261
262
263static int vcn_v3_0_sw_fini(void *handle)
264{
265 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
266 int i, r;
267
268 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
269 volatile struct amdgpu_fw_shared *fw_shared;
270
271 if (adev->vcn.harvest_config & (1 << i))
272 continue;
273 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
274 fw_shared->present_flag_0 = 0;
275 fw_shared->sw_ring.is_enabled = false;
276 }
277
278 if (amdgpu_sriov_vf(adev))
279 amdgpu_virt_free_mm_table(adev);
280
281 r = amdgpu_vcn_suspend(adev);
282 if (r)
283 return r;
284
285 r = amdgpu_vcn_sw_fini(adev);
286
287 return r;
288}
289
290
291
292
293
294
295
296
297static int vcn_v3_0_hw_init(void *handle)
298{
299 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
300 struct amdgpu_ring *ring;
301 int i, j, r;
302
303 if (amdgpu_sriov_vf(adev)) {
304 r = vcn_v3_0_start_sriov(adev);
305 if (r)
306 goto done;
307
308
309 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
310 if (adev->vcn.harvest_config & (1 << i))
311 continue;
312
313 ring = &adev->vcn.inst[i].ring_dec;
314 if (ring->sched.ready) {
315 ring->wptr = 0;
316 ring->wptr_old = 0;
317 vcn_v3_0_dec_ring_set_wptr(ring);
318 }
319
320 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
321 ring = &adev->vcn.inst[i].ring_enc[j];
322 if (ring->sched.ready) {
323 ring->wptr = 0;
324 ring->wptr_old = 0;
325 vcn_v3_0_enc_ring_set_wptr(ring);
326 }
327 }
328 }
329 } else {
330 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
331 if (adev->vcn.harvest_config & (1 << i))
332 continue;
333
334 ring = &adev->vcn.inst[i].ring_dec;
335
336 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
337 ring->doorbell_index, i);
338
339 r = amdgpu_ring_test_helper(ring);
340 if (r)
341 goto done;
342
343 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
344 ring = &adev->vcn.inst[i].ring_enc[j];
345 r = amdgpu_ring_test_helper(ring);
346 if (r)
347 goto done;
348 }
349 }
350 }
351
352done:
353 if (!r)
354 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
355 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
356
357 return r;
358}
359
360
361
362
363
364
365
366
367static int vcn_v3_0_hw_fini(void *handle)
368{
369 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
370 struct amdgpu_ring *ring;
371 int i, j;
372
373 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
374 if (adev->vcn.harvest_config & (1 << i))
375 continue;
376
377 ring = &adev->vcn.inst[i].ring_dec;
378
379 if (!amdgpu_sriov_vf(adev)) {
380 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
381 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
382 RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
383 vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
384 }
385 }
386 ring->sched.ready = false;
387
388 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
389 ring = &adev->vcn.inst[i].ring_enc[j];
390 ring->sched.ready = false;
391 }
392 }
393
394 return 0;
395}
396
397
398
399
400
401
402
403
404static int vcn_v3_0_suspend(void *handle)
405{
406 int r;
407 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
408
409 r = vcn_v3_0_hw_fini(adev);
410 if (r)
411 return r;
412
413 r = amdgpu_vcn_suspend(adev);
414
415 return r;
416}
417
418
419
420
421
422
423
424
425static int vcn_v3_0_resume(void *handle)
426{
427 int r;
428 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
429
430 r = amdgpu_vcn_resume(adev);
431 if (r)
432 return r;
433
434 r = vcn_v3_0_hw_init(adev);
435
436 return r;
437}
438
439
440
441
442
443
444
445
446
447static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
448{
449 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
450 uint32_t offset;
451
452
453 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
454 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
455 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
456 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
457 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
458 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
459 offset = 0;
460 } else {
461 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
462 lower_32_bits(adev->vcn.inst[inst].gpu_addr));
463 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
464 upper_32_bits(adev->vcn.inst[inst].gpu_addr));
465 offset = size;
466 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
467 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
468 }
469 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
470
471
472 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
473 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
474 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
475 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
476 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
477 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
478
479
480 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
481 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
482 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
483 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
484 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
485 WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
486
487
488 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
489 lower_32_bits(adev->vcn.inst[inst].fw_shared_gpu_addr));
490 WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
491 upper_32_bits(adev->vcn.inst[inst].fw_shared_gpu_addr));
492 WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
493 WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
494 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
495}
496
497static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
498{
499 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
500 uint32_t offset;
501
502
503 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
504 if (!indirect) {
505 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
506 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
507 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
508 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
509 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
510 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
511 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
512 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
513 } else {
514 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
515 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
516 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
517 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
518 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
519 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
520 }
521 offset = 0;
522 } else {
523 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
524 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
525 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
526 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
527 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
528 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
529 offset = size;
530 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
531 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
532 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
533 }
534
535 if (!indirect)
536 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
537 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
538 else
539 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
540 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
541
542
543 if (!indirect) {
544 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
545 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
546 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
547 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
548 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
549 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
550 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
551 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
552 } else {
553 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
554 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
555 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
556 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
557 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
558 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
559 }
560 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
561 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
562
563
564 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
565 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
566 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
567 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
568 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
569 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
570 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
571 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
572 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
573 VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
574
575
576 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
577 VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
578 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
579 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
580 VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
581 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
582 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
583 VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
584 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
585 VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
586 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
587}
588
589static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
590{
591 uint32_t data = 0;
592
593 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
594 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
595 | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
596 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
597 | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
598 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
599 | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
600 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
601 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
602 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
603 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
604 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
605 | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
606 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
607 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
608
609 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
610 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
611 UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
612 } else {
613 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
614 | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
615 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
616 | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
617 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
618 | 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
619 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
620 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
621 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
622 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
623 | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
624 | 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
625 | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
626 | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
627 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
628 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0, 0x3F3FFFFF);
629 }
630
631 data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
632 data &= ~0x103;
633 if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
634 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
635 UVD_POWER_STATUS__UVD_PG_EN_MASK;
636
637 WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
638}
639
640static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
641{
642 uint32_t data;
643
644 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
645
646 data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
647 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
648 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
649 WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
650
651 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
652 | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
653 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
654 | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
655 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
656 | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
657 | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
658 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
659 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
660 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
661 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
662 | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
663 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
664 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
665 WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
666
667 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
668 | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
669 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
670 | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
671 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
672 | 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
673 | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
674 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
675 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
676 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
677 | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
678 | 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
679 | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
680 | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
681 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
682 }
683}
684
685
686
687
688
689
690
691
692
693static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
694{
695 uint32_t data;
696
697
698 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
699 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
700 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
701 else
702 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
703 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
704 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
705 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
706
707 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
708 data &= ~(UVD_CGC_GATE__SYS_MASK
709 | UVD_CGC_GATE__UDEC_MASK
710 | UVD_CGC_GATE__MPEG2_MASK
711 | UVD_CGC_GATE__REGS_MASK
712 | UVD_CGC_GATE__RBC_MASK
713 | UVD_CGC_GATE__LMI_MC_MASK
714 | UVD_CGC_GATE__LMI_UMC_MASK
715 | UVD_CGC_GATE__IDCT_MASK
716 | UVD_CGC_GATE__MPRD_MASK
717 | UVD_CGC_GATE__MPC_MASK
718 | UVD_CGC_GATE__LBSI_MASK
719 | UVD_CGC_GATE__LRBBM_MASK
720 | UVD_CGC_GATE__UDEC_RE_MASK
721 | UVD_CGC_GATE__UDEC_CM_MASK
722 | UVD_CGC_GATE__UDEC_IT_MASK
723 | UVD_CGC_GATE__UDEC_DB_MASK
724 | UVD_CGC_GATE__UDEC_MP_MASK
725 | UVD_CGC_GATE__WCB_MASK
726 | UVD_CGC_GATE__VCPU_MASK
727 | UVD_CGC_GATE__MMSCH_MASK);
728
729 WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
730
731 SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);
732
733 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
734 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
735 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
736 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
737 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
738 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
739 | UVD_CGC_CTRL__SYS_MODE_MASK
740 | UVD_CGC_CTRL__UDEC_MODE_MASK
741 | UVD_CGC_CTRL__MPEG2_MODE_MASK
742 | UVD_CGC_CTRL__REGS_MODE_MASK
743 | UVD_CGC_CTRL__RBC_MODE_MASK
744 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
745 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
746 | UVD_CGC_CTRL__IDCT_MODE_MASK
747 | UVD_CGC_CTRL__MPRD_MODE_MASK
748 | UVD_CGC_CTRL__MPC_MODE_MASK
749 | UVD_CGC_CTRL__LBSI_MODE_MASK
750 | UVD_CGC_CTRL__LRBBM_MODE_MASK
751 | UVD_CGC_CTRL__WCB_MODE_MASK
752 | UVD_CGC_CTRL__VCPU_MODE_MASK
753 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
754 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
755
756 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
757 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
758 | UVD_SUVD_CGC_GATE__SIT_MASK
759 | UVD_SUVD_CGC_GATE__SMP_MASK
760 | UVD_SUVD_CGC_GATE__SCM_MASK
761 | UVD_SUVD_CGC_GATE__SDB_MASK
762 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
763 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
764 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
765 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
766 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
767 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
768 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
769 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
770 | UVD_SUVD_CGC_GATE__SCLR_MASK
771 | UVD_SUVD_CGC_GATE__ENT_MASK
772 | UVD_SUVD_CGC_GATE__IME_MASK
773 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
774 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
775 | UVD_SUVD_CGC_GATE__SITE_MASK
776 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
777 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
778 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
779 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
780 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK
781 | UVD_SUVD_CGC_GATE__EFC_MASK
782 | UVD_SUVD_CGC_GATE__SAOE_MASK
783 | UVD_SUVD_CGC_GATE__SRE_AV1_MASK
784 | UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
785 | UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
786 | UVD_SUVD_CGC_GATE__SCM_AV1_MASK
787 | UVD_SUVD_CGC_GATE__SMPA_MASK);
788 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
789
790 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
791 data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
792 | UVD_SUVD_CGC_GATE2__MPBE1_MASK
793 | UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
794 | UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
795 | UVD_SUVD_CGC_GATE2__MPC1_MASK);
796 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
797
798 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
799 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
800 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
801 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
802 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
803 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
804 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
805 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
806 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
807 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
808 | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
809 | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
810 | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
811 | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
812 | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
813 | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
814 | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
815 | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
816 | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
817 | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
818 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
819}
820
821static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
822 uint8_t sram_sel, int inst_idx, uint8_t indirect)
823{
824 uint32_t reg_data = 0;
825
826
827 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
828 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
829 else
830 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
831 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
832 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
833 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
834 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
835 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
836 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
837 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
838 UVD_CGC_CTRL__SYS_MODE_MASK |
839 UVD_CGC_CTRL__UDEC_MODE_MASK |
840 UVD_CGC_CTRL__MPEG2_MODE_MASK |
841 UVD_CGC_CTRL__REGS_MODE_MASK |
842 UVD_CGC_CTRL__RBC_MODE_MASK |
843 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
844 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
845 UVD_CGC_CTRL__IDCT_MODE_MASK |
846 UVD_CGC_CTRL__MPRD_MODE_MASK |
847 UVD_CGC_CTRL__MPC_MODE_MASK |
848 UVD_CGC_CTRL__LBSI_MODE_MASK |
849 UVD_CGC_CTRL__LRBBM_MODE_MASK |
850 UVD_CGC_CTRL__WCB_MODE_MASK |
851 UVD_CGC_CTRL__VCPU_MODE_MASK |
852 UVD_CGC_CTRL__MMSCH_MODE_MASK);
853 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
854 VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
855
856
857 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
858 VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
859
860
861 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
862 VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
863
864
865 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
866 VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
867}
868
869
870
871
872
873
874
875
876
877static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
878{
879 uint32_t data;
880
881
882 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
883 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
884 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
885 else
886 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
887 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
888 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
889 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
890
891 data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
892 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
893 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
894 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
895 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
896 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
897 | UVD_CGC_CTRL__SYS_MODE_MASK
898 | UVD_CGC_CTRL__UDEC_MODE_MASK
899 | UVD_CGC_CTRL__MPEG2_MODE_MASK
900 | UVD_CGC_CTRL__REGS_MODE_MASK
901 | UVD_CGC_CTRL__RBC_MODE_MASK
902 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
903 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
904 | UVD_CGC_CTRL__IDCT_MODE_MASK
905 | UVD_CGC_CTRL__MPRD_MODE_MASK
906 | UVD_CGC_CTRL__MPC_MODE_MASK
907 | UVD_CGC_CTRL__LBSI_MODE_MASK
908 | UVD_CGC_CTRL__LRBBM_MODE_MASK
909 | UVD_CGC_CTRL__WCB_MODE_MASK
910 | UVD_CGC_CTRL__VCPU_MODE_MASK
911 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
912 WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
913
914 data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
915 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
916 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
917 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
918 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
919 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
920 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
921 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
922 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
923 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
924 | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
925 | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
926 | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
927 | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
928 | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
929 | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
930 | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
931 | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
932 | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
933 | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
934 WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
935}
936
937static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
938{
939 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
940 struct amdgpu_ring *ring;
941 uint32_t rb_bufsz, tmp;
942
943
944 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
945 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
946
947 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
948 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
949 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
950 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
951
952 if (indirect)
953 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
954
955
956 vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
957
958
959 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
960 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
961 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
962 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
963 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
964
965
966 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
967 VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
968
969
970 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
971 UVD_LMI_CTRL__REQ_MODE_MASK |
972 UVD_LMI_CTRL__CRC_RESET_MASK |
973 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
974 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
975 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
976 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
977 0x00100000L);
978 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
979 VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
980
981 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
982 VCN, inst_idx, mmUVD_MPC_CNTL),
983 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
984
985 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
986 VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
987 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
988 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
989 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
990 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
991
992 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
993 VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
994 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
995 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
996 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
997 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
998
999 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1000 VCN, inst_idx, mmUVD_MPC_SET_MUX),
1001 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1002 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1003 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1004
1005 vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
1006
1007 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1008 VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1009 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1010 VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1011
1012
1013 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1014 VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1015
1016
1017 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1018 VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1019
1020 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1021 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1022 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1023 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1024
1025
1026 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1027 VCN, inst_idx, mmUVD_MASTINT_EN),
1028 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1029
1030
1031 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1032 VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1033
1034 if (indirect)
1035 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
1036 (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
1037 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
1038
1039 ring = &adev->vcn.inst[inst_idx].ring_dec;
1040
1041 rb_bufsz = order_base_2(ring->ring_size);
1042 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1043 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1044 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1045 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1046 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1047 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1048
1049
1050 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1051 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1052 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1053 fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1054
1055
1056 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1057
1058
1059 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1060 (upper_32_bits(ring->gpu_addr) >> 2));
1061
1062
1063 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1064 lower_32_bits(ring->gpu_addr));
1065 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1066 upper_32_bits(ring->gpu_addr));
1067
1068
1069 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1070
1071 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1072
1073 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1074 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1075 lower_32_bits(ring->wptr));
1076
1077 fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1078
1079 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1080 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1081
1082 return 0;
1083}
1084
1085static int vcn_v3_0_start(struct amdgpu_device *adev)
1086{
1087 volatile struct amdgpu_fw_shared *fw_shared;
1088 struct amdgpu_ring *ring;
1089 uint32_t rb_bufsz, tmp;
1090 int i, j, k, r;
1091
1092 if (adev->pm.dpm_enabled)
1093 amdgpu_dpm_enable_uvd(adev, true);
1094
1095 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1096 if (adev->vcn.harvest_config & (1 << i))
1097 continue;
1098
1099 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG){
1100 r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1101 continue;
1102 }
1103
1104
1105 vcn_v3_0_disable_static_power_gating(adev, i);
1106
1107
1108 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1109 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1110
1111
1112 vcn_v3_0_disable_clock_gating(adev, i);
1113
1114
1115 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1116 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1117
1118
1119 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1120 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1121
1122
1123 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1124 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1125
1126 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1127 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1128 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1129 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1130
1131
1132 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1133 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1134 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1135 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1136 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1137 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1138
1139
1140 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1141 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1142 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1143 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1144
1145
1146 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1147 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1148 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1149 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1150 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1151
1152
1153 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1154 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1155 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1156 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1157 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1158
1159
1160 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1161 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1162 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1163 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1164
1165 vcn_v3_0_mc_resume(adev, i);
1166
1167
1168 WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1169 adev->gfx.config.gb_addr_config);
1170
1171
1172 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1173 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1174
1175
1176 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1177 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1178
1179 for (j = 0; j < 10; ++j) {
1180 uint32_t status;
1181
1182 for (k = 0; k < 100; ++k) {
1183 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1184 if (status & 2)
1185 break;
1186 mdelay(10);
1187 }
1188 r = 0;
1189 if (status & 2)
1190 break;
1191
1192 DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1193 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1194 UVD_VCPU_CNTL__BLK_RST_MASK,
1195 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1196 mdelay(10);
1197 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1198 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1199
1200 mdelay(10);
1201 r = -1;
1202 }
1203
1204 if (r) {
1205 DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1206 return r;
1207 }
1208
1209
1210 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1211 UVD_MASTINT_EN__VCPU_EN_MASK,
1212 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1213
1214
1215 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1216 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1217
1218 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1219
1220 ring = &adev->vcn.inst[i].ring_dec;
1221
1222 rb_bufsz = order_base_2(ring->ring_size);
1223 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1224 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1225 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1226 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1227 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1228 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1229
1230 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
1231 fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1232
1233
1234 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1235 lower_32_bits(ring->gpu_addr));
1236 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1237 upper_32_bits(ring->gpu_addr));
1238
1239
1240 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1241
1242 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1243 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1244 lower_32_bits(ring->wptr));
1245 fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1246
1247 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1248 ring = &adev->vcn.inst[i].ring_enc[0];
1249 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1250 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1251 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1252 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1253 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1254 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1255
1256 fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1257 ring = &adev->vcn.inst[i].ring_enc[1];
1258 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1259 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1260 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1261 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1262 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1263 fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1264 }
1265
1266 return 0;
1267}
1268
1269static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1270{
1271 int i, j;
1272 struct amdgpu_ring *ring;
1273 uint64_t cache_addr;
1274 uint64_t rb_addr;
1275 uint64_t ctx_addr;
1276 uint32_t param, resp, expected;
1277 uint32_t offset, cache_size;
1278 uint32_t tmp, timeout;
1279 uint32_t id;
1280
1281 struct amdgpu_mm_table *table = &adev->virt.mm_table;
1282 uint32_t *table_loc;
1283 uint32_t table_size;
1284 uint32_t size, size_dw;
1285
1286 bool is_vcn_ready;
1287
1288 struct mmsch_v3_0_cmd_direct_write
1289 direct_wt = { {0} };
1290 struct mmsch_v3_0_cmd_direct_read_modify_write
1291 direct_rd_mod_wt = { {0} };
1292 struct mmsch_v3_0_cmd_end end = { {0} };
1293 struct mmsch_v3_0_init_header header;
1294
1295 direct_wt.cmd_header.command_type =
1296 MMSCH_COMMAND__DIRECT_REG_WRITE;
1297 direct_rd_mod_wt.cmd_header.command_type =
1298 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1299 end.cmd_header.command_type =
1300 MMSCH_COMMAND__END;
1301
1302 header.version = MMSCH_VERSION;
1303 header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1304 for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1305 header.inst[i].init_status = 0;
1306 header.inst[i].table_offset = 0;
1307 header.inst[i].table_size = 0;
1308 }
1309
1310 table_loc = (uint32_t *)table->cpu_addr;
1311 table_loc += header.total_size;
1312 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1313 if (adev->vcn.harvest_config & (1 << i))
1314 continue;
1315
1316 table_size = 0;
1317
1318 MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1319 mmUVD_STATUS),
1320 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1321
1322 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1323
1324 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1325 id = amdgpu_ucode_id_vcns[i];
1326 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1327 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1328 adev->firmware.ucode[id].tmr_mc_addr_lo);
1329 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1330 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1331 adev->firmware.ucode[id].tmr_mc_addr_hi);
1332 offset = 0;
1333 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1334 mmUVD_VCPU_CACHE_OFFSET0),
1335 0);
1336 } else {
1337 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1338 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1339 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1340 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1341 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1342 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1343 offset = cache_size;
1344 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1345 mmUVD_VCPU_CACHE_OFFSET0),
1346 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1347 }
1348
1349 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1350 mmUVD_VCPU_CACHE_SIZE0),
1351 cache_size);
1352
1353 cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1354 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1355 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1356 lower_32_bits(cache_addr));
1357 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1358 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1359 upper_32_bits(cache_addr));
1360 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1361 mmUVD_VCPU_CACHE_OFFSET1),
1362 0);
1363 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1364 mmUVD_VCPU_CACHE_SIZE1),
1365 AMDGPU_VCN_STACK_SIZE);
1366
1367 cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1368 AMDGPU_VCN_STACK_SIZE;
1369 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1370 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1371 lower_32_bits(cache_addr));
1372 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1373 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1374 upper_32_bits(cache_addr));
1375 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1376 mmUVD_VCPU_CACHE_OFFSET2),
1377 0);
1378 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1379 mmUVD_VCPU_CACHE_SIZE2),
1380 AMDGPU_VCN_CONTEXT_SIZE);
1381
1382 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1383 ring = &adev->vcn.inst[i].ring_enc[j];
1384 ring->wptr = 0;
1385 rb_addr = ring->gpu_addr;
1386 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1387 mmUVD_RB_BASE_LO),
1388 lower_32_bits(rb_addr));
1389 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1390 mmUVD_RB_BASE_HI),
1391 upper_32_bits(rb_addr));
1392 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1393 mmUVD_RB_SIZE),
1394 ring->ring_size / 4);
1395 }
1396
1397 ring = &adev->vcn.inst[i].ring_dec;
1398 ring->wptr = 0;
1399 rb_addr = ring->gpu_addr;
1400 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1401 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1402 lower_32_bits(rb_addr));
1403 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1404 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1405 upper_32_bits(rb_addr));
1406
1407 tmp = order_base_2(ring->ring_size);
1408 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1409 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1410 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1411 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1412 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1413 MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1414 mmUVD_RBC_RB_CNTL),
1415 tmp);
1416
1417
1418 MMSCH_V3_0_INSERT_END();
1419
1420
1421 header.inst[i].init_status = 0;
1422 header.inst[i].table_offset = header.total_size;
1423 header.inst[i].table_size = table_size;
1424 header.total_size += table_size;
1425 }
1426
1427
1428 size = sizeof(struct mmsch_v3_0_init_header);
1429 table_loc = (uint32_t *)table->cpu_addr;
1430 memcpy((void *)table_loc, &header, size);
1431
1432
1433
1434
1435
1436 ctx_addr = table->gpu_addr;
1437 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1438 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1439
1440
1441 tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1442 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1443
1444 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1445 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1446
1447
1448 size = header.total_size;
1449 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1450
1451
1452 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1453
1454
1455
1456
1457 param = 0x10000001;
1458 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1459 tmp = 0;
1460 timeout = 1000;
1461 resp = 0;
1462 expected = param + 1;
1463 while (resp != expected) {
1464 resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1465 if (resp == expected)
1466 break;
1467
1468 udelay(10);
1469 tmp = tmp + 10;
1470 if (tmp >= timeout) {
1471 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1472 " waiting for mmMMSCH_VF_MAILBOX_RESP "\
1473 "(expected=0x%08x, readback=0x%08x)\n",
1474 tmp, expected, resp);
1475 return -EBUSY;
1476 }
1477 }
1478
1479
1480
1481
1482
1483 size = sizeof(struct mmsch_v3_0_init_header);
1484 table_loc = (uint32_t *)table->cpu_addr;
1485 memcpy(&header, (void *)table_loc, size);
1486
1487 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1488 if (adev->vcn.harvest_config & (1 << i))
1489 continue;
1490
1491 is_vcn_ready = (header.inst[i].init_status == 1);
1492 if (!is_vcn_ready)
1493 DRM_INFO("VCN(%d) engine is disabled by hypervisor\n", i);
1494
1495 ring = &adev->vcn.inst[i].ring_dec;
1496 ring->sched.ready = is_vcn_ready;
1497 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1498 ring = &adev->vcn.inst[i].ring_enc[j];
1499 ring->sched.ready = is_vcn_ready;
1500 }
1501 }
1502
1503 return 0;
1504}
1505
1506static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1507{
1508 uint32_t tmp;
1509
1510
1511 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1512 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1513
1514
1515 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1516 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1517
1518 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1519 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1520
1521 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1522 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1523
1524 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1525 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1526
1527
1528 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1529 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1530
1531 return 0;
1532}
1533
1534static int vcn_v3_0_stop(struct amdgpu_device *adev)
1535{
1536 uint32_t tmp;
1537 int i, r = 0;
1538
1539 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1540 if (adev->vcn.harvest_config & (1 << i))
1541 continue;
1542
1543 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1544 r = vcn_v3_0_stop_dpg_mode(adev, i);
1545 continue;
1546 }
1547
1548
1549 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1550 if (r)
1551 return r;
1552
1553 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1554 UVD_LMI_STATUS__READ_CLEAN_MASK |
1555 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1556 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1557 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1558 if (r)
1559 return r;
1560
1561
1562 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1563 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1564 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1565 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1566 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1567 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1568 if (r)
1569 return r;
1570
1571
1572 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1573 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1574 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1575
1576
1577 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1578 UVD_VCPU_CNTL__BLK_RST_MASK,
1579 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1580
1581
1582 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1583 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1584
1585
1586 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1587 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1588 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1589 tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1590 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1591 WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1592
1593
1594 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1595
1596
1597 vcn_v3_0_enable_clock_gating(adev, i);
1598
1599
1600 vcn_v3_0_enable_static_power_gating(adev, i);
1601 }
1602
1603 if (adev->pm.dpm_enabled)
1604 amdgpu_dpm_enable_uvd(adev, false);
1605
1606 return 0;
1607}
1608
1609static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1610 int inst_idx, struct dpg_pause_state *new_state)
1611{
1612 volatile struct amdgpu_fw_shared *fw_shared;
1613 struct amdgpu_ring *ring;
1614 uint32_t reg_data = 0;
1615 int ret_code;
1616
1617
1618 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1619 DRM_DEBUG("dpg pause state changed %d -> %d",
1620 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1621 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1622 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1623
1624 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1625 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1626 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1627
1628 if (!ret_code) {
1629
1630 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1631 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1632
1633
1634 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1635 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1636 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1637
1638
1639 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1640 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1641 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1642
1643
1644 fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
1645 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1646 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1647 ring->wptr = 0;
1648 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1649 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1650 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1651 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1652 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1653 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1654
1655 fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1656 ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1657 ring->wptr = 0;
1658 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1659 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1660 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1661 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1662 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1663 fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1664
1665
1666 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1667 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1668
1669 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1670 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1671 }
1672 } else {
1673
1674 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1675 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1676 }
1677 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1678 }
1679
1680 return 0;
1681}
1682
1683
1684
1685
1686
1687
1688
1689
1690static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1691{
1692 struct amdgpu_device *adev = ring->adev;
1693
1694 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1695}
1696
1697
1698
1699
1700
1701
1702
1703
1704static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1705{
1706 struct amdgpu_device *adev = ring->adev;
1707
1708 if (ring->use_doorbell)
1709 return adev->wb.wb[ring->wptr_offs];
1710 else
1711 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1712}
1713
1714
1715
1716
1717
1718
1719
1720
1721static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1722{
1723 struct amdgpu_device *adev = ring->adev;
1724
1725 if (ring->use_doorbell) {
1726 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1727 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1728 } else {
1729 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1730 }
1731}
1732
1733static void vcn_v3_0_dec_sw_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1734 u64 seq, uint32_t flags)
1735{
1736 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1737
1738 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_FENCE);
1739 amdgpu_ring_write(ring, addr);
1740 amdgpu_ring_write(ring, upper_32_bits(addr));
1741 amdgpu_ring_write(ring, seq);
1742 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_TRAP);
1743}
1744
1745static void vcn_v3_0_dec_sw_ring_insert_end(struct amdgpu_ring *ring)
1746{
1747 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_END);
1748}
1749
1750static void vcn_v3_0_dec_sw_ring_emit_ib(struct amdgpu_ring *ring,
1751 struct amdgpu_job *job,
1752 struct amdgpu_ib *ib,
1753 uint32_t flags)
1754{
1755 uint32_t vmid = AMDGPU_JOB_GET_VMID(job);
1756
1757 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_IB);
1758 amdgpu_ring_write(ring, vmid);
1759 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1760 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1761 amdgpu_ring_write(ring, ib->length_dw);
1762}
1763
1764static void vcn_v3_0_dec_sw_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1765 uint32_t val, uint32_t mask)
1766{
1767 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_REG_WAIT);
1768 amdgpu_ring_write(ring, reg << 2);
1769 amdgpu_ring_write(ring, mask);
1770 amdgpu_ring_write(ring, val);
1771}
1772
1773static void vcn_v3_0_dec_sw_ring_emit_vm_flush(struct amdgpu_ring *ring,
1774 uint32_t vmid, uint64_t pd_addr)
1775{
1776 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1777 uint32_t data0, data1, mask;
1778
1779 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1780
1781
1782 data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
1783 data1 = lower_32_bits(pd_addr);
1784 mask = 0xffffffff;
1785 vcn_v3_0_dec_sw_ring_emit_reg_wait(ring, data0, data1, mask);
1786}
1787
1788static void vcn_v3_0_dec_sw_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
1789{
1790 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_REG_WRITE);
1791 amdgpu_ring_write(ring, reg << 2);
1792 amdgpu_ring_write(ring, val);
1793}
1794
1795static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1796 .type = AMDGPU_RING_TYPE_VCN_DEC,
1797 .align_mask = 0x3f,
1798 .nop = VCN_DEC_SW_CMD_NO_OP,
1799 .vmhub = AMDGPU_MMHUB_0,
1800 .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1801 .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1802 .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1803 .emit_frame_size =
1804 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1805 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1806 4 +
1807 5 + 5 +
1808 1,
1809 .emit_ib_size = 5,
1810 .emit_ib = vcn_v3_0_dec_sw_ring_emit_ib,
1811 .emit_fence = vcn_v3_0_dec_sw_ring_emit_fence,
1812 .emit_vm_flush = vcn_v3_0_dec_sw_ring_emit_vm_flush,
1813 .test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1814 .test_ib = NULL,
1815 .insert_nop = amdgpu_ring_insert_nop,
1816 .insert_end = vcn_v3_0_dec_sw_ring_insert_end,
1817 .pad_ib = amdgpu_ring_generic_pad_ib,
1818 .begin_use = amdgpu_vcn_ring_begin_use,
1819 .end_use = amdgpu_vcn_ring_end_use,
1820 .emit_wreg = vcn_v3_0_dec_sw_ring_emit_wreg,
1821 .emit_reg_wait = vcn_v3_0_dec_sw_ring_emit_reg_wait,
1822 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1823};
1824
1825static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1826 .type = AMDGPU_RING_TYPE_VCN_DEC,
1827 .align_mask = 0xf,
1828 .vmhub = AMDGPU_MMHUB_0,
1829 .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1830 .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1831 .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1832 .emit_frame_size =
1833 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1834 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1835 8 +
1836 14 + 14 +
1837 6,
1838 .emit_ib_size = 8,
1839 .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1840 .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1841 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1842 .test_ring = vcn_v2_0_dec_ring_test_ring,
1843 .test_ib = amdgpu_vcn_dec_ring_test_ib,
1844 .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1845 .insert_start = vcn_v2_0_dec_ring_insert_start,
1846 .insert_end = vcn_v2_0_dec_ring_insert_end,
1847 .pad_ib = amdgpu_ring_generic_pad_ib,
1848 .begin_use = amdgpu_vcn_ring_begin_use,
1849 .end_use = amdgpu_vcn_ring_end_use,
1850 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1851 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1852 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1853};
1854
1855
1856
1857
1858
1859
1860
1861
1862static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1863{
1864 struct amdgpu_device *adev = ring->adev;
1865
1866 if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1867 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1868 else
1869 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1870}
1871
1872
1873
1874
1875
1876
1877
1878
1879static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1880{
1881 struct amdgpu_device *adev = ring->adev;
1882
1883 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1884 if (ring->use_doorbell)
1885 return adev->wb.wb[ring->wptr_offs];
1886 else
1887 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1888 } else {
1889 if (ring->use_doorbell)
1890 return adev->wb.wb[ring->wptr_offs];
1891 else
1892 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1893 }
1894}
1895
1896
1897
1898
1899
1900
1901
1902
1903static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1904{
1905 struct amdgpu_device *adev = ring->adev;
1906
1907 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1908 if (ring->use_doorbell) {
1909 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1910 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1911 } else {
1912 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1913 }
1914 } else {
1915 if (ring->use_doorbell) {
1916 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1917 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1918 } else {
1919 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1920 }
1921 }
1922}
1923
1924static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
1925 .type = AMDGPU_RING_TYPE_VCN_ENC,
1926 .align_mask = 0x3f,
1927 .nop = VCN_ENC_CMD_NO_OP,
1928 .vmhub = AMDGPU_MMHUB_0,
1929 .get_rptr = vcn_v3_0_enc_ring_get_rptr,
1930 .get_wptr = vcn_v3_0_enc_ring_get_wptr,
1931 .set_wptr = vcn_v3_0_enc_ring_set_wptr,
1932 .emit_frame_size =
1933 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1934 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1935 4 +
1936 5 + 5 +
1937 1,
1938 .emit_ib_size = 5,
1939 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1940 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1941 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1942 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1943 .test_ib = amdgpu_vcn_enc_ring_test_ib,
1944 .insert_nop = amdgpu_ring_insert_nop,
1945 .insert_end = vcn_v2_0_enc_ring_insert_end,
1946 .pad_ib = amdgpu_ring_generic_pad_ib,
1947 .begin_use = amdgpu_vcn_ring_begin_use,
1948 .end_use = amdgpu_vcn_ring_end_use,
1949 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1950 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1951 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1952};
1953
1954static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
1955{
1956 int i;
1957
1958 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1959 if (adev->vcn.harvest_config & (1 << i))
1960 continue;
1961
1962 if (!DEC_SW_RING_ENABLED)
1963 adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
1964 else
1965 adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
1966 adev->vcn.inst[i].ring_dec.me = i;
1967 DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
1968 DEC_SW_RING_ENABLED?"(Software Ring)":"");
1969 }
1970}
1971
1972static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
1973{
1974 int i, j;
1975
1976 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1977 if (adev->vcn.harvest_config & (1 << i))
1978 continue;
1979
1980 for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1981 adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
1982 adev->vcn.inst[i].ring_enc[j].me = i;
1983 }
1984 DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
1985 }
1986}
1987
1988static bool vcn_v3_0_is_idle(void *handle)
1989{
1990 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1991 int i, ret = 1;
1992
1993 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1994 if (adev->vcn.harvest_config & (1 << i))
1995 continue;
1996
1997 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
1998 }
1999
2000 return ret;
2001}
2002
2003static int vcn_v3_0_wait_for_idle(void *handle)
2004{
2005 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2006 int i, ret = 0;
2007
2008 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2009 if (adev->vcn.harvest_config & (1 << i))
2010 continue;
2011
2012 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2013 UVD_STATUS__IDLE);
2014 if (ret)
2015 return ret;
2016 }
2017
2018 return ret;
2019}
2020
2021static int vcn_v3_0_set_clockgating_state(void *handle,
2022 enum amd_clockgating_state state)
2023{
2024 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2025 bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
2026 int i;
2027
2028 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2029 if (adev->vcn.harvest_config & (1 << i))
2030 continue;
2031
2032 if (enable) {
2033 if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2034 return -EBUSY;
2035 vcn_v3_0_enable_clock_gating(adev, i);
2036 } else {
2037 vcn_v3_0_disable_clock_gating(adev, i);
2038 }
2039 }
2040
2041 return 0;
2042}
2043
2044static int vcn_v3_0_set_powergating_state(void *handle,
2045 enum amd_powergating_state state)
2046{
2047 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2048 int ret;
2049
2050
2051
2052
2053
2054 if (amdgpu_sriov_vf(adev)) {
2055 adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2056 return 0;
2057 }
2058
2059 if(state == adev->vcn.cur_state)
2060 return 0;
2061
2062 if (state == AMD_PG_STATE_GATE)
2063 ret = vcn_v3_0_stop(adev);
2064 else
2065 ret = vcn_v3_0_start(adev);
2066
2067 if(!ret)
2068 adev->vcn.cur_state = state;
2069
2070 return ret;
2071}
2072
2073static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2074 struct amdgpu_irq_src *source,
2075 unsigned type,
2076 enum amdgpu_interrupt_state state)
2077{
2078 return 0;
2079}
2080
2081static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2082 struct amdgpu_irq_src *source,
2083 struct amdgpu_iv_entry *entry)
2084{
2085 uint32_t ip_instance;
2086
2087 switch (entry->client_id) {
2088 case SOC15_IH_CLIENTID_VCN:
2089 ip_instance = 0;
2090 break;
2091 case SOC15_IH_CLIENTID_VCN1:
2092 ip_instance = 1;
2093 break;
2094 default:
2095 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2096 return 0;
2097 }
2098
2099 DRM_DEBUG("IH: VCN TRAP\n");
2100
2101 switch (entry->src_id) {
2102 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2103 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2104 break;
2105 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2106 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2107 break;
2108 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2109 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2110 break;
2111 default:
2112 DRM_ERROR("Unhandled interrupt: %d %d\n",
2113 entry->src_id, entry->src_data[0]);
2114 break;
2115 }
2116
2117 return 0;
2118}
2119
2120static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2121 .set = vcn_v3_0_set_interrupt_state,
2122 .process = vcn_v3_0_process_interrupt,
2123};
2124
2125static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2126{
2127 int i;
2128
2129 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2130 if (adev->vcn.harvest_config & (1 << i))
2131 continue;
2132
2133 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2134 adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2135 }
2136}
2137
2138static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2139 .name = "vcn_v3_0",
2140 .early_init = vcn_v3_0_early_init,
2141 .late_init = NULL,
2142 .sw_init = vcn_v3_0_sw_init,
2143 .sw_fini = vcn_v3_0_sw_fini,
2144 .hw_init = vcn_v3_0_hw_init,
2145 .hw_fini = vcn_v3_0_hw_fini,
2146 .suspend = vcn_v3_0_suspend,
2147 .resume = vcn_v3_0_resume,
2148 .is_idle = vcn_v3_0_is_idle,
2149 .wait_for_idle = vcn_v3_0_wait_for_idle,
2150 .check_soft_reset = NULL,
2151 .pre_soft_reset = NULL,
2152 .soft_reset = NULL,
2153 .post_soft_reset = NULL,
2154 .set_clockgating_state = vcn_v3_0_set_clockgating_state,
2155 .set_powergating_state = vcn_v3_0_set_powergating_state,
2156};
2157
2158const struct amdgpu_ip_block_version vcn_v3_0_ip_block =
2159{
2160 .type = AMD_IP_BLOCK_TYPE_VCN,
2161 .major = 3,
2162 .minor = 0,
2163 .rev = 0,
2164 .funcs = &vcn_v3_0_ip_funcs,
2165};
2166
