1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/cpufreq.h>
29#include <linux/module.h>
30#include <linux/pm_runtime.h>
31
32#include <drm/drm_atomic_helper.h>
33#include <drm/drm_fourcc.h>
34#include <drm/drm_plane_helper.h>
35
36#include "display/intel_atomic.h"
37#include "display/intel_fbc.h"
38#include "display/intel_sprite.h"
39
40#include "i915_drv.h"
41#include "i915_irq.h"
42#include "intel_drv.h"
43#include "intel_pm.h"
44#include "intel_sideband.h"
45#include "../../../platform/x86/intel_ips.h"
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
68{
69 if (HAS_LLC(dev_priv)) {
70
71
72
73
74
75
76
77 I915_WRITE(CHICKEN_PAR1_1,
78 I915_READ(CHICKEN_PAR1_1) |
79 SKL_DE_COMPRESSED_HASH_MODE);
80 }
81
82
83 I915_WRITE(CHICKEN_PAR1_1,
84 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
85
86
87 I915_WRITE(GEN8_CHICKEN_DCPR_1,
88 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
89
90
91
92 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
93 DISP_FBC_WM_DIS |
94 DISP_FBC_MEMORY_WAKE);
95
96
97 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
98 ILK_DPFC_DISABLE_DUMMY0);
99
100 if (IS_SKYLAKE(dev_priv)) {
101
102 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
103 & ~GEN7_DOP_CLOCK_GATE_ENABLE);
104 }
105}
106
107static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
108{
109 gen9_init_clock_gating(dev_priv);
110
111
112 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
113 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
114
115
116
117
118
119 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
120 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
121
122
123
124
125
126 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
127 PWM1_GATING_DIS | PWM2_GATING_DIS);
128}
129
130static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
131{
132 gen9_init_clock_gating(dev_priv);
133
134
135
136
137
138
139 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
140 PWM1_GATING_DIS | PWM2_GATING_DIS);
141
142
143 if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
144 u32 val = I915_READ(CHICKEN_MISC_2);
145 val &= ~(GLK_CL0_PWR_DOWN |
146 GLK_CL1_PWR_DOWN |
147 GLK_CL2_PWR_DOWN);
148 I915_WRITE(CHICKEN_MISC_2, val);
149 }
150
151}
152
153static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
154{
155 u32 tmp;
156
157 tmp = I915_READ(CLKCFG);
158
159 switch (tmp & CLKCFG_FSB_MASK) {
160 case CLKCFG_FSB_533:
161 dev_priv->fsb_freq = 533;
162 break;
163 case CLKCFG_FSB_800:
164 dev_priv->fsb_freq = 800;
165 break;
166 case CLKCFG_FSB_667:
167 dev_priv->fsb_freq = 667;
168 break;
169 case CLKCFG_FSB_400:
170 dev_priv->fsb_freq = 400;
171 break;
172 }
173
174 switch (tmp & CLKCFG_MEM_MASK) {
175 case CLKCFG_MEM_533:
176 dev_priv->mem_freq = 533;
177 break;
178 case CLKCFG_MEM_667:
179 dev_priv->mem_freq = 667;
180 break;
181 case CLKCFG_MEM_800:
182 dev_priv->mem_freq = 800;
183 break;
184 }
185
186
187 tmp = I915_READ(CSHRDDR3CTL);
188 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
189}
190
191static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
192{
193 u16 ddrpll, csipll;
194
195 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
196 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
197
198 switch (ddrpll & 0xff) {
199 case 0xc:
200 dev_priv->mem_freq = 800;
201 break;
202 case 0x10:
203 dev_priv->mem_freq = 1066;
204 break;
205 case 0x14:
206 dev_priv->mem_freq = 1333;
207 break;
208 case 0x18:
209 dev_priv->mem_freq = 1600;
210 break;
211 default:
212 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
213 ddrpll & 0xff);
214 dev_priv->mem_freq = 0;
215 break;
216 }
217
218 dev_priv->ips.r_t = dev_priv->mem_freq;
219
220 switch (csipll & 0x3ff) {
221 case 0x00c:
222 dev_priv->fsb_freq = 3200;
223 break;
224 case 0x00e:
225 dev_priv->fsb_freq = 3733;
226 break;
227 case 0x010:
228 dev_priv->fsb_freq = 4266;
229 break;
230 case 0x012:
231 dev_priv->fsb_freq = 4800;
232 break;
233 case 0x014:
234 dev_priv->fsb_freq = 5333;
235 break;
236 case 0x016:
237 dev_priv->fsb_freq = 5866;
238 break;
239 case 0x018:
240 dev_priv->fsb_freq = 6400;
241 break;
242 default:
243 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
244 csipll & 0x3ff);
245 dev_priv->fsb_freq = 0;
246 break;
247 }
248
249 if (dev_priv->fsb_freq == 3200) {
250 dev_priv->ips.c_m = 0;
251 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
252 dev_priv->ips.c_m = 1;
253 } else {
254 dev_priv->ips.c_m = 2;
255 }
256}
257
258static const struct cxsr_latency cxsr_latency_table[] = {
259 {1, 0, 800, 400, 3382, 33382, 3983, 33983},
260 {1, 0, 800, 667, 3354, 33354, 3807, 33807},
261 {1, 0, 800, 800, 3347, 33347, 3763, 33763},
262 {1, 1, 800, 667, 6420, 36420, 6873, 36873},
263 {1, 1, 800, 800, 5902, 35902, 6318, 36318},
264
265 {1, 0, 667, 400, 3400, 33400, 4021, 34021},
266 {1, 0, 667, 667, 3372, 33372, 3845, 33845},
267 {1, 0, 667, 800, 3386, 33386, 3822, 33822},
268 {1, 1, 667, 667, 6438, 36438, 6911, 36911},
269 {1, 1, 667, 800, 5941, 35941, 6377, 36377},
270
271 {1, 0, 400, 400, 3472, 33472, 4173, 34173},
272 {1, 0, 400, 667, 3443, 33443, 3996, 33996},
273 {1, 0, 400, 800, 3430, 33430, 3946, 33946},
274 {1, 1, 400, 667, 6509, 36509, 7062, 37062},
275 {1, 1, 400, 800, 5985, 35985, 6501, 36501},
276
277 {0, 0, 800, 400, 3438, 33438, 4065, 34065},
278 {0, 0, 800, 667, 3410, 33410, 3889, 33889},
279 {0, 0, 800, 800, 3403, 33403, 3845, 33845},
280 {0, 1, 800, 667, 6476, 36476, 6955, 36955},
281 {0, 1, 800, 800, 5958, 35958, 6400, 36400},
282
283 {0, 0, 667, 400, 3456, 33456, 4103, 34106},
284 {0, 0, 667, 667, 3428, 33428, 3927, 33927},
285 {0, 0, 667, 800, 3443, 33443, 3905, 33905},
286 {0, 1, 667, 667, 6494, 36494, 6993, 36993},
287 {0, 1, 667, 800, 5998, 35998, 6460, 36460},
288
289 {0, 0, 400, 400, 3528, 33528, 4255, 34255},
290 {0, 0, 400, 667, 3500, 33500, 4079, 34079},
291 {0, 0, 400, 800, 3487, 33487, 4029, 34029},
292 {0, 1, 400, 667, 6566, 36566, 7145, 37145},
293 {0, 1, 400, 800, 6042, 36042, 6584, 36584},
294};
295
296static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
297 bool is_ddr3,
298 int fsb,
299 int mem)
300{
301 const struct cxsr_latency *latency;
302 int i;
303
304 if (fsb == 0 || mem == 0)
305 return NULL;
306
307 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
308 latency = &cxsr_latency_table[i];
309 if (is_desktop == latency->is_desktop &&
310 is_ddr3 == latency->is_ddr3 &&
311 fsb == latency->fsb_freq && mem == latency->mem_freq)
312 return latency;
313 }
314
315 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
316
317 return NULL;
318}
319
320static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
321{
322 u32 val;
323
324 vlv_punit_get(dev_priv);
325
326 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
327 if (enable)
328 val &= ~FORCE_DDR_HIGH_FREQ;
329 else
330 val |= FORCE_DDR_HIGH_FREQ;
331 val &= ~FORCE_DDR_LOW_FREQ;
332 val |= FORCE_DDR_FREQ_REQ_ACK;
333 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
334
335 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
336 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
337 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
338
339 vlv_punit_put(dev_priv);
340}
341
342static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
343{
344 u32 val;
345
346 vlv_punit_get(dev_priv);
347
348 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
349 if (enable)
350 val |= DSP_MAXFIFO_PM5_ENABLE;
351 else
352 val &= ~DSP_MAXFIFO_PM5_ENABLE;
353 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
354
355 vlv_punit_put(dev_priv);
356}
357
358#define FW_WM(value, plane) \
359 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
360
361static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
362{
363 bool was_enabled;
364 u32 val;
365
366 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
367 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
368 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
369 POSTING_READ(FW_BLC_SELF_VLV);
370 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
371 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
372 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
373 POSTING_READ(FW_BLC_SELF);
374 } else if (IS_PINEVIEW(dev_priv)) {
375 val = I915_READ(DSPFW3);
376 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
377 if (enable)
378 val |= PINEVIEW_SELF_REFRESH_EN;
379 else
380 val &= ~PINEVIEW_SELF_REFRESH_EN;
381 I915_WRITE(DSPFW3, val);
382 POSTING_READ(DSPFW3);
383 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
384 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
385 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
386 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
387 I915_WRITE(FW_BLC_SELF, val);
388 POSTING_READ(FW_BLC_SELF);
389 } else if (IS_I915GM(dev_priv)) {
390
391
392
393
394
395 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
396 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
397 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
398 I915_WRITE(INSTPM, val);
399 POSTING_READ(INSTPM);
400 } else {
401 return false;
402 }
403
404 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
405
406 DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
407 enableddisabled(enable),
408 enableddisabled(was_enabled));
409
410 return was_enabled;
411}
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
451{
452 bool ret;
453
454 mutex_lock(&dev_priv->wm.wm_mutex);
455 ret = _intel_set_memory_cxsr(dev_priv, enable);
456 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
457 dev_priv->wm.vlv.cxsr = enable;
458 else if (IS_G4X(dev_priv))
459 dev_priv->wm.g4x.cxsr = enable;
460 mutex_unlock(&dev_priv->wm.wm_mutex);
461
462 return ret;
463}
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479static const int pessimal_latency_ns = 5000;
480
481#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
482 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
483
484static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
485{
486 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
487 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
488 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
489 enum pipe pipe = crtc->pipe;
490 int sprite0_start, sprite1_start;
491
492 switch (pipe) {
493 u32 dsparb, dsparb2, dsparb3;
494 case PIPE_A:
495 dsparb = I915_READ(DSPARB);
496 dsparb2 = I915_READ(DSPARB2);
497 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
498 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
499 break;
500 case PIPE_B:
501 dsparb = I915_READ(DSPARB);
502 dsparb2 = I915_READ(DSPARB2);
503 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
504 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
505 break;
506 case PIPE_C:
507 dsparb2 = I915_READ(DSPARB2);
508 dsparb3 = I915_READ(DSPARB3);
509 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
510 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
511 break;
512 default:
513 MISSING_CASE(pipe);
514 return;
515 }
516
517 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
518 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
519 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
520 fifo_state->plane[PLANE_CURSOR] = 63;
521}
522
523static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
524 enum i9xx_plane_id i9xx_plane)
525{
526 u32 dsparb = I915_READ(DSPARB);
527 int size;
528
529 size = dsparb & 0x7f;
530 if (i9xx_plane == PLANE_B)
531 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
532
533 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
534 dsparb, plane_name(i9xx_plane), size);
535
536 return size;
537}
538
539static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
540 enum i9xx_plane_id i9xx_plane)
541{
542 u32 dsparb = I915_READ(DSPARB);
543 int size;
544
545 size = dsparb & 0x1ff;
546 if (i9xx_plane == PLANE_B)
547 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
548 size >>= 1;
549
550 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
551 dsparb, plane_name(i9xx_plane), size);
552
553 return size;
554}
555
556static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
557 enum i9xx_plane_id i9xx_plane)
558{
559 u32 dsparb = I915_READ(DSPARB);
560 int size;
561
562 size = dsparb & 0x7f;
563 size >>= 2;
564
565 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
566 dsparb, plane_name(i9xx_plane), size);
567
568 return size;
569}
570
571
572static const struct intel_watermark_params pineview_display_wm = {
573 .fifo_size = PINEVIEW_DISPLAY_FIFO,
574 .max_wm = PINEVIEW_MAX_WM,
575 .default_wm = PINEVIEW_DFT_WM,
576 .guard_size = PINEVIEW_GUARD_WM,
577 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
578};
579static const struct intel_watermark_params pineview_display_hplloff_wm = {
580 .fifo_size = PINEVIEW_DISPLAY_FIFO,
581 .max_wm = PINEVIEW_MAX_WM,
582 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
583 .guard_size = PINEVIEW_GUARD_WM,
584 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
585};
586static const struct intel_watermark_params pineview_cursor_wm = {
587 .fifo_size = PINEVIEW_CURSOR_FIFO,
588 .max_wm = PINEVIEW_CURSOR_MAX_WM,
589 .default_wm = PINEVIEW_CURSOR_DFT_WM,
590 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
591 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
592};
593static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
594 .fifo_size = PINEVIEW_CURSOR_FIFO,
595 .max_wm = PINEVIEW_CURSOR_MAX_WM,
596 .default_wm = PINEVIEW_CURSOR_DFT_WM,
597 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
598 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
599};
600static const struct intel_watermark_params i965_cursor_wm_info = {
601 .fifo_size = I965_CURSOR_FIFO,
602 .max_wm = I965_CURSOR_MAX_WM,
603 .default_wm = I965_CURSOR_DFT_WM,
604 .guard_size = 2,
605 .cacheline_size = I915_FIFO_LINE_SIZE,
606};
607static const struct intel_watermark_params i945_wm_info = {
608 .fifo_size = I945_FIFO_SIZE,
609 .max_wm = I915_MAX_WM,
610 .default_wm = 1,
611 .guard_size = 2,
612 .cacheline_size = I915_FIFO_LINE_SIZE,
613};
614static const struct intel_watermark_params i915_wm_info = {
615 .fifo_size = I915_FIFO_SIZE,
616 .max_wm = I915_MAX_WM,
617 .default_wm = 1,
618 .guard_size = 2,
619 .cacheline_size = I915_FIFO_LINE_SIZE,
620};
621static const struct intel_watermark_params i830_a_wm_info = {
622 .fifo_size = I855GM_FIFO_SIZE,
623 .max_wm = I915_MAX_WM,
624 .default_wm = 1,
625 .guard_size = 2,
626 .cacheline_size = I830_FIFO_LINE_SIZE,
627};
628static const struct intel_watermark_params i830_bc_wm_info = {
629 .fifo_size = I855GM_FIFO_SIZE,
630 .max_wm = I915_MAX_WM/2,
631 .default_wm = 1,
632 .guard_size = 2,
633 .cacheline_size = I830_FIFO_LINE_SIZE,
634};
635static const struct intel_watermark_params i845_wm_info = {
636 .fifo_size = I830_FIFO_SIZE,
637 .max_wm = I915_MAX_WM,
638 .default_wm = 1,
639 .guard_size = 2,
640 .cacheline_size = I830_FIFO_LINE_SIZE,
641};
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676static unsigned int intel_wm_method1(unsigned int pixel_rate,
677 unsigned int cpp,
678 unsigned int latency)
679{
680 u64 ret;
681
682 ret = mul_u32_u32(pixel_rate, cpp * latency);
683 ret = DIV_ROUND_UP_ULL(ret, 10000);
684
685 return ret;
686}
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718static unsigned int intel_wm_method2(unsigned int pixel_rate,
719 unsigned int htotal,
720 unsigned int width,
721 unsigned int cpp,
722 unsigned int latency)
723{
724 unsigned int ret;
725
726
727
728
729
730 if (WARN_ON_ONCE(htotal == 0))
731 htotal = 1;
732
733 ret = (latency * pixel_rate) / (htotal * 10000);
734 ret = (ret + 1) * width * cpp;
735
736 return ret;
737}
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758static unsigned int intel_calculate_wm(int pixel_rate,
759 const struct intel_watermark_params *wm,
760 int fifo_size, int cpp,
761 unsigned int latency_ns)
762{
763 int entries, wm_size;
764
765
766
767
768
769
770
771 entries = intel_wm_method1(pixel_rate, cpp,
772 latency_ns / 100);
773 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
774 wm->guard_size;
775 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
776
777 wm_size = fifo_size - entries;
778 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
779
780
781 if (wm_size > wm->max_wm)
782 wm_size = wm->max_wm;
783 if (wm_size <= 0)
784 wm_size = wm->default_wm;
785
786
787
788
789
790
791
792
793 if (wm_size <= 8)
794 wm_size = 8;
795
796 return wm_size;
797}
798
799static bool is_disabling(int old, int new, int threshold)
800{
801 return old >= threshold && new < threshold;
802}
803
804static bool is_enabling(int old, int new, int threshold)
805{
806 return old < threshold && new >= threshold;
807}
808
809static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
810{
811 return dev_priv->wm.max_level + 1;
812}
813
814static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
815 const struct intel_plane_state *plane_state)
816{
817 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
818
819
820 if (!crtc_state->base.active)
821 return false;
822
823
824
825
826
827
828
829
830
831 if (plane->id == PLANE_CURSOR)
832 return plane_state->base.fb != NULL;
833 else
834 return plane_state->base.visible;
835}
836
837static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
838{
839 struct intel_crtc *crtc, *enabled = NULL;
840
841 for_each_intel_crtc(&dev_priv->drm, crtc) {
842 if (intel_crtc_active(crtc)) {
843 if (enabled)
844 return NULL;
845 enabled = crtc;
846 }
847 }
848
849 return enabled;
850}
851
852static void pineview_update_wm(struct intel_crtc *unused_crtc)
853{
854 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
855 struct intel_crtc *crtc;
856 const struct cxsr_latency *latency;
857 u32 reg;
858 unsigned int wm;
859
860 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
861 dev_priv->is_ddr3,
862 dev_priv->fsb_freq,
863 dev_priv->mem_freq);
864 if (!latency) {
865 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
866 intel_set_memory_cxsr(dev_priv, false);
867 return;
868 }
869
870 crtc = single_enabled_crtc(dev_priv);
871 if (crtc) {
872 const struct drm_display_mode *adjusted_mode =
873 &crtc->config->base.adjusted_mode;
874 const struct drm_framebuffer *fb =
875 crtc->base.primary->state->fb;
876 int cpp = fb->format->cpp[0];
877 int clock = adjusted_mode->crtc_clock;
878
879
880 wm = intel_calculate_wm(clock, &pineview_display_wm,
881 pineview_display_wm.fifo_size,
882 cpp, latency->display_sr);
883 reg = I915_READ(DSPFW1);
884 reg &= ~DSPFW_SR_MASK;
885 reg |= FW_WM(wm, SR);
886 I915_WRITE(DSPFW1, reg);
887 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
888
889
890 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
891 pineview_display_wm.fifo_size,
892 4, latency->cursor_sr);
893 reg = I915_READ(DSPFW3);
894 reg &= ~DSPFW_CURSOR_SR_MASK;
895 reg |= FW_WM(wm, CURSOR_SR);
896 I915_WRITE(DSPFW3, reg);
897
898
899 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
900 pineview_display_hplloff_wm.fifo_size,
901 cpp, latency->display_hpll_disable);
902 reg = I915_READ(DSPFW3);
903 reg &= ~DSPFW_HPLL_SR_MASK;
904 reg |= FW_WM(wm, HPLL_SR);
905 I915_WRITE(DSPFW3, reg);
906
907
908 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
909 pineview_display_hplloff_wm.fifo_size,
910 4, latency->cursor_hpll_disable);
911 reg = I915_READ(DSPFW3);
912 reg &= ~DSPFW_HPLL_CURSOR_MASK;
913 reg |= FW_WM(wm, HPLL_CURSOR);
914 I915_WRITE(DSPFW3, reg);
915 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
916
917 intel_set_memory_cxsr(dev_priv, true);
918 } else {
919 intel_set_memory_cxsr(dev_priv, false);
920 }
921}
922
923
924
925
926
927
928
929
930
931
932
933static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
934{
935 int tlb_miss = fifo_size * 64 - width * cpp * 8;
936
937 return max(0, tlb_miss);
938}
939
940static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
941 const struct g4x_wm_values *wm)
942{
943 enum pipe pipe;
944
945 for_each_pipe(dev_priv, pipe)
946 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
947
948 I915_WRITE(DSPFW1,
949 FW_WM(wm->sr.plane, SR) |
950 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
951 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
952 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
953 I915_WRITE(DSPFW2,
954 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
955 FW_WM(wm->sr.fbc, FBC_SR) |
956 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
957 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
958 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
959 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
960 I915_WRITE(DSPFW3,
961 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
962 FW_WM(wm->sr.cursor, CURSOR_SR) |
963 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
964 FW_WM(wm->hpll.plane, HPLL_SR));
965
966 POSTING_READ(DSPFW1);
967}
968
969#define FW_WM_VLV(value, plane) \
970 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
971
972static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
973 const struct vlv_wm_values *wm)
974{
975 enum pipe pipe;
976
977 for_each_pipe(dev_priv, pipe) {
978 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
979
980 I915_WRITE(VLV_DDL(pipe),
981 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
982 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
983 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
984 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
985 }
986
987
988
989
990
991
992 I915_WRITE(DSPHOWM, 0);
993 I915_WRITE(DSPHOWM1, 0);
994 I915_WRITE(DSPFW4, 0);
995 I915_WRITE(DSPFW5, 0);
996 I915_WRITE(DSPFW6, 0);
997
998 I915_WRITE(DSPFW1,
999 FW_WM(wm->sr.plane, SR) |
1000 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1001 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1002 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1003 I915_WRITE(DSPFW2,
1004 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1005 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1006 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1007 I915_WRITE(DSPFW3,
1008 FW_WM(wm->sr.cursor, CURSOR_SR));
1009
1010 if (IS_CHERRYVIEW(dev_priv)) {
1011 I915_WRITE(DSPFW7_CHV,
1012 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1013 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1014 I915_WRITE(DSPFW8_CHV,
1015 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1016 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1017 I915_WRITE(DSPFW9_CHV,
1018 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1019 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1020 I915_WRITE(DSPHOWM,
1021 FW_WM(wm->sr.plane >> 9, SR_HI) |
1022 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1023 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1024 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1025 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1026 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1027 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1028 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1029 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1030 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1031 } else {
1032 I915_WRITE(DSPFW7,
1033 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1034 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1035 I915_WRITE(DSPHOWM,
1036 FW_WM(wm->sr.plane >> 9, SR_HI) |
1037 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1038 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1039 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1040 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1041 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1042 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1043 }
1044
1045 POSTING_READ(DSPFW1);
1046}
1047
1048#undef FW_WM_VLV
1049
1050static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1051{
1052
1053 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1054 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1055 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1056
1057 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1058}
1059
1060static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1061{
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076 switch (plane_id) {
1077 case PLANE_CURSOR:
1078 return 63;
1079 case PLANE_PRIMARY:
1080 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1081 case PLANE_SPRITE0:
1082 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1083 default:
1084 MISSING_CASE(plane_id);
1085 return 0;
1086 }
1087}
1088
1089static int g4x_fbc_fifo_size(int level)
1090{
1091 switch (level) {
1092 case G4X_WM_LEVEL_SR:
1093 return 7;
1094 case G4X_WM_LEVEL_HPLL:
1095 return 15;
1096 default:
1097 MISSING_CASE(level);
1098 return 0;
1099 }
1100}
1101
1102static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1103 const struct intel_plane_state *plane_state,
1104 int level)
1105{
1106 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1107 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1108 const struct drm_display_mode *adjusted_mode =
1109 &crtc_state->base.adjusted_mode;
1110 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1111 unsigned int clock, htotal, cpp, width, wm;
1112
1113 if (latency == 0)
1114 return USHRT_MAX;
1115
1116 if (!intel_wm_plane_visible(crtc_state, plane_state))
1117 return 0;
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1131 level != G4X_WM_LEVEL_NORMAL)
1132 cpp = 4;
1133 else
1134 cpp = plane_state->base.fb->format->cpp[0];
1135
1136 clock = adjusted_mode->crtc_clock;
1137 htotal = adjusted_mode->crtc_htotal;
1138
1139 if (plane->id == PLANE_CURSOR)
1140 width = plane_state->base.crtc_w;
1141 else
1142 width = drm_rect_width(&plane_state->base.dst);
1143
1144 if (plane->id == PLANE_CURSOR) {
1145 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1146 } else if (plane->id == PLANE_PRIMARY &&
1147 level == G4X_WM_LEVEL_NORMAL) {
1148 wm = intel_wm_method1(clock, cpp, latency);
1149 } else {
1150 unsigned int small, large;
1151
1152 small = intel_wm_method1(clock, cpp, latency);
1153 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1154
1155 wm = min(small, large);
1156 }
1157
1158 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1159 width, cpp);
1160
1161 wm = DIV_ROUND_UP(wm, 64) + 2;
1162
1163 return min_t(unsigned int, wm, USHRT_MAX);
1164}
1165
1166static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1167 int level, enum plane_id plane_id, u16 value)
1168{
1169 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1170 bool dirty = false;
1171
1172 for (; level < intel_wm_num_levels(dev_priv); level++) {
1173 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1174
1175 dirty |= raw->plane[plane_id] != value;
1176 raw->plane[plane_id] = value;
1177 }
1178
1179 return dirty;
1180}
1181
1182static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1183 int level, u16 value)
1184{
1185 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1186 bool dirty = false;
1187
1188
1189 level = max(level, G4X_WM_LEVEL_SR);
1190
1191 for (; level < intel_wm_num_levels(dev_priv); level++) {
1192 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1193
1194 dirty |= raw->fbc != value;
1195 raw->fbc = value;
1196 }
1197
1198 return dirty;
1199}
1200
1201static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1202 const struct intel_plane_state *pstate,
1203 u32 pri_val);
1204
1205static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1206 const struct intel_plane_state *plane_state)
1207{
1208 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1209 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1210 enum plane_id plane_id = plane->id;
1211 bool dirty = false;
1212 int level;
1213
1214 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1215 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1216 if (plane_id == PLANE_PRIMARY)
1217 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1218 goto out;
1219 }
1220
1221 for (level = 0; level < num_levels; level++) {
1222 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1223 int wm, max_wm;
1224
1225 wm = g4x_compute_wm(crtc_state, plane_state, level);
1226 max_wm = g4x_plane_fifo_size(plane_id, level);
1227
1228 if (wm > max_wm)
1229 break;
1230
1231 dirty |= raw->plane[plane_id] != wm;
1232 raw->plane[plane_id] = wm;
1233
1234 if (plane_id != PLANE_PRIMARY ||
1235 level == G4X_WM_LEVEL_NORMAL)
1236 continue;
1237
1238 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1239 raw->plane[plane_id]);
1240 max_wm = g4x_fbc_fifo_size(level);
1241
1242
1243
1244
1245
1246 if (wm > max_wm)
1247 wm = USHRT_MAX;
1248
1249 dirty |= raw->fbc != wm;
1250 raw->fbc = wm;
1251 }
1252
1253
1254 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1255
1256 if (plane_id == PLANE_PRIMARY)
1257 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1258
1259 out:
1260 if (dirty) {
1261 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1262 plane->base.name,
1263 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1264 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1265 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1266
1267 if (plane_id == PLANE_PRIMARY)
1268 DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1269 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1270 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1271 }
1272
1273 return dirty;
1274}
1275
1276static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1277 enum plane_id plane_id, int level)
1278{
1279 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1280
1281 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1282}
1283
1284static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1285 int level)
1286{
1287 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1288
1289 if (level > dev_priv->wm.max_level)
1290 return false;
1291
1292 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1293 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1294 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1295}
1296
1297
1298static void g4x_invalidate_wms(struct intel_crtc *crtc,
1299 struct g4x_wm_state *wm_state, int level)
1300{
1301 if (level <= G4X_WM_LEVEL_NORMAL) {
1302 enum plane_id plane_id;
1303
1304 for_each_plane_id_on_crtc(crtc, plane_id)
1305 wm_state->wm.plane[plane_id] = USHRT_MAX;
1306 }
1307
1308 if (level <= G4X_WM_LEVEL_SR) {
1309 wm_state->cxsr = false;
1310 wm_state->sr.cursor = USHRT_MAX;
1311 wm_state->sr.plane = USHRT_MAX;
1312 wm_state->sr.fbc = USHRT_MAX;
1313 }
1314
1315 if (level <= G4X_WM_LEVEL_HPLL) {
1316 wm_state->hpll_en = false;
1317 wm_state->hpll.cursor = USHRT_MAX;
1318 wm_state->hpll.plane = USHRT_MAX;
1319 wm_state->hpll.fbc = USHRT_MAX;
1320 }
1321}
1322
1323static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1324{
1325 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1326 struct intel_atomic_state *state =
1327 to_intel_atomic_state(crtc_state->base.state);
1328 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1329 int num_active_planes = hweight32(crtc_state->active_planes &
1330 ~BIT(PLANE_CURSOR));
1331 const struct g4x_pipe_wm *raw;
1332 const struct intel_plane_state *old_plane_state;
1333 const struct intel_plane_state *new_plane_state;
1334 struct intel_plane *plane;
1335 enum plane_id plane_id;
1336 int i, level;
1337 unsigned int dirty = 0;
1338
1339 for_each_oldnew_intel_plane_in_state(state, plane,
1340 old_plane_state,
1341 new_plane_state, i) {
1342 if (new_plane_state->base.crtc != &crtc->base &&
1343 old_plane_state->base.crtc != &crtc->base)
1344 continue;
1345
1346 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1347 dirty |= BIT(plane->id);
1348 }
1349
1350 if (!dirty)
1351 return 0;
1352
1353 level = G4X_WM_LEVEL_NORMAL;
1354 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1355 goto out;
1356
1357 raw = &crtc_state->wm.g4x.raw[level];
1358 for_each_plane_id_on_crtc(crtc, plane_id)
1359 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1360
1361 level = G4X_WM_LEVEL_SR;
1362
1363 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1364 goto out;
1365
1366 raw = &crtc_state->wm.g4x.raw[level];
1367 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1368 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1369 wm_state->sr.fbc = raw->fbc;
1370
1371 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1372
1373 level = G4X_WM_LEVEL_HPLL;
1374
1375 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1376 goto out;
1377
1378 raw = &crtc_state->wm.g4x.raw[level];
1379 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1380 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1381 wm_state->hpll.fbc = raw->fbc;
1382
1383 wm_state->hpll_en = wm_state->cxsr;
1384
1385 level++;
1386
1387 out:
1388 if (level == G4X_WM_LEVEL_NORMAL)
1389 return -EINVAL;
1390
1391
1392 g4x_invalidate_wms(crtc, wm_state, level);
1393
1394
1395
1396
1397
1398
1399
1400 wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1401
1402 if (level >= G4X_WM_LEVEL_SR &&
1403 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1404 wm_state->fbc_en = false;
1405 else if (level >= G4X_WM_LEVEL_HPLL &&
1406 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1407 wm_state->fbc_en = false;
1408
1409 return 0;
1410}
1411
1412static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1413{
1414 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1415 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1416 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1417 struct intel_atomic_state *intel_state =
1418 to_intel_atomic_state(new_crtc_state->base.state);
1419 const struct intel_crtc_state *old_crtc_state =
1420 intel_atomic_get_old_crtc_state(intel_state, crtc);
1421 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1422 enum plane_id plane_id;
1423
1424 if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1425 *intermediate = *optimal;
1426
1427 intermediate->cxsr = false;
1428 intermediate->hpll_en = false;
1429 goto out;
1430 }
1431
1432 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1433 !new_crtc_state->disable_cxsr;
1434 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1435 !new_crtc_state->disable_cxsr;
1436 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1437
1438 for_each_plane_id_on_crtc(crtc, plane_id) {
1439 intermediate->wm.plane[plane_id] =
1440 max(optimal->wm.plane[plane_id],
1441 active->wm.plane[plane_id]);
1442
1443 WARN_ON(intermediate->wm.plane[plane_id] >
1444 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1445 }
1446
1447 intermediate->sr.plane = max(optimal->sr.plane,
1448 active->sr.plane);
1449 intermediate->sr.cursor = max(optimal->sr.cursor,
1450 active->sr.cursor);
1451 intermediate->sr.fbc = max(optimal->sr.fbc,
1452 active->sr.fbc);
1453
1454 intermediate->hpll.plane = max(optimal->hpll.plane,
1455 active->hpll.plane);
1456 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1457 active->hpll.cursor);
1458 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1459 active->hpll.fbc);
1460
1461 WARN_ON((intermediate->sr.plane >
1462 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1463 intermediate->sr.cursor >
1464 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1465 intermediate->cxsr);
1466 WARN_ON((intermediate->sr.plane >
1467 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1468 intermediate->sr.cursor >
1469 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1470 intermediate->hpll_en);
1471
1472 WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1473 intermediate->fbc_en && intermediate->cxsr);
1474 WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1475 intermediate->fbc_en && intermediate->hpll_en);
1476
1477out:
1478
1479
1480
1481
1482 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1483 new_crtc_state->wm.need_postvbl_update = true;
1484
1485 return 0;
1486}
1487
1488static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1489 struct g4x_wm_values *wm)
1490{
1491 struct intel_crtc *crtc;
1492 int num_active_crtcs = 0;
1493
1494 wm->cxsr = true;
1495 wm->hpll_en = true;
1496 wm->fbc_en = true;
1497
1498 for_each_intel_crtc(&dev_priv->drm, crtc) {
1499 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1500
1501 if (!crtc->active)
1502 continue;
1503
1504 if (!wm_state->cxsr)
1505 wm->cxsr = false;
1506 if (!wm_state->hpll_en)
1507 wm->hpll_en = false;
1508 if (!wm_state->fbc_en)
1509 wm->fbc_en = false;
1510
1511 num_active_crtcs++;
1512 }
1513
1514 if (num_active_crtcs != 1) {
1515 wm->cxsr = false;
1516 wm->hpll_en = false;
1517 wm->fbc_en = false;
1518 }
1519
1520 for_each_intel_crtc(&dev_priv->drm, crtc) {
1521 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1522 enum pipe pipe = crtc->pipe;
1523
1524 wm->pipe[pipe] = wm_state->wm;
1525 if (crtc->active && wm->cxsr)
1526 wm->sr = wm_state->sr;
1527 if (crtc->active && wm->hpll_en)
1528 wm->hpll = wm_state->hpll;
1529 }
1530}
1531
1532static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1533{
1534 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1535 struct g4x_wm_values new_wm = {};
1536
1537 g4x_merge_wm(dev_priv, &new_wm);
1538
1539 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1540 return;
1541
1542 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1543 _intel_set_memory_cxsr(dev_priv, false);
1544
1545 g4x_write_wm_values(dev_priv, &new_wm);
1546
1547 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1548 _intel_set_memory_cxsr(dev_priv, true);
1549
1550 *old_wm = new_wm;
1551}
1552
1553static void g4x_initial_watermarks(struct intel_atomic_state *state,
1554 struct intel_crtc_state *crtc_state)
1555{
1556 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1557 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1558
1559 mutex_lock(&dev_priv->wm.wm_mutex);
1560 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1561 g4x_program_watermarks(dev_priv);
1562 mutex_unlock(&dev_priv->wm.wm_mutex);
1563}
1564
1565static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1566 struct intel_crtc_state *crtc_state)
1567{
1568 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1569 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1570
1571 if (!crtc_state->wm.need_postvbl_update)
1572 return;
1573
1574 mutex_lock(&dev_priv->wm.wm_mutex);
1575 intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1576 g4x_program_watermarks(dev_priv);
1577 mutex_unlock(&dev_priv->wm.wm_mutex);
1578}
1579
1580
1581static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1582 unsigned int htotal,
1583 unsigned int width,
1584 unsigned int cpp,
1585 unsigned int latency)
1586{
1587 unsigned int ret;
1588
1589 ret = intel_wm_method2(pixel_rate, htotal,
1590 width, cpp, latency);
1591 ret = DIV_ROUND_UP(ret, 64);
1592
1593 return ret;
1594}
1595
1596static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1597{
1598
1599 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1600
1601 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1602
1603 if (IS_CHERRYVIEW(dev_priv)) {
1604 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1605 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1606
1607 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1608 }
1609}
1610
1611static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1612 const struct intel_plane_state *plane_state,
1613 int level)
1614{
1615 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1616 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1617 const struct drm_display_mode *adjusted_mode =
1618 &crtc_state->base.adjusted_mode;
1619 unsigned int clock, htotal, cpp, width, wm;
1620
1621 if (dev_priv->wm.pri_latency[level] == 0)
1622 return USHRT_MAX;
1623
1624 if (!intel_wm_plane_visible(crtc_state, plane_state))
1625 return 0;
1626
1627 cpp = plane_state->base.fb->format->cpp[0];
1628 clock = adjusted_mode->crtc_clock;
1629 htotal = adjusted_mode->crtc_htotal;
1630 width = crtc_state->pipe_src_w;
1631
1632 if (plane->id == PLANE_CURSOR) {
1633
1634
1635
1636
1637
1638
1639 wm = 63;
1640 } else {
1641 wm = vlv_wm_method2(clock, htotal, width, cpp,
1642 dev_priv->wm.pri_latency[level] * 10);
1643 }
1644
1645 return min_t(unsigned int, wm, USHRT_MAX);
1646}
1647
1648static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1649{
1650 return (active_planes & (BIT(PLANE_SPRITE0) |
1651 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1652}
1653
1654static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1655{
1656 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1657 const struct g4x_pipe_wm *raw =
1658 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1659 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1660 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1661 int num_active_planes = hweight32(active_planes);
1662 const int fifo_size = 511;
1663 int fifo_extra, fifo_left = fifo_size;
1664 int sprite0_fifo_extra = 0;
1665 unsigned int total_rate;
1666 enum plane_id plane_id;
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676 if (vlv_need_sprite0_fifo_workaround(active_planes))
1677 sprite0_fifo_extra = 1;
1678
1679 total_rate = raw->plane[PLANE_PRIMARY] +
1680 raw->plane[PLANE_SPRITE0] +
1681 raw->plane[PLANE_SPRITE1] +
1682 sprite0_fifo_extra;
1683
1684 if (total_rate > fifo_size)
1685 return -EINVAL;
1686
1687 if (total_rate == 0)
1688 total_rate = 1;
1689
1690 for_each_plane_id_on_crtc(crtc, plane_id) {
1691 unsigned int rate;
1692
1693 if ((active_planes & BIT(plane_id)) == 0) {
1694 fifo_state->plane[plane_id] = 0;
1695 continue;
1696 }
1697
1698 rate = raw->plane[plane_id];
1699 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1700 fifo_left -= fifo_state->plane[plane_id];
1701 }
1702
1703 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1704 fifo_left -= sprite0_fifo_extra;
1705
1706 fifo_state->plane[PLANE_CURSOR] = 63;
1707
1708 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1709
1710
1711 for_each_plane_id_on_crtc(crtc, plane_id) {
1712 int plane_extra;
1713
1714 if (fifo_left == 0)
1715 break;
1716
1717 if ((active_planes & BIT(plane_id)) == 0)
1718 continue;
1719
1720 plane_extra = min(fifo_extra, fifo_left);
1721 fifo_state->plane[plane_id] += plane_extra;
1722 fifo_left -= plane_extra;
1723 }
1724
1725 WARN_ON(active_planes != 0 && fifo_left != 0);
1726
1727
1728 if (active_planes == 0) {
1729 WARN_ON(fifo_left != fifo_size);
1730 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1731 }
1732
1733 return 0;
1734}
1735
1736
1737static void vlv_invalidate_wms(struct intel_crtc *crtc,
1738 struct vlv_wm_state *wm_state, int level)
1739{
1740 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1741
1742 for (; level < intel_wm_num_levels(dev_priv); level++) {
1743 enum plane_id plane_id;
1744
1745 for_each_plane_id_on_crtc(crtc, plane_id)
1746 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1747
1748 wm_state->sr[level].cursor = USHRT_MAX;
1749 wm_state->sr[level].plane = USHRT_MAX;
1750 }
1751}
1752
1753static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1754{
1755 if (wm > fifo_size)
1756 return USHRT_MAX;
1757 else
1758 return fifo_size - wm;
1759}
1760
1761
1762
1763
1764
1765static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1766 int level, enum plane_id plane_id, u16 value)
1767{
1768 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1769 int num_levels = intel_wm_num_levels(dev_priv);
1770 bool dirty = false;
1771
1772 for (; level < num_levels; level++) {
1773 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1774
1775 dirty |= raw->plane[plane_id] != value;
1776 raw->plane[plane_id] = value;
1777 }
1778
1779 return dirty;
1780}
1781
1782static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1783 const struct intel_plane_state *plane_state)
1784{
1785 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1786 enum plane_id plane_id = plane->id;
1787 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1788 int level;
1789 bool dirty = false;
1790
1791 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1792 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1793 goto out;
1794 }
1795
1796 for (level = 0; level < num_levels; level++) {
1797 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1798 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1799 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1800
1801 if (wm > max_wm)
1802 break;
1803
1804 dirty |= raw->plane[plane_id] != wm;
1805 raw->plane[plane_id] = wm;
1806 }
1807
1808
1809 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1810
1811out:
1812 if (dirty)
1813 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1814 plane->base.name,
1815 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1816 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1817 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1818
1819 return dirty;
1820}
1821
1822static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1823 enum plane_id plane_id, int level)
1824{
1825 const struct g4x_pipe_wm *raw =
1826 &crtc_state->wm.vlv.raw[level];
1827 const struct vlv_fifo_state *fifo_state =
1828 &crtc_state->wm.vlv.fifo_state;
1829
1830 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1831}
1832
1833static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1834{
1835 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1836 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1837 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1838 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1839}
1840
1841static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1842{
1843 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1844 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1845 struct intel_atomic_state *state =
1846 to_intel_atomic_state(crtc_state->base.state);
1847 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1848 const struct vlv_fifo_state *fifo_state =
1849 &crtc_state->wm.vlv.fifo_state;
1850 int num_active_planes = hweight32(crtc_state->active_planes &
1851 ~BIT(PLANE_CURSOR));
1852 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1853 const struct intel_plane_state *old_plane_state;
1854 const struct intel_plane_state *new_plane_state;
1855 struct intel_plane *plane;
1856 enum plane_id plane_id;
1857 int level, ret, i;
1858 unsigned int dirty = 0;
1859
1860 for_each_oldnew_intel_plane_in_state(state, plane,
1861 old_plane_state,
1862 new_plane_state, i) {
1863 if (new_plane_state->base.crtc != &crtc->base &&
1864 old_plane_state->base.crtc != &crtc->base)
1865 continue;
1866
1867 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1868 dirty |= BIT(plane->id);
1869 }
1870
1871
1872
1873
1874
1875
1876
1877 if (needs_modeset)
1878 crtc_state->fifo_changed = true;
1879
1880 if (!dirty)
1881 return 0;
1882
1883
1884 if (dirty & ~BIT(PLANE_CURSOR)) {
1885 const struct intel_crtc_state *old_crtc_state =
1886 intel_atomic_get_old_crtc_state(state, crtc);
1887 const struct vlv_fifo_state *old_fifo_state =
1888 &old_crtc_state->wm.vlv.fifo_state;
1889
1890 ret = vlv_compute_fifo(crtc_state);
1891 if (ret)
1892 return ret;
1893
1894 if (needs_modeset ||
1895 memcmp(old_fifo_state, fifo_state,
1896 sizeof(*fifo_state)) != 0)
1897 crtc_state->fifo_changed = true;
1898 }
1899
1900
1901 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1902
1903
1904
1905
1906
1907 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1908
1909 for (level = 0; level < wm_state->num_levels; level++) {
1910 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1911 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1912
1913 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1914 break;
1915
1916 for_each_plane_id_on_crtc(crtc, plane_id) {
1917 wm_state->wm[level].plane[plane_id] =
1918 vlv_invert_wm_value(raw->plane[plane_id],
1919 fifo_state->plane[plane_id]);
1920 }
1921
1922 wm_state->sr[level].plane =
1923 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1924 raw->plane[PLANE_SPRITE0],
1925 raw->plane[PLANE_SPRITE1]),
1926 sr_fifo_size);
1927
1928 wm_state->sr[level].cursor =
1929 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1930 63);
1931 }
1932
1933 if (level == 0)
1934 return -EINVAL;
1935
1936
1937 wm_state->num_levels = level;
1938
1939
1940 vlv_invalidate_wms(crtc, wm_state, level);
1941
1942 return 0;
1943}
1944
1945#define VLV_FIFO(plane, value) \
1946 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1947
1948static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1949 struct intel_crtc_state *crtc_state)
1950{
1951 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1952 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1953 struct intel_uncore *uncore = &dev_priv->uncore;
1954 const struct vlv_fifo_state *fifo_state =
1955 &crtc_state->wm.vlv.fifo_state;
1956 int sprite0_start, sprite1_start, fifo_size;
1957
1958 if (!crtc_state->fifo_changed)
1959 return;
1960
1961 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1962 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1963 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1964
1965 WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1966 WARN_ON(fifo_size != 511);
1967
1968 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979 spin_lock(&uncore->lock);
1980
1981 switch (crtc->pipe) {
1982 u32 dsparb, dsparb2, dsparb3;
1983 case PIPE_A:
1984 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1985 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1986
1987 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1988 VLV_FIFO(SPRITEB, 0xff));
1989 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1990 VLV_FIFO(SPRITEB, sprite1_start));
1991
1992 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1993 VLV_FIFO(SPRITEB_HI, 0x1));
1994 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1995 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1996
1997 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1998 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1999 break;
2000 case PIPE_B:
2001 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2002 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2003
2004 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2005 VLV_FIFO(SPRITED, 0xff));
2006 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2007 VLV_FIFO(SPRITED, sprite1_start));
2008
2009 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2010 VLV_FIFO(SPRITED_HI, 0xff));
2011 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2012 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2013
2014 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2015 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2016 break;
2017 case PIPE_C:
2018 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2019 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2020
2021 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2022 VLV_FIFO(SPRITEF, 0xff));
2023 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2024 VLV_FIFO(SPRITEF, sprite1_start));
2025
2026 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2027 VLV_FIFO(SPRITEF_HI, 0xff));
2028 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2029 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2030
2031 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2032 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2033 break;
2034 default:
2035 break;
2036 }
2037
2038 intel_uncore_posting_read_fw(uncore, DSPARB);
2039
2040 spin_unlock(&uncore->lock);
2041}
2042
2043#undef VLV_FIFO
2044
2045static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2046{
2047 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
2048 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2049 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2050 struct intel_atomic_state *intel_state =
2051 to_intel_atomic_state(new_crtc_state->base.state);
2052 const struct intel_crtc_state *old_crtc_state =
2053 intel_atomic_get_old_crtc_state(intel_state, crtc);
2054 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2055 int level;
2056
2057 if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2058 *intermediate = *optimal;
2059
2060 intermediate->cxsr = false;
2061 goto out;
2062 }
2063
2064 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2065 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2066 !new_crtc_state->disable_cxsr;
2067
2068 for (level = 0; level < intermediate->num_levels; level++) {
2069 enum plane_id plane_id;
2070
2071 for_each_plane_id_on_crtc(crtc, plane_id) {
2072 intermediate->wm[level].plane[plane_id] =
2073 min(optimal->wm[level].plane[plane_id],
2074 active->wm[level].plane[plane_id]);
2075 }
2076
2077 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2078 active->sr[level].plane);
2079 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2080 active->sr[level].cursor);
2081 }
2082
2083 vlv_invalidate_wms(crtc, intermediate, level);
2084
2085out:
2086
2087
2088
2089
2090 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2091 new_crtc_state->wm.need_postvbl_update = true;
2092
2093 return 0;
2094}
2095
2096static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2097 struct vlv_wm_values *wm)
2098{
2099 struct intel_crtc *crtc;
2100 int num_active_crtcs = 0;
2101
2102 wm->level = dev_priv->wm.max_level;
2103 wm->cxsr = true;
2104
2105 for_each_intel_crtc(&dev_priv->drm, crtc) {
2106 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2107
2108 if (!crtc->active)
2109 continue;
2110
2111 if (!wm_state->cxsr)
2112 wm->cxsr = false;
2113
2114 num_active_crtcs++;
2115 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2116 }
2117
2118 if (num_active_crtcs != 1)
2119 wm->cxsr = false;
2120
2121 if (num_active_crtcs > 1)
2122 wm->level = VLV_WM_LEVEL_PM2;
2123
2124 for_each_intel_crtc(&dev_priv->drm, crtc) {
2125 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2126 enum pipe pipe = crtc->pipe;
2127
2128 wm->pipe[pipe] = wm_state->wm[wm->level];
2129 if (crtc->active && wm->cxsr)
2130 wm->sr = wm_state->sr[wm->level];
2131
2132 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2133 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2134 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2135 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2136 }
2137}
2138
2139static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2140{
2141 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2142 struct vlv_wm_values new_wm = {};
2143
2144 vlv_merge_wm(dev_priv, &new_wm);
2145
2146 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2147 return;
2148
2149 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2150 chv_set_memory_dvfs(dev_priv, false);
2151
2152 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2153 chv_set_memory_pm5(dev_priv, false);
2154
2155 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2156 _intel_set_memory_cxsr(dev_priv, false);
2157
2158 vlv_write_wm_values(dev_priv, &new_wm);
2159
2160 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2161 _intel_set_memory_cxsr(dev_priv, true);
2162
2163 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2164 chv_set_memory_pm5(dev_priv, true);
2165
2166 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2167 chv_set_memory_dvfs(dev_priv, true);
2168
2169 *old_wm = new_wm;
2170}
2171
2172static void vlv_initial_watermarks(struct intel_atomic_state *state,
2173 struct intel_crtc_state *crtc_state)
2174{
2175 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2176 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2177
2178 mutex_lock(&dev_priv->wm.wm_mutex);
2179 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2180 vlv_program_watermarks(dev_priv);
2181 mutex_unlock(&dev_priv->wm.wm_mutex);
2182}
2183
2184static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2185 struct intel_crtc_state *crtc_state)
2186{
2187 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2188 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2189
2190 if (!crtc_state->wm.need_postvbl_update)
2191 return;
2192
2193 mutex_lock(&dev_priv->wm.wm_mutex);
2194 intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2195 vlv_program_watermarks(dev_priv);
2196 mutex_unlock(&dev_priv->wm.wm_mutex);
2197}
2198
2199static void i965_update_wm(struct intel_crtc *unused_crtc)
2200{
2201 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2202 struct intel_crtc *crtc;
2203 int srwm = 1;
2204 int cursor_sr = 16;
2205 bool cxsr_enabled;
2206
2207
2208 crtc = single_enabled_crtc(dev_priv);
2209 if (crtc) {
2210
2211 static const int sr_latency_ns = 12000;
2212 const struct drm_display_mode *adjusted_mode =
2213 &crtc->config->base.adjusted_mode;
2214 const struct drm_framebuffer *fb =
2215 crtc->base.primary->state->fb;
2216 int clock = adjusted_mode->crtc_clock;
2217 int htotal = adjusted_mode->crtc_htotal;
2218 int hdisplay = crtc->config->pipe_src_w;
2219 int cpp = fb->format->cpp[0];
2220 int entries;
2221
2222 entries = intel_wm_method2(clock, htotal,
2223 hdisplay, cpp, sr_latency_ns / 100);
2224 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2225 srwm = I965_FIFO_SIZE - entries;
2226 if (srwm < 0)
2227 srwm = 1;
2228 srwm &= 0x1ff;
2229 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2230 entries, srwm);
2231
2232 entries = intel_wm_method2(clock, htotal,
2233 crtc->base.cursor->state->crtc_w, 4,
2234 sr_latency_ns / 100);
2235 entries = DIV_ROUND_UP(entries,
2236 i965_cursor_wm_info.cacheline_size) +
2237 i965_cursor_wm_info.guard_size;
2238
2239 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2240 if (cursor_sr > i965_cursor_wm_info.max_wm)
2241 cursor_sr = i965_cursor_wm_info.max_wm;
2242
2243 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2244 "cursor %d\n", srwm, cursor_sr);
2245
2246 cxsr_enabled = true;
2247 } else {
2248 cxsr_enabled = false;
2249
2250 intel_set_memory_cxsr(dev_priv, false);
2251 }
2252
2253 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2254 srwm);
2255
2256
2257 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2258 FW_WM(8, CURSORB) |
2259 FW_WM(8, PLANEB) |
2260 FW_WM(8, PLANEA));
2261 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2262 FW_WM(8, PLANEC_OLD));
2263
2264 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2265
2266 if (cxsr_enabled)
2267 intel_set_memory_cxsr(dev_priv, true);
2268}
2269
2270#undef FW_WM
2271
2272static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2273{
2274 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2275 const struct intel_watermark_params *wm_info;
2276 u32 fwater_lo;
2277 u32 fwater_hi;
2278 int cwm, srwm = 1;
2279 int fifo_size;
2280 int planea_wm, planeb_wm;
2281 struct intel_crtc *crtc, *enabled = NULL;
2282
2283 if (IS_I945GM(dev_priv))
2284 wm_info = &i945_wm_info;
2285 else if (!IS_GEN(dev_priv, 2))
2286 wm_info = &i915_wm_info;
2287 else
2288 wm_info = &i830_a_wm_info;
2289
2290 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2291 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2292 if (intel_crtc_active(crtc)) {
2293 const struct drm_display_mode *adjusted_mode =
2294 &crtc->config->base.adjusted_mode;
2295 const struct drm_framebuffer *fb =
2296 crtc->base.primary->state->fb;
2297 int cpp;
2298
2299 if (IS_GEN(dev_priv, 2))
2300 cpp = 4;
2301 else
2302 cpp = fb->format->cpp[0];
2303
2304 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2305 wm_info, fifo_size, cpp,
2306 pessimal_latency_ns);
2307 enabled = crtc;
2308 } else {
2309 planea_wm = fifo_size - wm_info->guard_size;
2310 if (planea_wm > (long)wm_info->max_wm)
2311 planea_wm = wm_info->max_wm;
2312 }
2313
2314 if (IS_GEN(dev_priv, 2))
2315 wm_info = &i830_bc_wm_info;
2316
2317 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2318 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2319 if (intel_crtc_active(crtc)) {
2320 const struct drm_display_mode *adjusted_mode =
2321 &crtc->config->base.adjusted_mode;
2322 const struct drm_framebuffer *fb =
2323 crtc->base.primary->state->fb;
2324 int cpp;
2325
2326 if (IS_GEN(dev_priv, 2))
2327 cpp = 4;
2328 else
2329 cpp = fb->format->cpp[0];
2330
2331 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2332 wm_info, fifo_size, cpp,
2333 pessimal_latency_ns);
2334 if (enabled == NULL)
2335 enabled = crtc;
2336 else
2337 enabled = NULL;
2338 } else {
2339 planeb_wm = fifo_size - wm_info->guard_size;
2340 if (planeb_wm > (long)wm_info->max_wm)
2341 planeb_wm = wm_info->max_wm;
2342 }
2343
2344 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2345
2346 if (IS_I915GM(dev_priv) && enabled) {
2347 struct drm_i915_gem_object *obj;
2348
2349 obj = intel_fb_obj(enabled->base.primary->state->fb);
2350
2351
2352 if (!i915_gem_object_is_tiled(obj))
2353 enabled = NULL;
2354 }
2355
2356
2357
2358
2359 cwm = 2;
2360
2361
2362 intel_set_memory_cxsr(dev_priv, false);
2363
2364
2365 if (HAS_FW_BLC(dev_priv) && enabled) {
2366
2367 static const int sr_latency_ns = 6000;
2368 const struct drm_display_mode *adjusted_mode =
2369 &enabled->config->base.adjusted_mode;
2370 const struct drm_framebuffer *fb =
2371 enabled->base.primary->state->fb;
2372 int clock = adjusted_mode->crtc_clock;
2373 int htotal = adjusted_mode->crtc_htotal;
2374 int hdisplay = enabled->config->pipe_src_w;
2375 int cpp;
2376 int entries;
2377
2378 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2379 cpp = 4;
2380 else
2381 cpp = fb->format->cpp[0];
2382
2383 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2384 sr_latency_ns / 100);
2385 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2386 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2387 srwm = wm_info->fifo_size - entries;
2388 if (srwm < 0)
2389 srwm = 1;
2390
2391 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2392 I915_WRITE(FW_BLC_SELF,
2393 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2394 else
2395 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2396 }
2397
2398 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2399 planea_wm, planeb_wm, cwm, srwm);
2400
2401 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2402 fwater_hi = (cwm & 0x1f);
2403
2404
2405 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2406 fwater_hi = fwater_hi | (1 << 8);
2407
2408 I915_WRITE(FW_BLC, fwater_lo);
2409 I915_WRITE(FW_BLC2, fwater_hi);
2410
2411 if (enabled)
2412 intel_set_memory_cxsr(dev_priv, true);
2413}
2414
2415static void i845_update_wm(struct intel_crtc *unused_crtc)
2416{
2417 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2418 struct intel_crtc *crtc;
2419 const struct drm_display_mode *adjusted_mode;
2420 u32 fwater_lo;
2421 int planea_wm;
2422
2423 crtc = single_enabled_crtc(dev_priv);
2424 if (crtc == NULL)
2425 return;
2426
2427 adjusted_mode = &crtc->config->base.adjusted_mode;
2428 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2429 &i845_wm_info,
2430 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2431 4, pessimal_latency_ns);
2432 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2433 fwater_lo |= (3<<8) | planea_wm;
2434
2435 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2436
2437 I915_WRITE(FW_BLC, fwater_lo);
2438}
2439
2440
2441static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2442 unsigned int cpp,
2443 unsigned int latency)
2444{
2445 unsigned int ret;
2446
2447 ret = intel_wm_method1(pixel_rate, cpp, latency);
2448 ret = DIV_ROUND_UP(ret, 64) + 2;
2449
2450 return ret;
2451}
2452
2453
2454static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2455 unsigned int htotal,
2456 unsigned int width,
2457 unsigned int cpp,
2458 unsigned int latency)
2459{
2460 unsigned int ret;
2461
2462 ret = intel_wm_method2(pixel_rate, htotal,
2463 width, cpp, latency);
2464 ret = DIV_ROUND_UP(ret, 64) + 2;
2465
2466 return ret;
2467}
2468
2469static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2470{
2471
2472
2473
2474
2475
2476
2477 if (WARN_ON(!cpp))
2478 return 0;
2479 if (WARN_ON(!horiz_pixels))
2480 return 0;
2481
2482 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2483}
2484
2485struct ilk_wm_maximums {
2486 u16 pri;
2487 u16 spr;
2488 u16 cur;
2489 u16 fbc;
2490};
2491
2492
2493
2494
2495
2496static u32 ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
2497 const struct intel_plane_state *pstate,
2498 u32 mem_value, bool is_lp)
2499{
2500 u32 method1, method2;
2501 int cpp;
2502
2503 if (mem_value == 0)
2504 return U32_MAX;
2505
2506 if (!intel_wm_plane_visible(cstate, pstate))
2507 return 0;
2508
2509 cpp = pstate->base.fb->format->cpp[0];
2510
2511 method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2512
2513 if (!is_lp)
2514 return method1;
2515
2516 method2 = ilk_wm_method2(cstate->pixel_rate,
2517 cstate->base.adjusted_mode.crtc_htotal,
2518 drm_rect_width(&pstate->base.dst),
2519 cpp, mem_value);
2520
2521 return min(method1, method2);
2522}
2523
2524
2525
2526
2527
2528static u32 ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2529 const struct intel_plane_state *pstate,
2530 u32 mem_value)
2531{
2532 u32 method1, method2;
2533 int cpp;
2534
2535 if (mem_value == 0)
2536 return U32_MAX;
2537
2538 if (!intel_wm_plane_visible(cstate, pstate))
2539 return 0;
2540
2541 cpp = pstate->base.fb->format->cpp[0];
2542
2543 method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2544 method2 = ilk_wm_method2(cstate->pixel_rate,
2545 cstate->base.adjusted_mode.crtc_htotal,
2546 drm_rect_width(&pstate->base.dst),
2547 cpp, mem_value);
2548 return min(method1, method2);
2549}
2550
2551
2552
2553
2554
2555static u32 ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2556 const struct intel_plane_state *pstate,
2557 u32 mem_value)
2558{
2559 int cpp;
2560
2561 if (mem_value == 0)
2562 return U32_MAX;
2563
2564 if (!intel_wm_plane_visible(cstate, pstate))
2565 return 0;
2566
2567 cpp = pstate->base.fb->format->cpp[0];
2568
2569 return ilk_wm_method2(cstate->pixel_rate,
2570 cstate->base.adjusted_mode.crtc_htotal,
2571 pstate->base.crtc_w, cpp, mem_value);
2572}
2573
2574
2575static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2576 const struct intel_plane_state *pstate,
2577 u32 pri_val)
2578{
2579 int cpp;
2580
2581 if (!intel_wm_plane_visible(cstate, pstate))
2582 return 0;
2583
2584 cpp = pstate->base.fb->format->cpp[0];
2585
2586 return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2587}
2588
2589static unsigned int
2590ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2591{
2592 if (INTEL_GEN(dev_priv) >= 8)
2593 return 3072;
2594 else if (INTEL_GEN(dev_priv) >= 7)
2595 return 768;
2596 else
2597 return 512;
2598}
2599
2600static unsigned int
2601ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2602 int level, bool is_sprite)
2603{
2604 if (INTEL_GEN(dev_priv) >= 8)
2605
2606 return level == 0 ? 255 : 2047;
2607 else if (INTEL_GEN(dev_priv) >= 7)
2608
2609 return level == 0 ? 127 : 1023;
2610 else if (!is_sprite)
2611
2612 return level == 0 ? 127 : 511;
2613 else
2614
2615 return level == 0 ? 63 : 255;
2616}
2617
2618static unsigned int
2619ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2620{
2621 if (INTEL_GEN(dev_priv) >= 7)
2622 return level == 0 ? 63 : 255;
2623 else
2624 return level == 0 ? 31 : 63;
2625}
2626
2627static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2628{
2629 if (INTEL_GEN(dev_priv) >= 8)
2630 return 31;
2631 else
2632 return 15;
2633}
2634
2635
2636static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2637 int level,
2638 const struct intel_wm_config *config,
2639 enum intel_ddb_partitioning ddb_partitioning,
2640 bool is_sprite)
2641{
2642 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2643
2644
2645 if (is_sprite && !config->sprites_enabled)
2646 return 0;
2647
2648
2649 if (level == 0 || config->num_pipes_active > 1) {
2650 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2651
2652
2653
2654
2655
2656
2657 if (INTEL_GEN(dev_priv) <= 6)
2658 fifo_size /= 2;
2659 }
2660
2661 if (config->sprites_enabled) {
2662
2663 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2664 if (is_sprite)
2665 fifo_size *= 5;
2666 fifo_size /= 6;
2667 } else {
2668 fifo_size /= 2;
2669 }
2670 }
2671
2672
2673 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2674}
2675
2676
2677static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2678 int level,
2679 const struct intel_wm_config *config)
2680{
2681
2682 if (level > 0 && config->num_pipes_active > 1)
2683 return 64;
2684
2685
2686 return ilk_cursor_wm_reg_max(dev_priv, level);
2687}
2688
2689static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2690 int level,
2691 const struct intel_wm_config *config,
2692 enum intel_ddb_partitioning ddb_partitioning,
2693 struct ilk_wm_maximums *max)
2694{
2695 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2696 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2697 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2698 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2699}
2700
2701static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2702 int level,
2703 struct ilk_wm_maximums *max)
2704{
2705 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2706 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2707 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2708 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2709}
2710
2711static bool ilk_validate_wm_level(int level,
2712 const struct ilk_wm_maximums *max,
2713 struct intel_wm_level *result)
2714{
2715 bool ret;
2716
2717
2718 if (!result->enable)
2719 return false;
2720
2721 result->enable = result->pri_val <= max->pri &&
2722 result->spr_val <= max->spr &&
2723 result->cur_val <= max->cur;
2724
2725 ret = result->enable;
2726
2727
2728
2729
2730
2731
2732 if (level == 0 && !result->enable) {
2733 if (result->pri_val > max->pri)
2734 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2735 level, result->pri_val, max->pri);
2736 if (result->spr_val > max->spr)
2737 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2738 level, result->spr_val, max->spr);
2739 if (result->cur_val > max->cur)
2740 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2741 level, result->cur_val, max->cur);
2742
2743 result->pri_val = min_t(u32, result->pri_val, max->pri);
2744 result->spr_val = min_t(u32, result->spr_val, max->spr);
2745 result->cur_val = min_t(u32, result->cur_val, max->cur);
2746 result->enable = true;
2747 }
2748
2749 return ret;
2750}
2751
2752static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2753 const struct intel_crtc *intel_crtc,
2754 int level,
2755 struct intel_crtc_state *cstate,
2756 const struct intel_plane_state *pristate,
2757 const struct intel_plane_state *sprstate,
2758 const struct intel_plane_state *curstate,
2759 struct intel_wm_level *result)
2760{
2761 u16 pri_latency = dev_priv->wm.pri_latency[level];
2762 u16 spr_latency = dev_priv->wm.spr_latency[level];
2763 u16 cur_latency = dev_priv->wm.cur_latency[level];
2764
2765
2766 if (level > 0) {
2767 pri_latency *= 5;
2768 spr_latency *= 5;
2769 cur_latency *= 5;
2770 }
2771
2772 if (pristate) {
2773 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2774 pri_latency, level);
2775 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2776 }
2777
2778 if (sprstate)
2779 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2780
2781 if (curstate)
2782 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2783
2784 result->enable = true;
2785}
2786
2787static u32
2788hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2789{
2790 const struct intel_atomic_state *intel_state =
2791 to_intel_atomic_state(cstate->base.state);
2792 const struct drm_display_mode *adjusted_mode =
2793 &cstate->base.adjusted_mode;
2794 u32 linetime, ips_linetime;
2795
2796 if (!cstate->base.active)
2797 return 0;
2798 if (WARN_ON(adjusted_mode->crtc_clock == 0))
2799 return 0;
2800 if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2801 return 0;
2802
2803
2804
2805
2806 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2807 adjusted_mode->crtc_clock);
2808 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2809 intel_state->cdclk.logical.cdclk);
2810
2811 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2812 PIPE_WM_LINETIME_TIME(linetime);
2813}
2814
2815static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2816 u16 wm[8])
2817{
2818 struct intel_uncore *uncore = &dev_priv->uncore;
2819
2820 if (INTEL_GEN(dev_priv) >= 9) {
2821 u32 val;
2822 int ret, i;
2823 int level, max_level = ilk_wm_max_level(dev_priv);
2824
2825
2826 val = 0;
2827 ret = sandybridge_pcode_read(dev_priv,
2828 GEN9_PCODE_READ_MEM_LATENCY,
2829 &val, NULL);
2830
2831 if (ret) {
2832 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2833 return;
2834 }
2835
2836 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2837 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2838 GEN9_MEM_LATENCY_LEVEL_MASK;
2839 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2840 GEN9_MEM_LATENCY_LEVEL_MASK;
2841 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2842 GEN9_MEM_LATENCY_LEVEL_MASK;
2843
2844
2845 val = 1;
2846 ret = sandybridge_pcode_read(dev_priv,
2847 GEN9_PCODE_READ_MEM_LATENCY,
2848 &val, NULL);
2849 if (ret) {
2850 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2851 return;
2852 }
2853
2854 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2855 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2856 GEN9_MEM_LATENCY_LEVEL_MASK;
2857 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2858 GEN9_MEM_LATENCY_LEVEL_MASK;
2859 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2860 GEN9_MEM_LATENCY_LEVEL_MASK;
2861
2862
2863
2864
2865
2866
2867 for (level = 1; level <= max_level; level++) {
2868 if (wm[level] == 0) {
2869 for (i = level + 1; i <= max_level; i++)
2870 wm[i] = 0;
2871 break;
2872 }
2873 }
2874
2875
2876
2877
2878
2879
2880
2881
2882 if (wm[0] == 0) {
2883 wm[0] += 2;
2884 for (level = 1; level <= max_level; level++) {
2885 if (wm[level] == 0)
2886 break;
2887 wm[level] += 2;
2888 }
2889 }
2890
2891
2892
2893
2894
2895
2896
2897 if (dev_priv->dram_info.is_16gb_dimm)
2898 wm[0] += 1;
2899
2900 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2901 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2902
2903 wm[0] = (sskpd >> 56) & 0xFF;
2904 if (wm[0] == 0)
2905 wm[0] = sskpd & 0xF;
2906 wm[1] = (sskpd >> 4) & 0xFF;
2907 wm[2] = (sskpd >> 12) & 0xFF;
2908 wm[3] = (sskpd >> 20) & 0x1FF;
2909 wm[4] = (sskpd >> 32) & 0x1FF;
2910 } else if (INTEL_GEN(dev_priv) >= 6) {
2911 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2912
2913 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2914 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2915 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2916 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2917 } else if (INTEL_GEN(dev_priv) >= 5) {
2918 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2919
2920
2921 wm[0] = 7;
2922 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2923 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2924 } else {
2925 MISSING_CASE(INTEL_DEVID(dev_priv));
2926 }
2927}
2928
2929static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2930 u16 wm[5])
2931{
2932
2933 if (IS_GEN(dev_priv, 5))
2934 wm[0] = 13;
2935}
2936
2937static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2938 u16 wm[5])
2939{
2940
2941 if (IS_GEN(dev_priv, 5))
2942 wm[0] = 13;
2943}
2944
2945int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2946{
2947
2948 if (INTEL_GEN(dev_priv) >= 9)
2949 return 7;
2950 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2951 return 4;
2952 else if (INTEL_GEN(dev_priv) >= 6)
2953 return 3;
2954 else
2955 return 2;
2956}
2957
2958static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2959 const char *name,
2960 const u16 wm[8])
2961{
2962 int level, max_level = ilk_wm_max_level(dev_priv);
2963
2964 for (level = 0; level <= max_level; level++) {
2965 unsigned int latency = wm[level];
2966
2967 if (latency == 0) {
2968 DRM_DEBUG_KMS("%s WM%d latency not provided\n",
2969 name, level);
2970 continue;
2971 }
2972
2973
2974
2975
2976
2977 if (INTEL_GEN(dev_priv) >= 9)
2978 latency *= 10;
2979 else if (level > 0)
2980 latency *= 5;
2981
2982 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2983 name, level, wm[level],
2984 latency / 10, latency % 10);
2985 }
2986}
2987
2988static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2989 u16 wm[5], u16 min)
2990{
2991 int level, max_level = ilk_wm_max_level(dev_priv);
2992
2993 if (wm[0] >= min)
2994 return false;
2995
2996 wm[0] = max(wm[0], min);
2997 for (level = 1; level <= max_level; level++)
2998 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2999
3000 return true;
3001}
3002
3003static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3004{
3005 bool changed;
3006
3007
3008
3009
3010
3011 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3012 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3013 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3014
3015 if (!changed)
3016 return;
3017
3018 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
3019 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3020 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3021 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3022}
3023
3024static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3025{
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037 if (dev_priv->wm.pri_latency[3] == 0 &&
3038 dev_priv->wm.spr_latency[3] == 0 &&
3039 dev_priv->wm.cur_latency[3] == 0)
3040 return;
3041
3042 dev_priv->wm.pri_latency[3] = 0;
3043 dev_priv->wm.spr_latency[3] = 0;
3044 dev_priv->wm.cur_latency[3] = 0;
3045
3046 DRM_DEBUG_KMS("LP3 watermarks disabled due to potential for lost interrupts\n");
3047 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3048 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3049 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3050}
3051
3052static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3053{
3054 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3055
3056 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3057 sizeof(dev_priv->wm.pri_latency));
3058 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3059 sizeof(dev_priv->wm.pri_latency));
3060
3061 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3062 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3063
3064 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3065 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3066 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3067
3068 if (IS_GEN(dev_priv, 6)) {
3069 snb_wm_latency_quirk(dev_priv);
3070 snb_wm_lp3_irq_quirk(dev_priv);
3071 }
3072}
3073
3074static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3075{
3076 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3077 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3078}
3079
3080static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3081 struct intel_pipe_wm *pipe_wm)
3082{
3083
3084 const struct intel_wm_config config = {
3085 .num_pipes_active = 1,
3086 .sprites_enabled = pipe_wm->sprites_enabled,
3087 .sprites_scaled = pipe_wm->sprites_scaled,
3088 };
3089 struct ilk_wm_maximums max;
3090
3091
3092 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3093
3094
3095 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3096 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3097 return false;
3098 }
3099
3100 return true;
3101}
3102
3103
3104static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
3105{
3106 struct drm_atomic_state *state = cstate->base.state;
3107 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3108 struct intel_pipe_wm *pipe_wm;
3109 struct drm_device *dev = state->dev;
3110 const struct drm_i915_private *dev_priv = to_i915(dev);
3111 struct drm_plane *plane;
3112 const struct drm_plane_state *plane_state;
3113 const struct intel_plane_state *pristate = NULL;
3114 const struct intel_plane_state *sprstate = NULL;
3115 const struct intel_plane_state *curstate = NULL;
3116 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3117 struct ilk_wm_maximums max;
3118
3119 pipe_wm = &cstate->wm.ilk.optimal;
3120
3121 drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &cstate->base) {
3122 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3123
3124 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3125 pristate = ps;
3126 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3127 sprstate = ps;
3128 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3129 curstate = ps;
3130 }
3131
3132 pipe_wm->pipe_enabled = cstate->base.active;
3133 if (sprstate) {
3134 pipe_wm->sprites_enabled = sprstate->base.visible;
3135 pipe_wm->sprites_scaled = sprstate->base.visible &&
3136 (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3137 drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3138 }
3139
3140 usable_level = max_level;
3141
3142
3143 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3144 usable_level = 1;
3145
3146
3147 if (pipe_wm->sprites_scaled)
3148 usable_level = 0;
3149
3150 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3151 ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
3152 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3153
3154 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3155 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
3156
3157 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3158 return -EINVAL;
3159
3160 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3161
3162 for (level = 1; level <= usable_level; level++) {
3163 struct intel_wm_level *wm = &pipe_wm->wm[level];
3164
3165 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
3166 pristate, sprstate, curstate, wm);
3167
3168
3169
3170
3171
3172
3173 if (!ilk_validate_wm_level(level, &max, wm)) {
3174 memset(wm, 0, sizeof(*wm));
3175 break;
3176 }
3177 }
3178
3179 return 0;
3180}
3181
3182
3183
3184
3185
3186
3187static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3188{
3189 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->base.crtc);
3190 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3191 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3192 struct intel_atomic_state *intel_state =
3193 to_intel_atomic_state(newstate->base.state);
3194 const struct intel_crtc_state *oldstate =
3195 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3196 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3197 int level, max_level = ilk_wm_max_level(dev_priv);
3198
3199
3200
3201
3202
3203
3204 *a = newstate->wm.ilk.optimal;
3205 if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base) ||
3206 intel_state->skip_intermediate_wm)
3207 return 0;
3208
3209 a->pipe_enabled |= b->pipe_enabled;
3210 a->sprites_enabled |= b->sprites_enabled;
3211 a->sprites_scaled |= b->sprites_scaled;
3212
3213 for (level = 0; level <= max_level; level++) {
3214 struct intel_wm_level *a_wm = &a->wm[level];
3215 const struct intel_wm_level *b_wm = &b->wm[level];
3216
3217 a_wm->enable &= b_wm->enable;
3218 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3219 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3220 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3221 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3222 }
3223
3224
3225
3226
3227
3228
3229
3230 if (!ilk_validate_pipe_wm(dev_priv, a))
3231 return -EINVAL;
3232
3233
3234
3235
3236
3237 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3238 newstate->wm.need_postvbl_update = true;
3239
3240 return 0;
3241}
3242
3243
3244
3245
3246static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3247 int level,
3248 struct intel_wm_level *ret_wm)
3249{
3250 const struct intel_crtc *intel_crtc;
3251
3252 ret_wm->enable = true;
3253
3254 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3255 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3256 const struct intel_wm_level *wm = &active->wm[level];
3257
3258 if (!active->pipe_enabled)
3259 continue;
3260
3261
3262
3263
3264
3265
3266 if (!wm->enable)
3267 ret_wm->enable = false;
3268
3269 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3270 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3271 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3272 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3273 }
3274}
3275
3276
3277
3278
3279static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3280 const struct intel_wm_config *config,
3281 const struct ilk_wm_maximums *max,
3282 struct intel_pipe_wm *merged)
3283{
3284 int level, max_level = ilk_wm_max_level(dev_priv);
3285 int last_enabled_level = max_level;
3286
3287
3288 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3289 config->num_pipes_active > 1)
3290 last_enabled_level = 0;
3291
3292
3293 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3294
3295
3296 for (level = 1; level <= max_level; level++) {
3297 struct intel_wm_level *wm = &merged->wm[level];
3298
3299 ilk_merge_wm_level(dev_priv, level, wm);
3300
3301 if (level > last_enabled_level)
3302 wm->enable = false;
3303 else if (!ilk_validate_wm_level(level, max, wm))
3304
3305 last_enabled_level = level - 1;
3306
3307
3308
3309
3310
3311 if (wm->fbc_val > max->fbc) {
3312 if (wm->enable)
3313 merged->fbc_wm_enabled = false;
3314 wm->fbc_val = 0;
3315 }
3316 }
3317
3318
3319
3320
3321
3322
3323
3324 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3325 intel_fbc_is_active(dev_priv)) {
3326 for (level = 2; level <= max_level; level++) {
3327 struct intel_wm_level *wm = &merged->wm[level];
3328
3329 wm->enable = false;
3330 }
3331 }
3332}
3333
3334static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3335{
3336
3337 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3338}
3339
3340
3341static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3342 int level)
3343{
3344 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3345 return 2 * level;
3346 else
3347 return dev_priv->wm.pri_latency[level];
3348}
3349
3350static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3351 const struct intel_pipe_wm *merged,
3352 enum intel_ddb_partitioning partitioning,
3353 struct ilk_wm_values *results)
3354{
3355 struct intel_crtc *intel_crtc;
3356 int level, wm_lp;
3357
3358 results->enable_fbc_wm = merged->fbc_wm_enabled;
3359 results->partitioning = partitioning;
3360
3361
3362 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3363 const struct intel_wm_level *r;
3364
3365 level = ilk_wm_lp_to_level(wm_lp, merged);
3366
3367 r = &merged->wm[level];
3368
3369
3370
3371
3372
3373 results->wm_lp[wm_lp - 1] =
3374 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3375 (r->pri_val << WM1_LP_SR_SHIFT) |
3376 r->cur_val;
3377
3378 if (r->enable)
3379 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3380
3381 if (INTEL_GEN(dev_priv) >= 8)
3382 results->wm_lp[wm_lp - 1] |=
3383 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3384 else
3385 results->wm_lp[wm_lp - 1] |=
3386 r->fbc_val << WM1_LP_FBC_SHIFT;
3387
3388
3389
3390
3391
3392 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3393 WARN_ON(wm_lp != 1);
3394 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3395 } else
3396 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3397 }
3398
3399
3400 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3401 enum pipe pipe = intel_crtc->pipe;
3402 const struct intel_wm_level *r =
3403 &intel_crtc->wm.active.ilk.wm[0];
3404
3405 if (WARN_ON(!r->enable))
3406 continue;
3407
3408 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3409
3410 results->wm_pipe[pipe] =
3411 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3412 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3413 r->cur_val;
3414 }
3415}
3416
3417
3418
3419static struct intel_pipe_wm *
3420ilk_find_best_result(struct drm_i915_private *dev_priv,
3421 struct intel_pipe_wm *r1,
3422 struct intel_pipe_wm *r2)
3423{
3424 int level, max_level = ilk_wm_max_level(dev_priv);
3425 int level1 = 0, level2 = 0;
3426
3427 for (level = 1; level <= max_level; level++) {
3428 if (r1->wm[level].enable)
3429 level1 = level;
3430 if (r2->wm[level].enable)
3431 level2 = level;
3432 }
3433
3434 if (level1 == level2) {
3435 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3436 return r2;
3437 else
3438 return r1;
3439 } else if (level1 > level2) {
3440 return r1;
3441 } else {
3442 return r2;
3443 }
3444}
3445
3446
3447#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3448#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3449#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3450#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3451#define WM_DIRTY_FBC (1 << 24)
3452#define WM_DIRTY_DDB (1 << 25)
3453
3454static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3455 const struct ilk_wm_values *old,
3456 const struct ilk_wm_values *new)
3457{
3458 unsigned int dirty = 0;
3459 enum pipe pipe;
3460 int wm_lp;
3461
3462 for_each_pipe(dev_priv, pipe) {
3463 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3464 dirty |= WM_DIRTY_LINETIME(pipe);
3465
3466 dirty |= WM_DIRTY_LP_ALL;
3467 }
3468
3469 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3470 dirty |= WM_DIRTY_PIPE(pipe);
3471
3472 dirty |= WM_DIRTY_LP_ALL;
3473 }
3474 }
3475
3476 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3477 dirty |= WM_DIRTY_FBC;
3478
3479 dirty |= WM_DIRTY_LP_ALL;
3480 }
3481
3482 if (old->partitioning != new->partitioning) {
3483 dirty |= WM_DIRTY_DDB;
3484
3485 dirty |= WM_DIRTY_LP_ALL;
3486 }
3487
3488
3489 if (dirty & WM_DIRTY_LP_ALL)
3490 return dirty;
3491
3492
3493 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3494 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3495 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3496 break;
3497 }
3498
3499
3500 for (; wm_lp <= 3; wm_lp++)
3501 dirty |= WM_DIRTY_LP(wm_lp);
3502
3503 return dirty;
3504}
3505
3506static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3507 unsigned int dirty)
3508{
3509 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3510 bool changed = false;
3511
3512 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3513 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3514 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3515 changed = true;
3516 }
3517 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3518 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3519 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3520 changed = true;
3521 }
3522 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3523 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3524 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3525 changed = true;
3526 }
3527
3528
3529
3530
3531
3532
3533 return changed;
3534}
3535
3536
3537
3538
3539
3540static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3541 struct ilk_wm_values *results)
3542{
3543 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3544 unsigned int dirty;
3545 u32 val;
3546
3547 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3548 if (!dirty)
3549 return;
3550
3551 _ilk_disable_lp_wm(dev_priv, dirty);
3552
3553 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3554 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3555 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3556 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3557 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3558 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3559
3560 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3561 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3562 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3563 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3564 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3565 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3566
3567 if (dirty & WM_DIRTY_DDB) {
3568 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3569 val = I915_READ(WM_MISC);
3570 if (results->partitioning == INTEL_DDB_PART_1_2)
3571 val &= ~WM_MISC_DATA_PARTITION_5_6;
3572 else
3573 val |= WM_MISC_DATA_PARTITION_5_6;
3574 I915_WRITE(WM_MISC, val);
3575 } else {
3576 val = I915_READ(DISP_ARB_CTL2);
3577 if (results->partitioning == INTEL_DDB_PART_1_2)
3578 val &= ~DISP_DATA_PARTITION_5_6;
3579 else
3580 val |= DISP_DATA_PARTITION_5_6;
3581 I915_WRITE(DISP_ARB_CTL2, val);
3582 }
3583 }
3584
3585 if (dirty & WM_DIRTY_FBC) {
3586 val = I915_READ(DISP_ARB_CTL);
3587 if (results->enable_fbc_wm)
3588 val &= ~DISP_FBC_WM_DIS;
3589 else
3590 val |= DISP_FBC_WM_DIS;
3591 I915_WRITE(DISP_ARB_CTL, val);
3592 }
3593
3594 if (dirty & WM_DIRTY_LP(1) &&
3595 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3596 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3597
3598 if (INTEL_GEN(dev_priv) >= 7) {
3599 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3600 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3601 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3602 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3603 }
3604
3605 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3606 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3607 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3608 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3609 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3610 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3611
3612 dev_priv->wm.hw = *results;
3613}
3614
3615bool ilk_disable_lp_wm(struct drm_device *dev)
3616{
3617 struct drm_i915_private *dev_priv = to_i915(dev);
3618
3619 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3620}
3621
3622static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3623{
3624 u8 enabled_slices;
3625
3626
3627 enabled_slices = 1;
3628
3629
3630 if (INTEL_GEN(dev_priv) < 11)
3631 return enabled_slices;
3632
3633
3634
3635
3636
3637
3638 if (0 && I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3639 enabled_slices++;
3640
3641 return enabled_slices;
3642}
3643
3644
3645
3646
3647
3648static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3649{
3650 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3651}
3652
3653static bool
3654intel_has_sagv(struct drm_i915_private *dev_priv)
3655{
3656 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3657 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3658}
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671int
3672intel_enable_sagv(struct drm_i915_private *dev_priv)
3673{
3674 int ret;
3675
3676 if (!intel_has_sagv(dev_priv))
3677 return 0;
3678
3679 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3680 return 0;
3681
3682 DRM_DEBUG_KMS("Enabling SAGV\n");
3683 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3684 GEN9_SAGV_ENABLE);
3685
3686
3687
3688
3689
3690
3691
3692 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3693 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3694 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3695 return 0;
3696 } else if (ret < 0) {
3697 DRM_ERROR("Failed to enable SAGV\n");
3698 return ret;
3699 }
3700
3701 dev_priv->sagv_status = I915_SAGV_ENABLED;
3702 return 0;
3703}
3704
3705int
3706intel_disable_sagv(struct drm_i915_private *dev_priv)
3707{
3708 int ret;
3709
3710 if (!intel_has_sagv(dev_priv))
3711 return 0;
3712
3713 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3714 return 0;
3715
3716 DRM_DEBUG_KMS("Disabling SAGV\n");
3717
3718 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3719 GEN9_SAGV_DISABLE,
3720 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3721 1);
3722
3723
3724
3725
3726 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3727 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3728 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3729 return 0;
3730 } else if (ret < 0) {
3731 DRM_ERROR("Failed to disable SAGV (%d)\n", ret);
3732 return ret;
3733 }
3734
3735 dev_priv->sagv_status = I915_SAGV_DISABLED;
3736 return 0;
3737}
3738
3739bool intel_can_enable_sagv(struct drm_atomic_state *state)
3740{
3741 struct drm_device *dev = state->dev;
3742 struct drm_i915_private *dev_priv = to_i915(dev);
3743 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3744 struct intel_crtc *crtc;
3745 struct intel_plane *plane;
3746 struct intel_crtc_state *cstate;
3747 enum pipe pipe;
3748 int level, latency;
3749 int sagv_block_time_us;
3750
3751 if (!intel_has_sagv(dev_priv))
3752 return false;
3753
3754 if (IS_GEN(dev_priv, 9))
3755 sagv_block_time_us = 30;
3756 else if (IS_GEN(dev_priv, 10))
3757 sagv_block_time_us = 20;
3758 else
3759 sagv_block_time_us = 10;
3760
3761
3762
3763
3764 if (hweight32(intel_state->active_crtcs) == 0)
3765 return true;
3766
3767
3768
3769
3770
3771 if (hweight32(intel_state->active_crtcs) > 1)
3772 return false;
3773
3774
3775 pipe = ffs(intel_state->active_crtcs) - 1;
3776 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3777 cstate = to_intel_crtc_state(crtc->base.state);
3778
3779 if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3780 return false;
3781
3782 for_each_intel_plane_on_crtc(dev, crtc, plane) {
3783 struct skl_plane_wm *wm =
3784 &cstate->wm.skl.optimal.planes[plane->id];
3785
3786
3787 if (!wm->wm[0].plane_en)
3788 continue;
3789
3790
3791 for (level = ilk_wm_max_level(dev_priv);
3792 !wm->wm[level].plane_en; --level)
3793 { }
3794
3795 latency = dev_priv->wm.skl_latency[level];
3796
3797 if (skl_needs_memory_bw_wa(dev_priv) &&
3798 plane->base.state->fb->modifier ==
3799 I915_FORMAT_MOD_X_TILED)
3800 latency += 15;
3801
3802
3803
3804
3805
3806
3807 if (latency < sagv_block_time_us)
3808 return false;
3809 }
3810
3811 return true;
3812}
3813
3814static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv,
3815 const struct intel_crtc_state *cstate,
3816 const u64 total_data_rate,
3817 const int num_active,
3818 struct skl_ddb_allocation *ddb)
3819{
3820 const struct drm_display_mode *adjusted_mode;
3821 u64 total_data_bw;
3822 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3823
3824 WARN_ON(ddb_size == 0);
3825
3826 if (INTEL_GEN(dev_priv) < 11)
3827 return ddb_size - 4;
3828
3829 adjusted_mode = &cstate->base.adjusted_mode;
3830 total_data_bw = total_data_rate * drm_mode_vrefresh(adjusted_mode);
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840 if (0 && (num_active > 1 || total_data_bw >= GBps(12))) {
3841 ddb->enabled_slices = 2;
3842 } else {
3843 ddb->enabled_slices = 1;
3844 ddb_size /= 2;
3845 }
3846
3847 return ddb_size;
3848}
3849
3850static void
3851skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
3852 const struct intel_crtc_state *cstate,
3853 const u64 total_data_rate,
3854 struct skl_ddb_allocation *ddb,
3855 struct skl_ddb_entry *alloc,
3856 int *num_active )
3857{
3858 struct drm_atomic_state *state = cstate->base.state;
3859 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3860 struct drm_crtc *for_crtc = cstate->base.crtc;
3861 const struct drm_crtc_state *crtc_state;
3862 const struct drm_crtc *crtc;
3863 u32 pipe_width = 0, total_width = 0, width_before_pipe = 0;
3864 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
3865 u16 ddb_size;
3866 u32 i;
3867
3868 if (WARN_ON(!state) || !cstate->base.active) {
3869 alloc->start = 0;
3870 alloc->end = 0;
3871 *num_active = hweight32(dev_priv->active_crtcs);
3872 return;
3873 }
3874
3875 if (intel_state->active_pipe_changes)
3876 *num_active = hweight32(intel_state->active_crtcs);
3877 else
3878 *num_active = hweight32(dev_priv->active_crtcs);
3879
3880 ddb_size = intel_get_ddb_size(dev_priv, cstate, total_data_rate,
3881 *num_active, ddb);
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891 if (!intel_state->active_pipe_changes && !intel_state->modeset) {
3892
3893
3894
3895
3896 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3897 return;
3898 }
3899
3900
3901
3902
3903
3904
3905 for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3906 const struct drm_display_mode *adjusted_mode;
3907 int hdisplay, vdisplay;
3908 enum pipe pipe;
3909
3910 if (!crtc_state->enable)
3911 continue;
3912
3913 pipe = to_intel_crtc(crtc)->pipe;
3914 adjusted_mode = &crtc_state->adjusted_mode;
3915 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
3916 total_width += hdisplay;
3917
3918 if (pipe < for_pipe)
3919 width_before_pipe += hdisplay;
3920 else if (pipe == for_pipe)
3921 pipe_width = hdisplay;
3922 }
3923
3924 alloc->start = ddb_size * width_before_pipe / total_width;
3925 alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width;
3926}
3927
3928static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
3929 int width, const struct drm_format_info *format,
3930 u64 modifier, unsigned int rotation,
3931 u32 plane_pixel_rate, struct skl_wm_params *wp,
3932 int color_plane);
3933static void skl_compute_plane_wm(const struct intel_crtc_state *cstate,
3934 int level,
3935 const struct skl_wm_params *wp,
3936 const struct skl_wm_level *result_prev,
3937 struct skl_wm_level *result );
3938
3939static unsigned int
3940skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
3941 int num_active)
3942{
3943 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3944 int level, max_level = ilk_wm_max_level(dev_priv);
3945 struct skl_wm_level wm = {};
3946 int ret, min_ddb_alloc = 0;
3947 struct skl_wm_params wp;
3948
3949 ret = skl_compute_wm_params(crtc_state, 256,
3950 drm_format_info(DRM_FORMAT_ARGB8888),
3951 DRM_FORMAT_MOD_LINEAR,
3952 DRM_MODE_ROTATE_0,
3953 crtc_state->pixel_rate, &wp, 0);
3954 WARN_ON(ret);
3955
3956 for (level = 0; level <= max_level; level++) {
3957 skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm);
3958 if (wm.min_ddb_alloc == U16_MAX)
3959 break;
3960
3961 min_ddb_alloc = wm.min_ddb_alloc;
3962 }
3963
3964 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
3965}
3966
3967static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3968 struct skl_ddb_entry *entry, u32 reg)
3969{
3970
3971 entry->start = reg & DDB_ENTRY_MASK;
3972 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
3973
3974 if (entry->end)
3975 entry->end += 1;
3976}
3977
3978static void
3979skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3980 const enum pipe pipe,
3981 const enum plane_id plane_id,
3982 struct skl_ddb_entry *ddb_y,
3983 struct skl_ddb_entry *ddb_uv)
3984{
3985 u32 val, val2;
3986 u32 fourcc = 0;
3987
3988
3989 if (plane_id == PLANE_CURSOR) {
3990 val = I915_READ(CUR_BUF_CFG(pipe));
3991 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
3992 return;
3993 }
3994
3995 val = I915_READ(PLANE_CTL(pipe, plane_id));
3996
3997
3998 if (val & PLANE_CTL_ENABLE)
3999 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4000 val & PLANE_CTL_ORDER_RGBX,
4001 val & PLANE_CTL_ALPHA_MASK);
4002
4003 if (INTEL_GEN(dev_priv) >= 11) {
4004 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4005 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4006 } else {
4007 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4008 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4009
4010 if (is_planar_yuv_format(fourcc))
4011 swap(val, val2);
4012
4013 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4014 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4015 }
4016}
4017
4018void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4019 struct skl_ddb_entry *ddb_y,
4020 struct skl_ddb_entry *ddb_uv)
4021{
4022 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4023 enum intel_display_power_domain power_domain;
4024 enum pipe pipe = crtc->pipe;
4025 intel_wakeref_t wakeref;
4026 enum plane_id plane_id;
4027
4028 power_domain = POWER_DOMAIN_PIPE(pipe);
4029 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4030 if (!wakeref)
4031 return;
4032
4033 for_each_plane_id_on_crtc(crtc, plane_id)
4034 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4035 plane_id,
4036 &ddb_y[plane_id],
4037 &ddb_uv[plane_id]);
4038
4039 intel_display_power_put(dev_priv, power_domain, wakeref);
4040}
4041
4042void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
4043 struct skl_ddb_allocation *ddb )
4044{
4045 ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
4046}
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064static uint_fixed_16_16_t
4065skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
4066 const struct intel_plane_state *pstate)
4067{
4068 struct intel_plane *plane = to_intel_plane(pstate->base.plane);
4069 u32 src_w, src_h, dst_w, dst_h;
4070 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4071 uint_fixed_16_16_t downscale_h, downscale_w;
4072
4073 if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4074 return u32_to_fixed16(0);
4075
4076
4077 if (plane->id == PLANE_CURSOR) {
4078
4079
4080
4081
4082 src_w = pstate->base.src_w >> 16;
4083 src_h = pstate->base.src_h >> 16;
4084 dst_w = pstate->base.crtc_w;
4085 dst_h = pstate->base.crtc_h;
4086 } else {
4087
4088
4089
4090
4091
4092 src_w = drm_rect_width(&pstate->base.src) >> 16;
4093 src_h = drm_rect_height(&pstate->base.src) >> 16;
4094 dst_w = drm_rect_width(&pstate->base.dst);
4095 dst_h = drm_rect_height(&pstate->base.dst);
4096 }
4097
4098 fp_w_ratio = div_fixed16(src_w, dst_w);
4099 fp_h_ratio = div_fixed16(src_h, dst_h);
4100 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4101 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4102
4103 return mul_fixed16(downscale_w, downscale_h);
4104}
4105
4106static uint_fixed_16_16_t
4107skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
4108{
4109 uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
4110
4111 if (!crtc_state->base.enable)
4112 return pipe_downscale;
4113
4114 if (crtc_state->pch_pfit.enabled) {
4115 u32 src_w, src_h, dst_w, dst_h;
4116 u32 pfit_size = crtc_state->pch_pfit.size;
4117 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4118 uint_fixed_16_16_t downscale_h, downscale_w;
4119
4120 src_w = crtc_state->pipe_src_w;
4121 src_h = crtc_state->pipe_src_h;
4122 dst_w = pfit_size >> 16;
4123 dst_h = pfit_size & 0xffff;
4124
4125 if (!dst_w || !dst_h)
4126 return pipe_downscale;
4127
4128 fp_w_ratio = div_fixed16(src_w, dst_w);
4129 fp_h_ratio = div_fixed16(src_h, dst_h);
4130 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4131 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4132
4133 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
4134 }
4135
4136 return pipe_downscale;
4137}
4138
4139int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
4140 struct intel_crtc_state *cstate)
4141{
4142 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4143 struct drm_crtc_state *crtc_state = &cstate->base;
4144 struct drm_atomic_state *state = crtc_state->state;
4145 struct drm_plane *plane;
4146 const struct drm_plane_state *pstate;
4147 struct intel_plane_state *intel_pstate;
4148 int crtc_clock, dotclk;
4149 u32 pipe_max_pixel_rate;
4150 uint_fixed_16_16_t pipe_downscale;
4151 uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
4152
4153 if (!cstate->base.enable)
4154 return 0;
4155
4156 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4157 uint_fixed_16_16_t plane_downscale;
4158 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
4159 int bpp;
4160
4161 if (!intel_wm_plane_visible(cstate,
4162 to_intel_plane_state(pstate)))
4163 continue;
4164
4165 if (WARN_ON(!pstate->fb))
4166 return -EINVAL;
4167
4168 intel_pstate = to_intel_plane_state(pstate);
4169 plane_downscale = skl_plane_downscale_amount(cstate,
4170 intel_pstate);
4171 bpp = pstate->fb->format->cpp[0] * 8;
4172 if (bpp == 64)
4173 plane_downscale = mul_fixed16(plane_downscale,
4174 fp_9_div_8);
4175
4176 max_downscale = max_fixed16(plane_downscale, max_downscale);
4177 }
4178 pipe_downscale = skl_pipe_downscale_amount(cstate);
4179
4180 pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
4181
4182 crtc_clock = crtc_state->adjusted_mode.crtc_clock;
4183 dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
4184
4185 if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
4186 dotclk *= 2;
4187
4188 pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
4189
4190 if (pipe_max_pixel_rate < crtc_clock) {
4191 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
4192 return -EINVAL;
4193 }
4194
4195 return 0;
4196}
4197
4198static u64
4199skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
4200 const struct intel_plane_state *intel_pstate,
4201 const int plane)
4202{
4203 struct intel_plane *intel_plane =
4204 to_intel_plane(intel_pstate->base.plane);
4205 u32 data_rate;
4206 u32 width = 0, height = 0;
4207 struct drm_framebuffer *fb;
4208 u32 format;
4209 uint_fixed_16_16_t down_scale_amount;
4210 u64 rate;
4211
4212 if (!intel_pstate->base.visible)
4213 return 0;
4214
4215 fb = intel_pstate->base.fb;
4216 format = fb->format->format;
4217
4218 if (intel_plane->id == PLANE_CURSOR)
4219 return 0;
4220 if (plane == 1 && !is_planar_yuv_format(format))
4221 return 0;
4222
4223
4224
4225
4226
4227
4228 width = drm_rect_width(&intel_pstate->base.src) >> 16;
4229 height = drm_rect_height(&intel_pstate->base.src) >> 16;
4230
4231
4232 if (plane == 1 && is_planar_yuv_format(format)) {
4233 width /= 2;
4234 height /= 2;
4235 }
4236
4237 data_rate = width * height;
4238
4239 down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
4240
4241 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4242
4243 rate *= fb->format->cpp[plane];
4244 return rate;
4245}
4246
4247static u64
4248skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4249 u64 *plane_data_rate,
4250 u64 *uv_plane_data_rate)
4251{
4252 struct drm_crtc_state *cstate = &intel_cstate->base;
4253 struct drm_atomic_state *state = cstate->state;
4254 struct drm_plane *plane;
4255 const struct drm_plane_state *pstate;
4256 u64 total_data_rate = 0;
4257
4258 if (WARN_ON(!state))
4259 return 0;
4260
4261
4262 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4263 enum plane_id plane_id = to_intel_plane(plane)->id;
4264 u64 rate;
4265 const struct intel_plane_state *intel_pstate =
4266 to_intel_plane_state(pstate);
4267
4268
4269 rate = skl_plane_relative_data_rate(intel_cstate,
4270 intel_pstate, 0);
4271 plane_data_rate[plane_id] = rate;
4272 total_data_rate += rate;
4273
4274
4275 rate = skl_plane_relative_data_rate(intel_cstate,
4276 intel_pstate, 1);
4277 uv_plane_data_rate[plane_id] = rate;
4278 total_data_rate += rate;
4279 }
4280
4281 return total_data_rate;
4282}
4283
4284static u64
4285icl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4286 u64 *plane_data_rate)
4287{
4288 struct drm_crtc_state *cstate = &intel_cstate->base;
4289 struct drm_atomic_state *state = cstate->state;
4290 struct drm_plane *plane;
4291 const struct drm_plane_state *pstate;
4292 u64 total_data_rate = 0;
4293
4294 if (WARN_ON(!state))
4295 return 0;
4296
4297
4298 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4299 const struct intel_plane_state *intel_pstate =
4300 to_intel_plane_state(pstate);
4301 enum plane_id plane_id = to_intel_plane(plane)->id;
4302 u64 rate;
4303
4304 if (!intel_pstate->linked_plane) {
4305 rate = skl_plane_relative_data_rate(intel_cstate,
4306 intel_pstate, 0);
4307 plane_data_rate[plane_id] = rate;
4308 total_data_rate += rate;
4309 } else {
4310 enum plane_id y_plane_id;
4311
4312
4313
4314
4315
4316
4317
4318
4319 if (intel_pstate->slave)
4320 continue;
4321
4322
4323 rate = skl_plane_relative_data_rate(intel_cstate,
4324 intel_pstate, 0);
4325 y_plane_id = intel_pstate->linked_plane->id;
4326 plane_data_rate[y_plane_id] = rate;
4327 total_data_rate += rate;
4328
4329 rate = skl_plane_relative_data_rate(intel_cstate,
4330 intel_pstate, 1);
4331 plane_data_rate[plane_id] = rate;
4332 total_data_rate += rate;
4333 }
4334 }
4335
4336 return total_data_rate;
4337}
4338
4339static int
4340skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
4341 struct skl_ddb_allocation *ddb )
4342{
4343 struct drm_atomic_state *state = cstate->base.state;
4344 struct drm_crtc *crtc = cstate->base.crtc;
4345 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
4346 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4347 struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
4348 u16 alloc_size, start = 0;
4349 u16 total[I915_MAX_PLANES] = {};
4350 u16 uv_total[I915_MAX_PLANES] = {};
4351 u64 total_data_rate;
4352 enum plane_id plane_id;
4353 int num_active;
4354 u64 plane_data_rate[I915_MAX_PLANES] = {};
4355 u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4356 u32 blocks;
4357 int level;
4358
4359
4360 memset(cstate->wm.skl.plane_ddb_y, 0, sizeof(cstate->wm.skl.plane_ddb_y));
4361 memset(cstate->wm.skl.plane_ddb_uv, 0, sizeof(cstate->wm.skl.plane_ddb_uv));
4362
4363 if (WARN_ON(!state))
4364 return 0;
4365
4366 if (!cstate->base.active) {
4367 alloc->start = alloc->end = 0;
4368 return 0;
4369 }
4370
4371 if (INTEL_GEN(dev_priv) >= 11)
4372 total_data_rate =
4373 icl_get_total_relative_data_rate(cstate,
4374 plane_data_rate);
4375 else
4376 total_data_rate =
4377 skl_get_total_relative_data_rate(cstate,
4378 plane_data_rate,
4379 uv_plane_data_rate);
4380
4381
4382 skl_ddb_get_pipe_allocation_limits(dev_priv, cstate, total_data_rate,
4383 ddb, alloc, &num_active);
4384 alloc_size = skl_ddb_entry_size(alloc);
4385 if (alloc_size == 0)
4386 return 0;
4387
4388
4389 total[PLANE_CURSOR] = skl_cursor_allocation(cstate, num_active);
4390 alloc_size -= total[PLANE_CURSOR];
4391 cstate->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4392 alloc->end - total[PLANE_CURSOR];
4393 cstate->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4394
4395 if (total_data_rate == 0)
4396 return 0;
4397
4398
4399
4400
4401
4402 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4403 blocks = 0;
4404 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4405 const struct skl_plane_wm *wm =
4406 &cstate->wm.skl.optimal.planes[plane_id];
4407
4408 if (plane_id == PLANE_CURSOR) {
4409 if (WARN_ON(wm->wm[level].min_ddb_alloc >
4410 total[PLANE_CURSOR])) {
4411 blocks = U32_MAX;
4412 break;
4413 }
4414 continue;
4415 }
4416
4417 blocks += wm->wm[level].min_ddb_alloc;
4418 blocks += wm->uv_wm[level].min_ddb_alloc;
4419 }
4420
4421 if (blocks <= alloc_size) {
4422 alloc_size -= blocks;
4423 break;
4424 }
4425 }
4426
4427 if (level < 0) {
4428 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4429 DRM_DEBUG_KMS("minimum required %d/%d\n", blocks,
4430 alloc_size);
4431 return -EINVAL;
4432 }
4433
4434
4435
4436
4437
4438
4439 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4440 const struct skl_plane_wm *wm =
4441 &cstate->wm.skl.optimal.planes[plane_id];
4442 u64 rate;
4443 u16 extra;
4444
4445 if (plane_id == PLANE_CURSOR)
4446 continue;
4447
4448
4449
4450
4451
4452 if (total_data_rate == 0)
4453 break;
4454
4455 rate = plane_data_rate[plane_id];
4456 extra = min_t(u16, alloc_size,
4457 DIV64_U64_ROUND_UP(alloc_size * rate,
4458 total_data_rate));
4459 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4460 alloc_size -= extra;
4461 total_data_rate -= rate;
4462
4463 if (total_data_rate == 0)
4464 break;
4465
4466 rate = uv_plane_data_rate[plane_id];
4467 extra = min_t(u16, alloc_size,
4468 DIV64_U64_ROUND_UP(alloc_size * rate,
4469 total_data_rate));
4470 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4471 alloc_size -= extra;
4472 total_data_rate -= rate;
4473 }
4474 WARN_ON(alloc_size != 0 || total_data_rate != 0);
4475
4476
4477 start = alloc->start;
4478 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4479 struct skl_ddb_entry *plane_alloc =
4480 &cstate->wm.skl.plane_ddb_y[plane_id];
4481 struct skl_ddb_entry *uv_plane_alloc =
4482 &cstate->wm.skl.plane_ddb_uv[plane_id];
4483
4484 if (plane_id == PLANE_CURSOR)
4485 continue;
4486
4487
4488 WARN_ON(INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4489
4490
4491 if (total[plane_id]) {
4492 plane_alloc->start = start;
4493 start += total[plane_id];
4494 plane_alloc->end = start;
4495 }
4496
4497 if (uv_total[plane_id]) {
4498 uv_plane_alloc->start = start;
4499 start += uv_total[plane_id];
4500 uv_plane_alloc->end = start;
4501 }
4502 }
4503
4504
4505
4506
4507
4508
4509
4510 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4511 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4512 struct skl_plane_wm *wm =
4513 &cstate->wm.skl.optimal.planes[plane_id];
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
4528 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
4529 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
4530
4531
4532
4533
4534
4535 if (IS_GEN(dev_priv, 11) &&
4536 level == 1 && wm->wm[0].plane_en) {
4537 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
4538 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
4539 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4540 }
4541 }
4542 }
4543
4544
4545
4546
4547
4548 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4549 struct skl_plane_wm *wm =
4550 &cstate->wm.skl.optimal.planes[plane_id];
4551
4552 if (wm->trans_wm.plane_res_b >= total[plane_id])
4553 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
4554 }
4555
4556 return 0;
4557}
4558
4559
4560
4561
4562
4563
4564
4565static uint_fixed_16_16_t
4566skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
4567 u8 cpp, u32 latency, u32 dbuf_block_size)
4568{
4569 u32 wm_intermediate_val;
4570 uint_fixed_16_16_t ret;
4571
4572 if (latency == 0)
4573 return FP_16_16_MAX;
4574
4575 wm_intermediate_val = latency * pixel_rate * cpp;
4576 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4577
4578 if (INTEL_GEN(dev_priv) >= 10)
4579 ret = add_fixed16_u32(ret, 1);
4580
4581 return ret;
4582}
4583
4584static uint_fixed_16_16_t
4585skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
4586 uint_fixed_16_16_t plane_blocks_per_line)
4587{
4588 u32 wm_intermediate_val;
4589 uint_fixed_16_16_t ret;
4590
4591 if (latency == 0)
4592 return FP_16_16_MAX;
4593
4594 wm_intermediate_val = latency * pixel_rate;
4595 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4596 pipe_htotal * 1000);
4597 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4598 return ret;
4599}
4600
4601static uint_fixed_16_16_t
4602intel_get_linetime_us(const struct intel_crtc_state *cstate)
4603{
4604 u32 pixel_rate;
4605 u32 crtc_htotal;
4606 uint_fixed_16_16_t linetime_us;
4607
4608 if (!cstate->base.active)
4609 return u32_to_fixed16(0);
4610
4611 pixel_rate = cstate->pixel_rate;
4612
4613 if (WARN_ON(pixel_rate == 0))
4614 return u32_to_fixed16(0);
4615
4616 crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
4617 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4618
4619 return linetime_us;
4620}
4621
4622static u32
4623skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
4624 const struct intel_plane_state *pstate)
4625{
4626 u64 adjusted_pixel_rate;
4627 uint_fixed_16_16_t downscale_amount;
4628
4629
4630 if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4631 return 0;
4632
4633
4634
4635
4636
4637 adjusted_pixel_rate = cstate->pixel_rate;
4638 downscale_amount = skl_plane_downscale_amount(cstate, pstate);
4639
4640 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4641 downscale_amount);
4642}
4643
4644static int
4645skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4646 int width, const struct drm_format_info *format,
4647 u64 modifier, unsigned int rotation,
4648 u32 plane_pixel_rate, struct skl_wm_params *wp,
4649 int color_plane)
4650{
4651 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4652 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4653 u32 interm_pbpl;
4654
4655
4656 if (color_plane == 1 && !is_planar_yuv_format(format->format)) {
4657 DRM_DEBUG_KMS("Non planar format have single plane\n");
4658 return -EINVAL;
4659 }
4660
4661 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
4662 modifier == I915_FORMAT_MOD_Yf_TILED ||
4663 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4664 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4665 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
4666 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4667 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4668 wp->is_planar = is_planar_yuv_format(format->format);
4669
4670 wp->width = width;
4671 if (color_plane == 1 && wp->is_planar)
4672 wp->width /= 2;
4673
4674 wp->cpp = format->cpp[color_plane];
4675 wp->plane_pixel_rate = plane_pixel_rate;
4676
4677 if (INTEL_GEN(dev_priv) >= 11 &&
4678 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
4679 wp->dbuf_block_size = 256;
4680 else
4681 wp->dbuf_block_size = 512;
4682
4683 if (drm_rotation_90_or_270(rotation)) {
4684 switch (wp->cpp) {
4685 case 1:
4686 wp->y_min_scanlines = 16;
4687 break;
4688 case 2:
4689 wp->y_min_scanlines = 8;
4690 break;
4691 case 4:
4692 wp->y_min_scanlines = 4;
4693 break;
4694 default:
4695 MISSING_CASE(wp->cpp);
4696 return -EINVAL;
4697 }
4698 } else {
4699 wp->y_min_scanlines = 4;
4700 }
4701
4702 if (skl_needs_memory_bw_wa(dev_priv))
4703 wp->y_min_scanlines *= 2;
4704
4705 wp->plane_bytes_per_line = wp->width * wp->cpp;
4706 if (wp->y_tiled) {
4707 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4708 wp->y_min_scanlines,
4709 wp->dbuf_block_size);
4710
4711 if (INTEL_GEN(dev_priv) >= 10)
4712 interm_pbpl++;
4713
4714 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4715 wp->y_min_scanlines);
4716 } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) {
4717 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4718 wp->dbuf_block_size);
4719 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4720 } else {
4721 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4722 wp->dbuf_block_size) + 1;
4723 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4724 }
4725
4726 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4727 wp->plane_blocks_per_line);
4728
4729 wp->linetime_us = fixed16_to_u32_round_up(
4730 intel_get_linetime_us(crtc_state));
4731
4732 return 0;
4733}
4734
4735static int
4736skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
4737 const struct intel_plane_state *plane_state,
4738 struct skl_wm_params *wp, int color_plane)
4739{
4740 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4741 const struct drm_framebuffer *fb = plane_state->base.fb;
4742 int width;
4743
4744 if (plane->id == PLANE_CURSOR) {
4745 width = plane_state->base.crtc_w;
4746 } else {
4747
4748
4749
4750
4751
4752 width = drm_rect_width(&plane_state->base.src) >> 16;
4753 }
4754
4755 return skl_compute_wm_params(crtc_state, width,
4756 fb->format, fb->modifier,
4757 plane_state->base.rotation,
4758 skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
4759 wp, color_plane);
4760}
4761
4762static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
4763{
4764 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4765 return true;
4766
4767
4768 return level > 0;
4769}
4770
4771static void skl_compute_plane_wm(const struct intel_crtc_state *cstate,
4772 int level,
4773 const struct skl_wm_params *wp,
4774 const struct skl_wm_level *result_prev,
4775 struct skl_wm_level *result )
4776{
4777 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
4778 u32 latency = dev_priv->wm.skl_latency[level];
4779 uint_fixed_16_16_t method1, method2;
4780 uint_fixed_16_16_t selected_result;
4781 u32 res_blocks, res_lines, min_ddb_alloc = 0;
4782
4783 if (latency == 0) {
4784
4785 result->min_ddb_alloc = U16_MAX;
4786 return;
4787 }
4788
4789
4790
4791
4792
4793 if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) ||
4794 dev_priv->ipc_enabled)
4795 latency += 4;
4796
4797 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
4798 latency += 15;
4799
4800 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4801 wp->cpp, latency, wp->dbuf_block_size);
4802 method2 = skl_wm_method2(wp->plane_pixel_rate,
4803 cstate->base.adjusted_mode.crtc_htotal,
4804 latency,
4805 wp->plane_blocks_per_line);
4806
4807 if (wp->y_tiled) {
4808 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4809 } else {
4810 if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
4811 wp->dbuf_block_size < 1) &&
4812 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
4813 selected_result = method2;
4814 } else if (latency >= wp->linetime_us) {
4815 if (IS_GEN(dev_priv, 9) &&
4816 !IS_GEMINILAKE(dev_priv))
4817 selected_result = min_fixed16(method1, method2);
4818 else
4819 selected_result = method2;
4820 } else {
4821 selected_result = method1;
4822 }
4823 }
4824
4825 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4826 res_lines = div_round_up_fixed16(selected_result,
4827 wp->plane_blocks_per_line);
4828
4829 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
4830
4831 if (level == 0 && wp->rc_surface)
4832 res_blocks +=
4833 fixed16_to_u32_round_up(wp->y_tile_minimum);
4834
4835
4836 if (level >= 1 && level <= 7) {
4837 if (wp->y_tiled) {
4838 res_blocks +=
4839 fixed16_to_u32_round_up(wp->y_tile_minimum);
4840 res_lines += wp->y_min_scanlines;
4841 } else {
4842 res_blocks++;
4843 }
4844
4845
4846
4847
4848
4849
4850
4851 if (result_prev->plane_res_b > res_blocks)
4852 res_blocks = result_prev->plane_res_b;
4853 }
4854 }
4855
4856 if (INTEL_GEN(dev_priv) >= 11) {
4857 if (wp->y_tiled) {
4858 int extra_lines;
4859
4860 if (res_lines % wp->y_min_scanlines == 0)
4861 extra_lines = wp->y_min_scanlines;
4862 else
4863 extra_lines = wp->y_min_scanlines * 2 -
4864 res_lines % wp->y_min_scanlines;
4865
4866 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
4867 wp->plane_blocks_per_line);
4868 } else {
4869 min_ddb_alloc = res_blocks +
4870 DIV_ROUND_UP(res_blocks, 10);
4871 }
4872 }
4873
4874 if (!skl_wm_has_lines(dev_priv, level))
4875 res_lines = 0;
4876
4877 if (res_lines > 31) {
4878
4879 result->min_ddb_alloc = U16_MAX;
4880 return;
4881 }
4882
4883
4884
4885
4886
4887
4888
4889 result->plane_res_b = res_blocks;
4890 result->plane_res_l = res_lines;
4891
4892 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
4893 result->plane_en = true;
4894}
4895
4896static void
4897skl_compute_wm_levels(const struct intel_crtc_state *cstate,
4898 const struct skl_wm_params *wm_params,
4899 struct skl_wm_level *levels)
4900{
4901 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
4902 int level, max_level = ilk_wm_max_level(dev_priv);
4903 struct skl_wm_level *result_prev = &levels[0];
4904
4905 for (level = 0; level <= max_level; level++) {
4906 struct skl_wm_level *result = &levels[level];
4907
4908 skl_compute_plane_wm(cstate, level, wm_params,
4909 result_prev, result);
4910
4911 result_prev = result;
4912 }
4913}
4914
4915static u32
4916skl_compute_linetime_wm(const struct intel_crtc_state *cstate)
4917{
4918 struct drm_atomic_state *state = cstate->base.state;
4919 struct drm_i915_private *dev_priv = to_i915(state->dev);
4920 uint_fixed_16_16_t linetime_us;
4921 u32 linetime_wm;
4922
4923 linetime_us = intel_get_linetime_us(cstate);
4924 linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4925
4926
4927 if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled)
4928 linetime_wm /= 2;
4929
4930 return linetime_wm;
4931}
4932
4933static void skl_compute_transition_wm(const struct intel_crtc_state *cstate,
4934 const struct skl_wm_params *wp,
4935 struct skl_plane_wm *wm)
4936{
4937 struct drm_device *dev = cstate->base.crtc->dev;
4938 const struct drm_i915_private *dev_priv = to_i915(dev);
4939 u16 trans_min, trans_y_tile_min;
4940 const u16 trans_amount = 10;
4941 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
4942
4943
4944 if (INTEL_GEN(dev_priv) <= 9)
4945 return;
4946
4947
4948 if (!dev_priv->ipc_enabled)
4949 return;
4950
4951 trans_min = 14;
4952 if (INTEL_GEN(dev_priv) >= 11)
4953 trans_min = 4;
4954
4955 trans_offset_b = trans_min + trans_amount;
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967 wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
4968
4969 if (wp->y_tiled) {
4970 trans_y_tile_min =
4971 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
4972 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
4973 trans_offset_b;
4974 } else {
4975 res_blocks = wm0_sel_res_b + trans_offset_b;
4976
4977
4978 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4979 res_blocks += 1;
4980
4981 }
4982
4983
4984
4985
4986
4987
4988 wm->trans_wm.plane_res_b = res_blocks + 1;
4989 wm->trans_wm.plane_en = true;
4990}
4991
4992static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
4993 const struct intel_plane_state *plane_state,
4994 enum plane_id plane_id, int color_plane)
4995{
4996 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
4997 struct skl_wm_params wm_params;
4998 int ret;
4999
5000 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5001 &wm_params, color_plane);
5002 if (ret)
5003 return ret;
5004
5005 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5006 skl_compute_transition_wm(crtc_state, &wm_params, wm);
5007
5008 return 0;
5009}
5010
5011static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5012 const struct intel_plane_state *plane_state,
5013 enum plane_id plane_id)
5014{
5015 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5016 struct skl_wm_params wm_params;
5017 int ret;
5018
5019 wm->is_planar = true;
5020
5021
5022 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5023 &wm_params, 1);
5024 if (ret)
5025 return ret;
5026
5027 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5028
5029 return 0;
5030}
5031
5032static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5033 const struct intel_plane_state *plane_state)
5034{
5035 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
5036 const struct drm_framebuffer *fb = plane_state->base.fb;
5037 enum plane_id plane_id = plane->id;
5038 int ret;
5039
5040 if (!intel_wm_plane_visible(crtc_state, plane_state))
5041 return 0;
5042
5043 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5044 plane_id, 0);
5045 if (ret)
5046 return ret;
5047
5048 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5049 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5050 plane_id);
5051 if (ret)
5052 return ret;
5053 }
5054
5055 return 0;
5056}
5057
5058static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5059 const struct intel_plane_state *plane_state)
5060{
5061 enum plane_id plane_id = to_intel_plane(plane_state->base.plane)->id;
5062 int ret;
5063
5064
5065 if (plane_state->slave)
5066 return 0;
5067
5068 if (plane_state->linked_plane) {
5069 const struct drm_framebuffer *fb = plane_state->base.fb;
5070 enum plane_id y_plane_id = plane_state->linked_plane->id;
5071
5072 WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state));
5073 WARN_ON(!fb->format->is_yuv ||
5074 fb->format->num_planes == 1);
5075
5076 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5077 y_plane_id, 0);
5078 if (ret)
5079 return ret;
5080
5081 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5082 plane_id, 1);
5083 if (ret)
5084 return ret;
5085 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5086 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5087 plane_id, 0);
5088 if (ret)
5089 return ret;
5090 }
5091
5092 return 0;
5093}
5094
5095static int skl_build_pipe_wm(struct intel_crtc_state *cstate)
5096{
5097 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5098 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5099 struct drm_crtc_state *crtc_state = &cstate->base;
5100 struct drm_plane *plane;
5101 const struct drm_plane_state *pstate;
5102 int ret;
5103
5104
5105
5106
5107
5108 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5109
5110 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
5111 const struct intel_plane_state *intel_pstate =
5112 to_intel_plane_state(pstate);
5113
5114 if (INTEL_GEN(dev_priv) >= 11)
5115 ret = icl_build_plane_wm(cstate, intel_pstate);
5116 else
5117 ret = skl_build_plane_wm(cstate, intel_pstate);
5118 if (ret)
5119 return ret;
5120 }
5121
5122 pipe_wm->linetime = skl_compute_linetime_wm(cstate);
5123
5124 return 0;
5125}
5126
5127static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5128 i915_reg_t reg,
5129 const struct skl_ddb_entry *entry)
5130{
5131 if (entry->end)
5132 I915_WRITE_FW(reg, (entry->end - 1) << 16 | entry->start);
5133 else
5134 I915_WRITE_FW(reg, 0);
5135}
5136
5137static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5138 i915_reg_t reg,
5139 const struct skl_wm_level *level)
5140{
5141 u32 val = 0;
5142
5143 if (level->plane_en)
5144 val |= PLANE_WM_EN;
5145 if (level->ignore_lines)
5146 val |= PLANE_WM_IGNORE_LINES;
5147 val |= level->plane_res_b;
5148 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5149
5150 I915_WRITE_FW(reg, val);
5151}
5152
5153void skl_write_plane_wm(struct intel_plane *plane,
5154 const struct intel_crtc_state *crtc_state)
5155{
5156 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5157 int level, max_level = ilk_wm_max_level(dev_priv);
5158 enum plane_id plane_id = plane->id;
5159 enum pipe pipe = plane->pipe;
5160 const struct skl_plane_wm *wm =
5161 &crtc_state->wm.skl.optimal.planes[plane_id];
5162 const struct skl_ddb_entry *ddb_y =
5163 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5164 const struct skl_ddb_entry *ddb_uv =
5165 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5166
5167 for (level = 0; level <= max_level; level++) {
5168 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5169 &wm->wm[level]);
5170 }
5171 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5172 &wm->trans_wm);
5173
5174 if (INTEL_GEN(dev_priv) >= 11) {
5175 skl_ddb_entry_write(dev_priv,
5176 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5177 return;
5178 }
5179
5180 if (wm->is_planar)
5181 swap(ddb_y, ddb_uv);
5182
5183 skl_ddb_entry_write(dev_priv,
5184 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5185 skl_ddb_entry_write(dev_priv,
5186 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5187}
5188
5189void skl_write_cursor_wm(struct intel_plane *plane,
5190 const struct intel_crtc_state *crtc_state)
5191{
5192 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5193 int level, max_level = ilk_wm_max_level(dev_priv);
5194 enum plane_id plane_id = plane->id;
5195 enum pipe pipe = plane->pipe;
5196 const struct skl_plane_wm *wm =
5197 &crtc_state->wm.skl.optimal.planes[plane_id];
5198 const struct skl_ddb_entry *ddb =
5199 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5200
5201 for (level = 0; level <= max_level; level++) {
5202 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5203 &wm->wm[level]);
5204 }
5205 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5206
5207 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5208}
5209
5210bool skl_wm_level_equals(const struct skl_wm_level *l1,
5211 const struct skl_wm_level *l2)
5212{
5213 return l1->plane_en == l2->plane_en &&
5214 l1->ignore_lines == l2->ignore_lines &&
5215 l1->plane_res_l == l2->plane_res_l &&
5216 l1->plane_res_b == l2->plane_res_b;
5217}
5218
5219static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5220 const struct skl_plane_wm *wm1,
5221 const struct skl_plane_wm *wm2)
5222{
5223 int level, max_level = ilk_wm_max_level(dev_priv);
5224
5225 for (level = 0; level <= max_level; level++) {
5226 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) ||
5227 !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level]))
5228 return false;
5229 }
5230
5231 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5232}
5233
5234static bool skl_pipe_wm_equals(struct intel_crtc *crtc,
5235 const struct skl_pipe_wm *wm1,
5236 const struct skl_pipe_wm *wm2)
5237{
5238 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5239 enum plane_id plane_id;
5240
5241 for_each_plane_id_on_crtc(crtc, plane_id) {
5242 if (!skl_plane_wm_equals(dev_priv,
5243 &wm1->planes[plane_id],
5244 &wm2->planes[plane_id]))
5245 return false;
5246 }
5247
5248 return wm1->linetime == wm2->linetime;
5249}
5250
5251static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5252 const struct skl_ddb_entry *b)
5253{
5254 return a->start < b->end && b->start < a->end;
5255}
5256
5257bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5258 const struct skl_ddb_entry *entries,
5259 int num_entries, int ignore_idx)
5260{
5261 int i;
5262
5263 for (i = 0; i < num_entries; i++) {
5264 if (i != ignore_idx &&
5265 skl_ddb_entries_overlap(ddb, &entries[i]))
5266 return true;
5267 }
5268
5269 return false;
5270}
5271
5272static u32
5273pipes_modified(struct intel_atomic_state *state)
5274{
5275 struct intel_crtc *crtc;
5276 struct intel_crtc_state *cstate;
5277 u32 i, ret = 0;
5278
5279 for_each_new_intel_crtc_in_state(state, crtc, cstate, i)
5280 ret |= drm_crtc_mask(&crtc->base);
5281
5282 return ret;
5283}
5284
5285static int
5286skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5287 struct intel_crtc_state *new_crtc_state)
5288{
5289 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->base.state);
5290 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5291 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5292 struct intel_plane *plane;
5293
5294 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5295 struct intel_plane_state *plane_state;
5296 enum plane_id plane_id = plane->id;
5297
5298 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5299 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5300 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5301 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5302 continue;
5303
5304 plane_state = intel_atomic_get_plane_state(state, plane);
5305 if (IS_ERR(plane_state))
5306 return PTR_ERR(plane_state);
5307
5308 new_crtc_state->update_planes |= BIT(plane_id);
5309 }
5310
5311 return 0;
5312}
5313
5314static int
5315skl_compute_ddb(struct intel_atomic_state *state)
5316{
5317 const struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5318 struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5319 struct intel_crtc_state *old_crtc_state;
5320 struct intel_crtc_state *new_crtc_state;
5321 struct intel_crtc *crtc;
5322 int ret, i;
5323
5324 memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5325
5326 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5327 new_crtc_state, i) {
5328 ret = skl_allocate_pipe_ddb(new_crtc_state, ddb);
5329 if (ret)
5330 return ret;
5331
5332 ret = skl_ddb_add_affected_planes(old_crtc_state,
5333 new_crtc_state);
5334 if (ret)
5335 return ret;
5336 }
5337
5338 return 0;
5339}
5340
5341static char enast(bool enable)
5342{
5343 return enable ? '*' : ' ';
5344}
5345
5346static void
5347skl_print_wm_changes(struct intel_atomic_state *state)
5348{
5349 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5350 const struct intel_crtc_state *old_crtc_state;
5351 const struct intel_crtc_state *new_crtc_state;
5352 struct intel_plane *plane;
5353 struct intel_crtc *crtc;
5354 int i;
5355
5356 if ((drm_debug & DRM_UT_KMS) == 0)
5357 return;
5358
5359 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5360 new_crtc_state, i) {
5361 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5362
5363 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5364 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5365
5366 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5367 enum plane_id plane_id = plane->id;
5368 const struct skl_ddb_entry *old, *new;
5369
5370 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5371 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5372
5373 if (skl_ddb_entry_equal(old, new))
5374 continue;
5375
5376 DRM_DEBUG_KMS("[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5377 plane->base.base.id, plane->base.name,
5378 old->start, old->end, new->start, new->end,
5379 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5380 }
5381
5382 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5383 enum plane_id plane_id = plane->id;
5384 const struct skl_plane_wm *old_wm, *new_wm;
5385
5386 old_wm = &old_pipe_wm->planes[plane_id];
5387 new_wm = &new_pipe_wm->planes[plane_id];
5388
5389 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5390 continue;
5391
5392 DRM_DEBUG_KMS("[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm"
5393 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm\n",
5394 plane->base.base.id, plane->base.name,
5395 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5396 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5397 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5398 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5399 enast(old_wm->trans_wm.plane_en),
5400 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5401 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5402 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5403 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5404 enast(new_wm->trans_wm.plane_en));
5405
5406 DRM_DEBUG_KMS("[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5407 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5408 plane->base.base.id, plane->base.name,
5409 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5410 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5411 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5412 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5413 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5414 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5415 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5416 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5417 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5418
5419 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5420 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5421 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5422 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5423 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5424 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5425 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5426 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5427 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l);
5428
5429 DRM_DEBUG_KMS("[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5430 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5431 plane->base.base.id, plane->base.name,
5432 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5433 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5434 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5435 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5436 old_wm->trans_wm.plane_res_b,
5437 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5438 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5439 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5440 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5441 new_wm->trans_wm.plane_res_b);
5442
5443 DRM_DEBUG_KMS("[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5444 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5445 plane->base.base.id, plane->base.name,
5446 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5447 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5448 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5449 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5450 old_wm->trans_wm.min_ddb_alloc,
5451 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5452 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5453 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5454 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5455 new_wm->trans_wm.min_ddb_alloc);
5456 }
5457 }
5458}
5459
5460static int
5461skl_ddb_add_affected_pipes(struct intel_atomic_state *state, bool *changed)
5462{
5463 struct drm_device *dev = state->base.dev;
5464 const struct drm_i915_private *dev_priv = to_i915(dev);
5465 struct intel_crtc *crtc;
5466 struct intel_crtc_state *crtc_state;
5467 u32 realloc_pipes = pipes_modified(state);
5468 int ret, i;
5469
5470
5471
5472
5473
5474 if (dev_priv->wm.distrust_bios_wm)
5475 (*changed) = true;
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i)
5486 (*changed) = true;
5487
5488 if (!*changed)
5489 return 0;
5490
5491
5492
5493
5494
5495
5496
5497 if (dev_priv->wm.distrust_bios_wm) {
5498 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
5499 state->base.acquire_ctx);
5500 if (ret)
5501 return ret;
5502
5503 state->active_pipe_changes = ~0;
5504
5505
5506
5507
5508
5509
5510
5511 if (!state->modeset)
5512 state->active_crtcs = dev_priv->active_crtcs;
5513 }
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528 if (state->active_pipe_changes || state->modeset) {
5529 realloc_pipes = ~0;
5530 state->wm_results.dirty_pipes = ~0;
5531 }
5532
5533
5534
5535
5536
5537 for_each_intel_crtc_mask(dev, crtc, realloc_pipes) {
5538 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5539 if (IS_ERR(crtc_state))
5540 return PTR_ERR(crtc_state);
5541 }
5542
5543 return 0;
5544}
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
5569 struct intel_crtc *crtc)
5570{
5571 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5572 const struct intel_crtc_state *old_crtc_state =
5573 intel_atomic_get_old_crtc_state(state, crtc);
5574 struct intel_crtc_state *new_crtc_state =
5575 intel_atomic_get_new_crtc_state(state, crtc);
5576 struct intel_plane *plane;
5577
5578 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5579 struct intel_plane_state *plane_state;
5580 enum plane_id plane_id = plane->id;
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->base) &&
5591 skl_plane_wm_equals(dev_priv,
5592 &old_crtc_state->wm.skl.optimal.planes[plane_id],
5593 &new_crtc_state->wm.skl.optimal.planes[plane_id]))
5594 continue;
5595
5596 plane_state = intel_atomic_get_plane_state(state, plane);
5597 if (IS_ERR(plane_state))
5598 return PTR_ERR(plane_state);
5599
5600 new_crtc_state->update_planes |= BIT(plane_id);
5601 }
5602
5603 return 0;
5604}
5605
5606static int
5607skl_compute_wm(struct intel_atomic_state *state)
5608{
5609 struct intel_crtc *crtc;
5610 struct intel_crtc_state *new_crtc_state;
5611 struct intel_crtc_state *old_crtc_state;
5612 struct skl_ddb_values *results = &state->wm_results;
5613 bool changed = false;
5614 int ret, i;
5615
5616
5617 results->dirty_pipes = 0;
5618
5619 ret = skl_ddb_add_affected_pipes(state, &changed);
5620 if (ret || !changed)
5621 return ret;
5622
5623
5624
5625
5626
5627
5628
5629 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5630 new_crtc_state, i) {
5631 ret = skl_build_pipe_wm(new_crtc_state);
5632 if (ret)
5633 return ret;
5634
5635 ret = skl_wm_add_affected_planes(state, crtc);
5636 if (ret)
5637 return ret;
5638
5639 if (!skl_pipe_wm_equals(crtc,
5640 &old_crtc_state->wm.skl.optimal,
5641 &new_crtc_state->wm.skl.optimal))
5642 results->dirty_pipes |= drm_crtc_mask(&crtc->base);
5643 }
5644
5645 ret = skl_compute_ddb(state);
5646 if (ret)
5647 return ret;
5648
5649 skl_print_wm_changes(state);
5650
5651 return 0;
5652}
5653
5654static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5655 struct intel_crtc_state *cstate)
5656{
5657 struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
5658 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5659 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5660 enum pipe pipe = crtc->pipe;
5661
5662 if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5663 return;
5664
5665 I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5666}
5667
5668static void skl_initial_wm(struct intel_atomic_state *state,
5669 struct intel_crtc_state *cstate)
5670{
5671 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5672 struct drm_device *dev = intel_crtc->base.dev;
5673 struct drm_i915_private *dev_priv = to_i915(dev);
5674 struct skl_ddb_values *results = &state->wm_results;
5675
5676 if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5677 return;
5678
5679 mutex_lock(&dev_priv->wm.wm_mutex);
5680
5681 if (cstate->base.active_changed)
5682 skl_atomic_update_crtc_wm(state, cstate);
5683
5684 mutex_unlock(&dev_priv->wm.wm_mutex);
5685}
5686
5687static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
5688 struct intel_wm_config *config)
5689{
5690 struct intel_crtc *crtc;
5691
5692
5693 for_each_intel_crtc(&dev_priv->drm, crtc) {
5694 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5695
5696 if (!wm->pipe_enabled)
5697 continue;
5698
5699 config->sprites_enabled |= wm->sprites_enabled;
5700 config->sprites_scaled |= wm->sprites_scaled;
5701 config->num_pipes_active++;
5702 }
5703}
5704
5705static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5706{
5707 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5708 struct ilk_wm_maximums max;
5709 struct intel_wm_config config = {};
5710 struct ilk_wm_values results = {};
5711 enum intel_ddb_partitioning partitioning;
5712
5713 ilk_compute_wm_config(dev_priv, &config);
5714
5715 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
5716 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
5717
5718
5719 if (INTEL_GEN(dev_priv) >= 7 &&
5720 config.num_pipes_active == 1 && config.sprites_enabled) {
5721 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
5722 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
5723
5724 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
5725 } else {
5726 best_lp_wm = &lp_wm_1_2;
5727 }
5728
5729 partitioning = (best_lp_wm == &lp_wm_1_2) ?
5730 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5731
5732 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
5733
5734 ilk_write_wm_values(dev_priv, &results);
5735}
5736
5737static void ilk_initial_watermarks(struct intel_atomic_state *state,
5738 struct intel_crtc_state *cstate)
5739{
5740 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5741 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5742
5743 mutex_lock(&dev_priv->wm.wm_mutex);
5744 intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
5745 ilk_program_watermarks(dev_priv);
5746 mutex_unlock(&dev_priv->wm.wm_mutex);
5747}
5748
5749static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5750 struct intel_crtc_state *cstate)
5751{
5752 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5753 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5754
5755 mutex_lock(&dev_priv->wm.wm_mutex);
5756 if (cstate->wm.need_postvbl_update) {
5757 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
5758 ilk_program_watermarks(dev_priv);
5759 }
5760 mutex_unlock(&dev_priv->wm.wm_mutex);
5761}
5762
5763static inline void skl_wm_level_from_reg_val(u32 val,
5764 struct skl_wm_level *level)
5765{
5766 level->plane_en = val & PLANE_WM_EN;
5767 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
5768 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5769 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5770 PLANE_WM_LINES_MASK;
5771}
5772
5773void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
5774 struct skl_pipe_wm *out)
5775{
5776 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5777 enum pipe pipe = crtc->pipe;
5778 int level, max_level;
5779 enum plane_id plane_id;
5780 u32 val;
5781
5782 max_level = ilk_wm_max_level(dev_priv);
5783
5784 for_each_plane_id_on_crtc(crtc, plane_id) {
5785 struct skl_plane_wm *wm = &out->planes[plane_id];
5786
5787 for (level = 0; level <= max_level; level++) {
5788 if (plane_id != PLANE_CURSOR)
5789 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5790 else
5791 val = I915_READ(CUR_WM(pipe, level));
5792
5793 skl_wm_level_from_reg_val(val, &wm->wm[level]);
5794 }
5795
5796 if (plane_id != PLANE_CURSOR)
5797 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5798 else
5799 val = I915_READ(CUR_WM_TRANS(pipe));
5800
5801 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5802 }
5803
5804 if (!crtc->active)
5805 return;
5806
5807 out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5808}
5809
5810void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
5811{
5812 struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5813 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5814 struct intel_crtc *crtc;
5815 struct intel_crtc_state *cstate;
5816
5817 skl_ddb_get_hw_state(dev_priv, ddb);
5818 for_each_intel_crtc(&dev_priv->drm, crtc) {
5819 cstate = to_intel_crtc_state(crtc->base.state);
5820
5821 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
5822
5823 if (crtc->active)
5824 hw->dirty_pipes |= drm_crtc_mask(&crtc->base);
5825 }
5826
5827 if (dev_priv->active_crtcs) {
5828
5829 dev_priv->wm.distrust_bios_wm = true;
5830 }
5831}
5832
5833static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
5834{
5835 struct drm_device *dev = crtc->base.dev;
5836 struct drm_i915_private *dev_priv = to_i915(dev);
5837 struct ilk_wm_values *hw = &dev_priv->wm.hw;
5838 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->base.state);
5839 struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
5840 enum pipe pipe = crtc->pipe;
5841 static const i915_reg_t wm0_pipe_reg[] = {
5842 [PIPE_A] = WM0_PIPEA_ILK,
5843 [PIPE_B] = WM0_PIPEB_ILK,
5844 [PIPE_C] = WM0_PIPEC_IVB,
5845 };
5846
5847 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5848 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5849 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5850
5851 memset(active, 0, sizeof(*active));
5852
5853 active->pipe_enabled = crtc->active;
5854
5855 if (active->pipe_enabled) {
5856 u32 tmp = hw->wm_pipe[pipe];
5857
5858
5859
5860
5861
5862
5863
5864 active->wm[0].enable = true;
5865 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5866 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5867 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5868 active->linetime = hw->wm_linetime[pipe];
5869 } else {
5870 int level, max_level = ilk_wm_max_level(dev_priv);
5871
5872
5873
5874
5875
5876
5877 for (level = 0; level <= max_level; level++)
5878 active->wm[level].enable = true;
5879 }
5880
5881 crtc->wm.active.ilk = *active;
5882}
5883
5884#define _FW_WM(value, plane) \
5885 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5886#define _FW_WM_VLV(value, plane) \
5887 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5888
5889static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5890 struct g4x_wm_values *wm)
5891{
5892 u32 tmp;
5893
5894 tmp = I915_READ(DSPFW1);
5895 wm->sr.plane = _FW_WM(tmp, SR);
5896 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5897 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5898 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5899
5900 tmp = I915_READ(DSPFW2);
5901 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5902 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5903 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5904 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5905 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5906 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5907
5908 tmp = I915_READ(DSPFW3);
5909 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5910 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5911 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5912 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5913}
5914
5915static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5916 struct vlv_wm_values *wm)
5917{
5918 enum pipe pipe;
5919 u32 tmp;
5920
5921 for_each_pipe(dev_priv, pipe) {
5922 tmp = I915_READ(VLV_DDL(pipe));
5923
5924 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5925 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5926 wm->ddl[pipe].plane[PLANE_CURSOR] =
5927 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5928 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5929 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5930 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5931 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5932 }
5933
5934 tmp = I915_READ(DSPFW1);
5935 wm->sr.plane = _FW_WM(tmp, SR);
5936 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5937 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5938 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5939
5940 tmp = I915_READ(DSPFW2);
5941 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5942 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5943 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5944
5945 tmp = I915_READ(DSPFW3);
5946 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5947
5948 if (IS_CHERRYVIEW(dev_priv)) {
5949 tmp = I915_READ(DSPFW7_CHV);
5950 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5951 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5952
5953 tmp = I915_READ(DSPFW8_CHV);
5954 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5955 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5956
5957 tmp = I915_READ(DSPFW9_CHV);
5958 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5959 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5960
5961 tmp = I915_READ(DSPHOWM);
5962 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5963 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5964 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5965 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5966 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5967 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5968 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5969 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5970 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5971 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5972 } else {
5973 tmp = I915_READ(DSPFW7);
5974 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5975 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5976
5977 tmp = I915_READ(DSPHOWM);
5978 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5979 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5980 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5981 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5982 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5983 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5984 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5985 }
5986}
5987
5988#undef _FW_WM
5989#undef _FW_WM_VLV
5990
5991void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
5992{
5993 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5994 struct intel_crtc *crtc;
5995
5996 g4x_read_wm_values(dev_priv, wm);
5997
5998 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5999
6000 for_each_intel_crtc(&dev_priv->drm, crtc) {
6001 struct intel_crtc_state *crtc_state =
6002 to_intel_crtc_state(crtc->base.state);
6003 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6004 struct g4x_pipe_wm *raw;
6005 enum pipe pipe = crtc->pipe;
6006 enum plane_id plane_id;
6007 int level, max_level;
6008
6009 active->cxsr = wm->cxsr;
6010 active->hpll_en = wm->hpll_en;
6011 active->fbc_en = wm->fbc_en;
6012
6013 active->sr = wm->sr;
6014 active->hpll = wm->hpll;
6015
6016 for_each_plane_id_on_crtc(crtc, plane_id) {
6017 active->wm.plane[plane_id] =
6018 wm->pipe[pipe].plane[plane_id];
6019 }
6020
6021 if (wm->cxsr && wm->hpll_en)
6022 max_level = G4X_WM_LEVEL_HPLL;
6023 else if (wm->cxsr)
6024 max_level = G4X_WM_LEVEL_SR;
6025 else
6026 max_level = G4X_WM_LEVEL_NORMAL;
6027
6028 level = G4X_WM_LEVEL_NORMAL;
6029 raw = &crtc_state->wm.g4x.raw[level];
6030 for_each_plane_id_on_crtc(crtc, plane_id)
6031 raw->plane[plane_id] = active->wm.plane[plane_id];
6032
6033 if (++level > max_level)
6034 goto out;
6035
6036 raw = &crtc_state->wm.g4x.raw[level];
6037 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6038 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6039 raw->plane[PLANE_SPRITE0] = 0;
6040 raw->fbc = active->sr.fbc;
6041
6042 if (++level > max_level)
6043 goto out;
6044
6045 raw = &crtc_state->wm.g4x.raw[level];
6046 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6047 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6048 raw->plane[PLANE_SPRITE0] = 0;
6049 raw->fbc = active->hpll.fbc;
6050
6051 out:
6052 for_each_plane_id_on_crtc(crtc, plane_id)
6053 g4x_raw_plane_wm_set(crtc_state, level,
6054 plane_id, USHRT_MAX);
6055 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6056
6057 crtc_state->wm.g4x.optimal = *active;
6058 crtc_state->wm.g4x.intermediate = *active;
6059
6060 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6061 pipe_name(pipe),
6062 wm->pipe[pipe].plane[PLANE_PRIMARY],
6063 wm->pipe[pipe].plane[PLANE_CURSOR],
6064 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6065 }
6066
6067 DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6068 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6069 DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6070 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6071 DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
6072 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6073}
6074
6075void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6076{
6077 struct intel_plane *plane;
6078 struct intel_crtc *crtc;
6079
6080 mutex_lock(&dev_priv->wm.wm_mutex);
6081
6082 for_each_intel_plane(&dev_priv->drm, plane) {
6083 struct intel_crtc *crtc =
6084 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6085 struct intel_crtc_state *crtc_state =
6086 to_intel_crtc_state(crtc->base.state);
6087 struct intel_plane_state *plane_state =
6088 to_intel_plane_state(plane->base.state);
6089 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6090 enum plane_id plane_id = plane->id;
6091 int level;
6092
6093 if (plane_state->base.visible)
6094 continue;
6095
6096 for (level = 0; level < 3; level++) {
6097 struct g4x_pipe_wm *raw =
6098 &crtc_state->wm.g4x.raw[level];
6099
6100 raw->plane[plane_id] = 0;
6101 wm_state->wm.plane[plane_id] = 0;
6102 }
6103
6104 if (plane_id == PLANE_PRIMARY) {
6105 for (level = 0; level < 3; level++) {
6106 struct g4x_pipe_wm *raw =
6107 &crtc_state->wm.g4x.raw[level];
6108 raw->fbc = 0;
6109 }
6110
6111 wm_state->sr.fbc = 0;
6112 wm_state->hpll.fbc = 0;
6113 wm_state->fbc_en = false;
6114 }
6115 }
6116
6117 for_each_intel_crtc(&dev_priv->drm, crtc) {
6118 struct intel_crtc_state *crtc_state =
6119 to_intel_crtc_state(crtc->base.state);
6120
6121 crtc_state->wm.g4x.intermediate =
6122 crtc_state->wm.g4x.optimal;
6123 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6124 }
6125
6126 g4x_program_watermarks(dev_priv);
6127
6128 mutex_unlock(&dev_priv->wm.wm_mutex);
6129}
6130
6131void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6132{
6133 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6134 struct intel_crtc *crtc;
6135 u32 val;
6136
6137 vlv_read_wm_values(dev_priv, wm);
6138
6139 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6140 wm->level = VLV_WM_LEVEL_PM2;
6141
6142 if (IS_CHERRYVIEW(dev_priv)) {
6143 vlv_punit_get(dev_priv);
6144
6145 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6146 if (val & DSP_MAXFIFO_PM5_ENABLE)
6147 wm->level = VLV_WM_LEVEL_PM5;
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6159 val |= FORCE_DDR_FREQ_REQ_ACK;
6160 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6161
6162 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6163 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6164 DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
6165 "assuming DDR DVFS is disabled\n");
6166 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6167 } else {
6168 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6169 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6170 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6171 }
6172
6173 vlv_punit_put(dev_priv);
6174 }
6175
6176 for_each_intel_crtc(&dev_priv->drm, crtc) {
6177 struct intel_crtc_state *crtc_state =
6178 to_intel_crtc_state(crtc->base.state);
6179 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6180 const struct vlv_fifo_state *fifo_state =
6181 &crtc_state->wm.vlv.fifo_state;
6182 enum pipe pipe = crtc->pipe;
6183 enum plane_id plane_id;
6184 int level;
6185
6186 vlv_get_fifo_size(crtc_state);
6187
6188 active->num_levels = wm->level + 1;
6189 active->cxsr = wm->cxsr;
6190
6191 for (level = 0; level < active->num_levels; level++) {
6192 struct g4x_pipe_wm *raw =
6193 &crtc_state->wm.vlv.raw[level];
6194
6195 active->sr[level].plane = wm->sr.plane;
6196 active->sr[level].cursor = wm->sr.cursor;
6197
6198 for_each_plane_id_on_crtc(crtc, plane_id) {
6199 active->wm[level].plane[plane_id] =
6200 wm->pipe[pipe].plane[plane_id];
6201
6202 raw->plane[plane_id] =
6203 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6204 fifo_state->plane[plane_id]);
6205 }
6206 }
6207
6208 for_each_plane_id_on_crtc(crtc, plane_id)
6209 vlv_raw_plane_wm_set(crtc_state, level,
6210 plane_id, USHRT_MAX);
6211 vlv_invalidate_wms(crtc, active, level);
6212
6213 crtc_state->wm.vlv.optimal = *active;
6214 crtc_state->wm.vlv.intermediate = *active;
6215
6216 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6217 pipe_name(pipe),
6218 wm->pipe[pipe].plane[PLANE_PRIMARY],
6219 wm->pipe[pipe].plane[PLANE_CURSOR],
6220 wm->pipe[pipe].plane[PLANE_SPRITE0],
6221 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6222 }
6223
6224 DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6225 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6226}
6227
6228void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6229{
6230 struct intel_plane *plane;
6231 struct intel_crtc *crtc;
6232
6233 mutex_lock(&dev_priv->wm.wm_mutex);
6234
6235 for_each_intel_plane(&dev_priv->drm, plane) {
6236 struct intel_crtc *crtc =
6237 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6238 struct intel_crtc_state *crtc_state =
6239 to_intel_crtc_state(crtc->base.state);
6240 struct intel_plane_state *plane_state =
6241 to_intel_plane_state(plane->base.state);
6242 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6243 const struct vlv_fifo_state *fifo_state =
6244 &crtc_state->wm.vlv.fifo_state;
6245 enum plane_id plane_id = plane->id;
6246 int level;
6247
6248 if (plane_state->base.visible)
6249 continue;
6250
6251 for (level = 0; level < wm_state->num_levels; level++) {
6252 struct g4x_pipe_wm *raw =
6253 &crtc_state->wm.vlv.raw[level];
6254
6255 raw->plane[plane_id] = 0;
6256
6257 wm_state->wm[level].plane[plane_id] =
6258 vlv_invert_wm_value(raw->plane[plane_id],
6259 fifo_state->plane[plane_id]);
6260 }
6261 }
6262
6263 for_each_intel_crtc(&dev_priv->drm, crtc) {
6264 struct intel_crtc_state *crtc_state =
6265 to_intel_crtc_state(crtc->base.state);
6266
6267 crtc_state->wm.vlv.intermediate =
6268 crtc_state->wm.vlv.optimal;
6269 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6270 }
6271
6272 vlv_program_watermarks(dev_priv);
6273
6274 mutex_unlock(&dev_priv->wm.wm_mutex);
6275}
6276
6277
6278
6279
6280
6281static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6282{
6283 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6284 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6285 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6286
6287
6288
6289
6290
6291}
6292
6293void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6294{
6295 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6296 struct intel_crtc *crtc;
6297
6298 ilk_init_lp_watermarks(dev_priv);
6299
6300 for_each_intel_crtc(&dev_priv->drm, crtc)
6301 ilk_pipe_wm_get_hw_state(crtc);
6302
6303 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6304 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6305 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6306
6307 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6308 if (INTEL_GEN(dev_priv) >= 7) {
6309 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6310 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6311 }
6312
6313 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6314 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6315 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6316 else if (IS_IVYBRIDGE(dev_priv))
6317 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6318 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6319
6320 hw->enable_fbc_wm =
6321 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6322}
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357void intel_update_watermarks(struct intel_crtc *crtc)
6358{
6359 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6360
6361 if (dev_priv->display.update_wm)
6362 dev_priv->display.update_wm(crtc);
6363}
6364
6365void intel_enable_ipc(struct drm_i915_private *dev_priv)
6366{
6367 u32 val;
6368
6369 if (!HAS_IPC(dev_priv))
6370 return;
6371
6372 val = I915_READ(DISP_ARB_CTL2);
6373
6374 if (dev_priv->ipc_enabled)
6375 val |= DISP_IPC_ENABLE;
6376 else
6377 val &= ~DISP_IPC_ENABLE;
6378
6379 I915_WRITE(DISP_ARB_CTL2, val);
6380}
6381
6382static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6383{
6384
6385 if (IS_SKYLAKE(dev_priv))
6386 return false;
6387
6388
6389 if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
6390 return dev_priv->dram_info.symmetric_memory;
6391
6392 return true;
6393}
6394
6395void intel_init_ipc(struct drm_i915_private *dev_priv)
6396{
6397 if (!HAS_IPC(dev_priv))
6398 return;
6399
6400 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6401
6402 intel_enable_ipc(dev_priv);
6403}
6404
6405
6406
6407
6408DEFINE_SPINLOCK(mchdev_lock);
6409
6410bool ironlake_set_drps(struct drm_i915_private *i915, u8 val)
6411{
6412 struct intel_uncore *uncore = &i915->uncore;
6413 u16 rgvswctl;
6414
6415 lockdep_assert_held(&mchdev_lock);
6416
6417 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
6418 if (rgvswctl & MEMCTL_CMD_STS) {
6419 DRM_DEBUG("gpu busy, RCS change rejected\n");
6420 return false;
6421 }
6422
6423 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
6424 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
6425 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
6426 intel_uncore_posting_read16(uncore, MEMSWCTL);
6427
6428 rgvswctl |= MEMCTL_CMD_STS;
6429 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
6430
6431 return true;
6432}
6433
6434static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
6435{
6436 struct intel_uncore *uncore = &dev_priv->uncore;
6437 u32 rgvmodectl;
6438 u8 fmax, fmin, fstart, vstart;
6439
6440 spin_lock_irq(&mchdev_lock);
6441
6442 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
6443
6444
6445 intel_uncore_write16(uncore, PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
6446 intel_uncore_write16(uncore, TSC1, I915_READ(TSC1) | TSE);
6447
6448
6449 intel_uncore_write(uncore, RCUPEI, 100000);
6450 intel_uncore_write(uncore, RCDNEI, 100000);
6451
6452
6453 intel_uncore_write(uncore, RCBMAXAVG, 90000);
6454 intel_uncore_write(uncore, RCBMINAVG, 80000);
6455
6456 intel_uncore_write(uncore, MEMIHYST, 1);
6457
6458
6459 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
6460 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
6461 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
6462 MEMMODE_FSTART_SHIFT;
6463
6464 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
6465 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
6466
6467 dev_priv->ips.fmax = fmax;
6468 dev_priv->ips.fstart = fstart;
6469
6470 dev_priv->ips.max_delay = fstart;
6471 dev_priv->ips.min_delay = fmin;
6472 dev_priv->ips.cur_delay = fstart;
6473
6474 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
6475 fmax, fmin, fstart);
6476
6477 intel_uncore_write(uncore,
6478 MEMINTREN,
6479 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
6480
6481
6482
6483
6484
6485 intel_uncore_write(uncore, VIDSTART, vstart);
6486 intel_uncore_posting_read(uncore, VIDSTART);
6487
6488 rgvmodectl |= MEMMODE_SWMODE_EN;
6489 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
6490
6491 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
6492 MEMCTL_CMD_STS) == 0, 10))
6493 DRM_ERROR("stuck trying to change perf mode\n");
6494 mdelay(1);
6495
6496 ironlake_set_drps(dev_priv, fstart);
6497
6498 dev_priv->ips.last_count1 =
6499 intel_uncore_read(uncore, DMIEC) +
6500 intel_uncore_read(uncore, DDREC) +
6501 intel_uncore_read(uncore, CSIEC);
6502 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6503 dev_priv->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
6504 dev_priv->ips.last_time2 = ktime_get_raw_ns();
6505
6506 spin_unlock_irq(&mchdev_lock);
6507}
6508
6509static void ironlake_disable_drps(struct drm_i915_private *i915)
6510{
6511 struct intel_uncore *uncore = &i915->uncore;
6512 u16 rgvswctl;
6513
6514 spin_lock_irq(&mchdev_lock);
6515
6516 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
6517
6518
6519 intel_uncore_write(uncore,
6520 MEMINTREN,
6521 intel_uncore_read(uncore, MEMINTREN) &
6522 ~MEMINT_EVAL_CHG_EN);
6523 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
6524 intel_uncore_write(uncore,
6525 DEIER,
6526 intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
6527 intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
6528 intel_uncore_write(uncore,
6529 DEIMR,
6530 intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
6531
6532
6533 ironlake_set_drps(i915, i915->ips.fstart);
6534 mdelay(1);
6535 rgvswctl |= MEMCTL_CMD_STS;
6536 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
6537 mdelay(1);
6538
6539 spin_unlock_irq(&mchdev_lock);
6540}
6541
6542
6543
6544
6545
6546
6547static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6548{
6549 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6550 u32 limits;
6551
6552
6553
6554
6555
6556
6557
6558 if (INTEL_GEN(dev_priv) >= 9) {
6559 limits = (rps->max_freq_softlimit) << 23;
6560 if (val <= rps->min_freq_softlimit)
6561 limits |= (rps->min_freq_softlimit) << 14;
6562 } else {
6563 limits = rps->max_freq_softlimit << 24;
6564 if (val <= rps->min_freq_softlimit)
6565 limits |= rps->min_freq_softlimit << 16;
6566 }
6567
6568 return limits;
6569}
6570
6571static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
6572{
6573 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6574 u32 threshold_up = 0, threshold_down = 0;
6575 u32 ei_up = 0, ei_down = 0;
6576
6577 lockdep_assert_held(&rps->power.mutex);
6578
6579 if (new_power == rps->power.mode)
6580 return;
6581
6582
6583 switch (new_power) {
6584 case LOW_POWER:
6585
6586 ei_up = 16000;
6587 threshold_up = 95;
6588
6589
6590 ei_down = 32000;
6591 threshold_down = 85;
6592 break;
6593
6594 case BETWEEN:
6595
6596 ei_up = 13000;
6597 threshold_up = 90;
6598
6599
6600 ei_down = 32000;
6601 threshold_down = 75;
6602 break;
6603
6604 case HIGH_POWER:
6605
6606 ei_up = 10000;
6607 threshold_up = 85;
6608
6609
6610 ei_down = 32000;
6611 threshold_down = 60;
6612 break;
6613 }
6614
6615
6616
6617
6618 if (IS_VALLEYVIEW(dev_priv))
6619 goto skip_hw_write;
6620
6621 I915_WRITE(GEN6_RP_UP_EI,
6622 GT_INTERVAL_FROM_US(dev_priv, ei_up));
6623 I915_WRITE(GEN6_RP_UP_THRESHOLD,
6624 GT_INTERVAL_FROM_US(dev_priv,
6625 ei_up * threshold_up / 100));
6626
6627 I915_WRITE(GEN6_RP_DOWN_EI,
6628 GT_INTERVAL_FROM_US(dev_priv, ei_down));
6629 I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6630 GT_INTERVAL_FROM_US(dev_priv,
6631 ei_down * threshold_down / 100));
6632
6633 I915_WRITE(GEN6_RP_CONTROL,
6634 (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
6635 GEN6_RP_MEDIA_HW_NORMAL_MODE |
6636 GEN6_RP_MEDIA_IS_GFX |
6637 GEN6_RP_ENABLE |
6638 GEN6_RP_UP_BUSY_AVG |
6639 GEN6_RP_DOWN_IDLE_AVG);
6640
6641skip_hw_write:
6642 rps->power.mode = new_power;
6643 rps->power.up_threshold = threshold_up;
6644 rps->power.down_threshold = threshold_down;
6645}
6646
6647static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6648{
6649 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6650 int new_power;
6651
6652 new_power = rps->power.mode;
6653 switch (rps->power.mode) {
6654 case LOW_POWER:
6655 if (val > rps->efficient_freq + 1 &&
6656 val > rps->cur_freq)
6657 new_power = BETWEEN;
6658 break;
6659
6660 case BETWEEN:
6661 if (val <= rps->efficient_freq &&
6662 val < rps->cur_freq)
6663 new_power = LOW_POWER;
6664 else if (val >= rps->rp0_freq &&
6665 val > rps->cur_freq)
6666 new_power = HIGH_POWER;
6667 break;
6668
6669 case HIGH_POWER:
6670 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6671 val < rps->cur_freq)
6672 new_power = BETWEEN;
6673 break;
6674 }
6675
6676 if (val <= rps->min_freq_softlimit)
6677 new_power = LOW_POWER;
6678 if (val >= rps->max_freq_softlimit)
6679 new_power = HIGH_POWER;
6680
6681 mutex_lock(&rps->power.mutex);
6682 if (rps->power.interactive)
6683 new_power = HIGH_POWER;
6684 rps_set_power(dev_priv, new_power);
6685 mutex_unlock(&rps->power.mutex);
6686}
6687
6688void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
6689{
6690 struct intel_rps *rps = &i915->gt_pm.rps;
6691
6692 if (INTEL_GEN(i915) < 6)
6693 return;
6694
6695 mutex_lock(&rps->power.mutex);
6696 if (interactive) {
6697 if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
6698 rps_set_power(i915, HIGH_POWER);
6699 } else {
6700 GEM_BUG_ON(!rps->power.interactive);
6701 rps->power.interactive--;
6702 }
6703 mutex_unlock(&rps->power.mutex);
6704}
6705
6706static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6707{
6708 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6709 u32 mask = 0;
6710
6711
6712 if (val > rps->min_freq_softlimit)
6713 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6714 if (val < rps->max_freq_softlimit)
6715 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6716
6717 mask &= dev_priv->pm_rps_events;
6718
6719 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6720}
6721
6722
6723
6724
6725static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6726{
6727 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6728
6729
6730
6731
6732 if (val != rps->cur_freq) {
6733 gen6_set_rps_thresholds(dev_priv, val);
6734
6735 if (INTEL_GEN(dev_priv) >= 9)
6736 I915_WRITE(GEN6_RPNSWREQ,
6737 GEN9_FREQUENCY(val));
6738 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6739 I915_WRITE(GEN6_RPNSWREQ,
6740 HSW_FREQUENCY(val));
6741 else
6742 I915_WRITE(GEN6_RPNSWREQ,
6743 GEN6_FREQUENCY(val) |
6744 GEN6_OFFSET(0) |
6745 GEN6_AGGRESSIVE_TURBO);
6746 }
6747
6748
6749
6750
6751 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6752 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6753
6754 rps->cur_freq = val;
6755 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6756
6757 return 0;
6758}
6759
6760static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6761{
6762 int err;
6763
6764 if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6765 "Odd GPU freq value\n"))
6766 val &= ~1;
6767
6768 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6769
6770 if (val != dev_priv->gt_pm.rps.cur_freq) {
6771 vlv_punit_get(dev_priv);
6772 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6773 vlv_punit_put(dev_priv);
6774 if (err)
6775 return err;
6776
6777 gen6_set_rps_thresholds(dev_priv, val);
6778 }
6779
6780 dev_priv->gt_pm.rps.cur_freq = val;
6781 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6782
6783 return 0;
6784}
6785
6786
6787
6788
6789
6790
6791
6792
6793static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6794{
6795 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6796 u32 val = rps->idle_freq;
6797 int err;
6798
6799 if (rps->cur_freq <= val)
6800 return;
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA);
6815 err = valleyview_set_rps(dev_priv, val);
6816 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA);
6817
6818 if (err)
6819 DRM_ERROR("Failed to set RPS for idle\n");
6820}
6821
6822void gen6_rps_busy(struct drm_i915_private *dev_priv)
6823{
6824 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6825
6826 mutex_lock(&rps->lock);
6827 if (rps->enabled) {
6828 u8 freq;
6829
6830 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6831 gen6_rps_reset_ei(dev_priv);
6832 I915_WRITE(GEN6_PMINTRMSK,
6833 gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6834
6835 gen6_enable_rps_interrupts(dev_priv);
6836
6837
6838
6839
6840 freq = max(rps->cur_freq,
6841 rps->efficient_freq);
6842
6843 if (intel_set_rps(dev_priv,
6844 clamp(freq,
6845 rps->min_freq_softlimit,
6846 rps->max_freq_softlimit)))
6847 DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6848 }
6849 mutex_unlock(&rps->lock);
6850}
6851
6852void gen6_rps_idle(struct drm_i915_private *dev_priv)
6853{
6854 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6855
6856
6857
6858
6859
6860
6861 gen6_disable_rps_interrupts(dev_priv);
6862
6863 mutex_lock(&rps->lock);
6864 if (rps->enabled) {
6865 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6866 vlv_set_rps_idle(dev_priv);
6867 else
6868 gen6_set_rps(dev_priv, rps->idle_freq);
6869 rps->last_adj = 0;
6870 I915_WRITE(GEN6_PMINTRMSK,
6871 gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6872 }
6873 mutex_unlock(&rps->lock);
6874}
6875
6876void gen6_rps_boost(struct i915_request *rq)
6877{
6878 struct intel_rps *rps = &rq->i915->gt_pm.rps;
6879 unsigned long flags;
6880 bool boost;
6881
6882
6883
6884
6885 if (!rps->enabled)
6886 return;
6887
6888 if (i915_request_signaled(rq))
6889 return;
6890
6891
6892 boost = false;
6893 spin_lock_irqsave(&rq->lock, flags);
6894 if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) {
6895 boost = !atomic_fetch_inc(&rps->num_waiters);
6896 rq->waitboost = true;
6897 }
6898 spin_unlock_irqrestore(&rq->lock, flags);
6899 if (!boost)
6900 return;
6901
6902 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6903 schedule_work(&rps->work);
6904
6905 atomic_inc(&rps->boosts);
6906}
6907
6908int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6909{
6910 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6911 int err;
6912
6913 lockdep_assert_held(&rps->lock);
6914 GEM_BUG_ON(val > rps->max_freq);
6915 GEM_BUG_ON(val < rps->min_freq);
6916
6917 if (!rps->enabled) {
6918 rps->cur_freq = val;
6919 return 0;
6920 }
6921
6922 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6923 err = valleyview_set_rps(dev_priv, val);
6924 else
6925 err = gen6_set_rps(dev_priv, val);
6926
6927 return err;
6928}
6929
6930static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6931{
6932 I915_WRITE(GEN6_RC_CONTROL, 0);
6933 I915_WRITE(GEN9_PG_ENABLE, 0);
6934}
6935
6936static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6937{
6938 I915_WRITE(GEN6_RP_CONTROL, 0);
6939}
6940
6941static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6942{
6943 I915_WRITE(GEN6_RC_CONTROL, 0);
6944}
6945
6946static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6947{
6948 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6949 I915_WRITE(GEN6_RP_CONTROL, 0);
6950}
6951
6952static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6953{
6954 I915_WRITE(GEN6_RC_CONTROL, 0);
6955}
6956
6957static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6958{
6959 I915_WRITE(GEN6_RP_CONTROL, 0);
6960}
6961
6962static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6963{
6964
6965
6966 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
6967
6968 I915_WRITE(GEN6_RC_CONTROL, 0);
6969
6970 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
6971}
6972
6973static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6974{
6975 I915_WRITE(GEN6_RP_CONTROL, 0);
6976}
6977
6978static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6979{
6980 bool enable_rc6 = true;
6981 unsigned long rc6_ctx_base;
6982 u32 rc_ctl;
6983 int rc_sw_target;
6984
6985 rc_ctl = I915_READ(GEN6_RC_CONTROL);
6986 rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6987 RC_SW_TARGET_STATE_SHIFT;
6988 DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6989 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6990 onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6991 onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6992 rc_sw_target);
6993
6994 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6995 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6996 enable_rc6 = false;
6997 }
6998
6999
7000
7001
7002
7003 rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
7004 if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
7005 (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
7006 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
7007 enable_rc6 = false;
7008 }
7009
7010 if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
7011 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
7012 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
7013 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
7014 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
7015 enable_rc6 = false;
7016 }
7017
7018 if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
7019 !I915_READ(GEN8_PUSHBUS_ENABLE) ||
7020 !I915_READ(GEN8_PUSHBUS_SHIFT)) {
7021 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
7022 enable_rc6 = false;
7023 }
7024
7025 if (!I915_READ(GEN6_GFXPAUSE)) {
7026 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
7027 enable_rc6 = false;
7028 }
7029
7030 if (!I915_READ(GEN8_MISC_CTRL0)) {
7031 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
7032 enable_rc6 = false;
7033 }
7034
7035 return enable_rc6;
7036}
7037
7038static bool sanitize_rc6(struct drm_i915_private *i915)
7039{
7040 struct intel_device_info *info = mkwrite_device_info(i915);
7041
7042
7043 if (intel_vgpu_active(i915)) {
7044 info->has_rc6 = 0;
7045 info->has_rps = false;
7046 }
7047
7048 if (info->has_rc6 &&
7049 IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
7050 DRM_INFO("RC6 disabled by BIOS\n");
7051 info->has_rc6 = 0;
7052 }
7053
7054
7055
7056
7057
7058
7059
7060 if (!info->has_rc6 && info->has_rc6p)
7061 info->has_rc6p = 0;
7062
7063 return info->has_rc6;
7064}
7065
7066static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
7067{
7068 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7069
7070
7071
7072
7073 if (IS_GEN9_LP(dev_priv)) {
7074 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
7075 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
7076 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
7077 rps->min_freq = (rp_state_cap >> 0) & 0xff;
7078 } else {
7079 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
7080 rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
7081 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
7082 rps->min_freq = (rp_state_cap >> 16) & 0xff;
7083 }
7084
7085 rps->max_freq = rps->rp0_freq;
7086
7087 rps->efficient_freq = rps->rp1_freq;
7088 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
7089 IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7090 u32 ddcc_status = 0;
7091
7092 if (sandybridge_pcode_read(dev_priv,
7093 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
7094 &ddcc_status, NULL) == 0)
7095 rps->efficient_freq =
7096 clamp_t(u8,
7097 ((ddcc_status >> 8) & 0xff),
7098 rps->min_freq,
7099 rps->max_freq);
7100 }
7101
7102 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7103
7104
7105
7106 rps->rp0_freq *= GEN9_FREQ_SCALER;
7107 rps->rp1_freq *= GEN9_FREQ_SCALER;
7108 rps->min_freq *= GEN9_FREQ_SCALER;
7109 rps->max_freq *= GEN9_FREQ_SCALER;
7110 rps->efficient_freq *= GEN9_FREQ_SCALER;
7111 }
7112}
7113
7114static void reset_rps(struct drm_i915_private *dev_priv,
7115 int (*set)(struct drm_i915_private *, u8))
7116{
7117 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7118 u8 freq = rps->cur_freq;
7119
7120
7121 rps->power.mode = -1;
7122 rps->cur_freq = -1;
7123
7124 if (set(dev_priv, freq))
7125 DRM_ERROR("Failed to reset RPS to initial values\n");
7126}
7127
7128
7129static void gen9_enable_rps(struct drm_i915_private *dev_priv)
7130{
7131 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7132
7133
7134 if (IS_GEN(dev_priv, 9))
7135 I915_WRITE(GEN6_RC_VIDEO_FREQ,
7136 GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
7137
7138
7139 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
7140 GT_INTERVAL_FROM_US(dev_priv, 1000000));
7141
7142 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
7143
7144
7145
7146
7147 reset_rps(dev_priv, gen6_set_rps);
7148
7149 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7150}
7151
7152static void gen11_enable_rc6(struct drm_i915_private *dev_priv)
7153{
7154 struct intel_engine_cs *engine;
7155 enum intel_engine_id id;
7156
7157
7158 I915_WRITE(GEN6_RC_STATE, 0);
7159
7160
7161
7162
7163
7164 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7165
7166
7167 I915_WRITE(GEN6_RC_CONTROL, 0);
7168
7169
7170 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
7171 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
7172
7173 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7174 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7175 for_each_engine(engine, dev_priv, id)
7176 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7177
7178 if (HAS_GUC(dev_priv))
7179 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
7180
7181 I915_WRITE(GEN6_RC_SLEEP, 0);
7182
7183 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
7207 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
7208
7209
7210 I915_WRITE(GEN6_RC_CONTROL,
7211 GEN6_RC_CTL_HW_ENABLE |
7212 GEN6_RC_CTL_RC6_ENABLE |
7213 GEN6_RC_CTL_EI_MODE(1));
7214
7215
7216 I915_WRITE(GEN9_PG_ENABLE,
7217 GEN9_RENDER_PG_ENABLE |
7218 GEN9_MEDIA_PG_ENABLE |
7219 GEN11_MEDIA_SAMPLER_PG_ENABLE);
7220
7221 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7222}
7223
7224static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
7225{
7226 struct intel_engine_cs *engine;
7227 enum intel_engine_id id;
7228 u32 rc6_mode;
7229
7230
7231 I915_WRITE(GEN6_RC_STATE, 0);
7232
7233
7234
7235 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7236
7237
7238 I915_WRITE(GEN6_RC_CONTROL, 0);
7239
7240
7241 if (INTEL_GEN(dev_priv) >= 10) {
7242 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
7243 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
7244 } else if (IS_SKYLAKE(dev_priv)) {
7245
7246
7247
7248
7249 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
7250 } else {
7251 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
7252 }
7253
7254 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7255 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7256 for_each_engine(engine, dev_priv, id)
7257 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7258
7259 if (HAS_GUC(dev_priv))
7260 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
7261
7262 I915_WRITE(GEN6_RC_SLEEP, 0);
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
7286 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
7287
7288
7289 I915_WRITE(GEN6_RC6_THRESHOLD, 37500);
7290
7291
7292 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
7293 rc6_mode = GEN7_RC_CTL_TO_MODE;
7294 else
7295 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
7296
7297 I915_WRITE(GEN6_RC_CONTROL,
7298 GEN6_RC_CTL_HW_ENABLE |
7299 GEN6_RC_CTL_RC6_ENABLE |
7300 rc6_mode);
7301
7302
7303
7304
7305
7306 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
7307 I915_WRITE(GEN9_PG_ENABLE, 0);
7308 else
7309 I915_WRITE(GEN9_PG_ENABLE,
7310 GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
7311
7312 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7313}
7314
7315static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
7316{
7317 struct intel_engine_cs *engine;
7318 enum intel_engine_id id;
7319
7320
7321 I915_WRITE(GEN6_RC_STATE, 0);
7322
7323
7324
7325 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7326
7327
7328 I915_WRITE(GEN6_RC_CONTROL, 0);
7329
7330
7331 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7332 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7333 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7334 for_each_engine(engine, dev_priv, id)
7335 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7336 I915_WRITE(GEN6_RC_SLEEP, 0);
7337 I915_WRITE(GEN6_RC6_THRESHOLD, 625);
7338
7339
7340
7341 I915_WRITE(GEN6_RC_CONTROL,
7342 GEN6_RC_CTL_HW_ENABLE |
7343 GEN7_RC_CTL_TO_MODE |
7344 GEN6_RC_CTL_RC6_ENABLE);
7345
7346 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7347}
7348
7349static void gen8_enable_rps(struct drm_i915_private *dev_priv)
7350{
7351 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7352
7353 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7354
7355
7356 I915_WRITE(GEN6_RPNSWREQ,
7357 HSW_FREQUENCY(rps->rp1_freq));
7358 I915_WRITE(GEN6_RC_VIDEO_FREQ,
7359 HSW_FREQUENCY(rps->rp1_freq));
7360
7361 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128);
7362
7363
7364 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
7365 rps->max_freq_softlimit << 24 |
7366 rps->min_freq_softlimit << 16);
7367
7368 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128);
7369 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128);
7370 I915_WRITE(GEN6_RP_UP_EI, 66000);
7371 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7372
7373 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7374
7375
7376 I915_WRITE(GEN6_RP_CONTROL,
7377 GEN6_RP_MEDIA_TURBO |
7378 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7379 GEN6_RP_MEDIA_IS_GFX |
7380 GEN6_RP_ENABLE |
7381 GEN6_RP_UP_BUSY_AVG |
7382 GEN6_RP_DOWN_IDLE_AVG);
7383
7384 reset_rps(dev_priv, gen6_set_rps);
7385
7386 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7387}
7388
7389static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
7390{
7391 struct intel_engine_cs *engine;
7392 enum intel_engine_id id;
7393 u32 rc6vids, rc6_mask;
7394 u32 gtfifodbg;
7395 int ret;
7396
7397 I915_WRITE(GEN6_RC_STATE, 0);
7398
7399
7400 gtfifodbg = I915_READ(GTFIFODBG);
7401 if (gtfifodbg) {
7402 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
7403 I915_WRITE(GTFIFODBG, gtfifodbg);
7404 }
7405
7406 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7407
7408
7409 I915_WRITE(GEN6_RC_CONTROL, 0);
7410
7411 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7412 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7413 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7414 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7415 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7416
7417 for_each_engine(engine, dev_priv, id)
7418 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7419
7420 I915_WRITE(GEN6_RC_SLEEP, 0);
7421 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7422 if (IS_IVYBRIDGE(dev_priv))
7423 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
7424 else
7425 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7426 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
7427 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000);
7428
7429
7430 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7431 if (HAS_RC6p(dev_priv))
7432 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
7433 if (HAS_RC6pp(dev_priv))
7434 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
7435 I915_WRITE(GEN6_RC_CONTROL,
7436 rc6_mask |
7437 GEN6_RC_CTL_EI_MODE(1) |
7438 GEN6_RC_CTL_HW_ENABLE);
7439
7440 rc6vids = 0;
7441 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS,
7442 &rc6vids, NULL);
7443 if (IS_GEN(dev_priv, 6) && ret) {
7444 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
7445 } else if (IS_GEN(dev_priv, 6) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
7446 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
7447 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
7448 rc6vids &= 0xffff00;
7449 rc6vids |= GEN6_ENCODE_RC6_VID(450);
7450 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
7451 if (ret)
7452 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
7453 }
7454
7455 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7456}
7457
7458static void gen6_enable_rps(struct drm_i915_private *dev_priv)
7459{
7460
7461
7462
7463
7464
7465
7466 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7467
7468
7469 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
7470 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7471
7472 reset_rps(dev_priv, gen6_set_rps);
7473
7474 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7475}
7476
7477static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7478{
7479 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7480 const int min_freq = 15;
7481 const int scaling_factor = 180;
7482 unsigned int gpu_freq;
7483 unsigned int max_ia_freq, min_ring_freq;
7484 unsigned int max_gpu_freq, min_gpu_freq;
7485 struct cpufreq_policy *policy;
7486
7487 lockdep_assert_held(&rps->lock);
7488
7489 if (rps->max_freq <= rps->min_freq)
7490 return;
7491
7492 policy = cpufreq_cpu_get(0);
7493 if (policy) {
7494 max_ia_freq = policy->cpuinfo.max_freq;
7495 cpufreq_cpu_put(policy);
7496 } else {
7497
7498
7499
7500
7501 max_ia_freq = tsc_khz;
7502 }
7503
7504
7505 max_ia_freq /= 1000;
7506
7507 min_ring_freq = I915_READ(DCLK) & 0xf;
7508
7509 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
7510
7511 min_gpu_freq = rps->min_freq;
7512 max_gpu_freq = rps->max_freq;
7513 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7514
7515 min_gpu_freq /= GEN9_FREQ_SCALER;
7516 max_gpu_freq /= GEN9_FREQ_SCALER;
7517 }
7518
7519
7520
7521
7522
7523
7524 for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
7525 const int diff = max_gpu_freq - gpu_freq;
7526 unsigned int ia_freq = 0, ring_freq = 0;
7527
7528 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7529
7530
7531
7532
7533 ring_freq = gpu_freq;
7534 } else if (INTEL_GEN(dev_priv) >= 8) {
7535
7536 ring_freq = max(min_ring_freq, gpu_freq);
7537 } else if (IS_HASWELL(dev_priv)) {
7538 ring_freq = mult_frac(gpu_freq, 5, 4);
7539 ring_freq = max(min_ring_freq, ring_freq);
7540
7541 } else {
7542
7543
7544
7545
7546
7547
7548
7549 if (gpu_freq < min_freq)
7550 ia_freq = 800;
7551 else
7552 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7553 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7554 }
7555
7556 sandybridge_pcode_write(dev_priv,
7557 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
7558 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
7559 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
7560 gpu_freq);
7561 }
7562}
7563
7564static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
7565{
7566 u32 val, rp0;
7567
7568 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7569
7570 switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) {
7571 case 8:
7572
7573 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
7574 break;
7575 case 12:
7576
7577 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
7578 break;
7579 case 16:
7580
7581 default:
7582
7583 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
7584 break;
7585 }
7586
7587 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
7588
7589 return rp0;
7590}
7591
7592static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7593{
7594 u32 val, rpe;
7595
7596 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
7597 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
7598
7599 return rpe;
7600}
7601
7602static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
7603{
7604 u32 val, rp1;
7605
7606 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7607 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
7608
7609 return rp1;
7610}
7611
7612static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
7613{
7614 u32 val, rpn;
7615
7616 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
7617 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
7618 FB_GFX_FREQ_FUSE_MASK);
7619
7620 return rpn;
7621}
7622
7623static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
7624{
7625 u32 val, rp1;
7626
7627 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7628
7629 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
7630
7631 return rp1;
7632}
7633
7634static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
7635{
7636 u32 val, rp0;
7637
7638 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7639
7640 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
7641
7642 rp0 = min_t(u32, rp0, 0xea);
7643
7644 return rp0;
7645}
7646
7647static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7648{
7649 u32 val, rpe;
7650
7651 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
7652 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
7653 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
7654 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
7655
7656 return rpe;
7657}
7658
7659static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7660{
7661 u32 val;
7662
7663 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7664
7665
7666
7667
7668
7669
7670
7671 return max_t(u32, val, 0xc0);
7672}
7673
7674
7675static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7676{
7677 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7678
7679 WARN_ON(pctx_addr != dev_priv->dsm.start +
7680 dev_priv->vlv_pctx->stolen->start);
7681}
7682
7683
7684
7685static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7686{
7687 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7688
7689 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7690}
7691
7692static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7693{
7694 resource_size_t pctx_paddr, paddr;
7695 resource_size_t pctx_size = 32*1024;
7696 u32 pcbr;
7697
7698 pcbr = I915_READ(VLV_PCBR);
7699 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7700 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7701 paddr = dev_priv->dsm.end + 1 - pctx_size;
7702 GEM_BUG_ON(paddr > U32_MAX);
7703
7704 pctx_paddr = (paddr & (~4095));
7705 I915_WRITE(VLV_PCBR, pctx_paddr);
7706 }
7707
7708 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7709}
7710
7711static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7712{
7713 struct drm_i915_gem_object *pctx;
7714 resource_size_t pctx_paddr;
7715 resource_size_t pctx_size = 24*1024;
7716 u32 pcbr;
7717
7718 pcbr = I915_READ(VLV_PCBR);
7719 if (pcbr) {
7720
7721 resource_size_t pcbr_offset;
7722
7723 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7724 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7725 pcbr_offset,
7726 I915_GTT_OFFSET_NONE,
7727 pctx_size);
7728 goto out;
7729 }
7730
7731 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741 pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7742 if (!pctx) {
7743 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7744 goto out;
7745 }
7746
7747 GEM_BUG_ON(range_overflows_t(u64,
7748 dev_priv->dsm.start,
7749 pctx->stolen->start,
7750 U32_MAX));
7751 pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7752 I915_WRITE(VLV_PCBR, pctx_paddr);
7753
7754out:
7755 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7756 dev_priv->vlv_pctx = pctx;
7757}
7758
7759static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7760{
7761 struct drm_i915_gem_object *pctx;
7762
7763 pctx = fetch_and_zero(&dev_priv->vlv_pctx);
7764 if (pctx)
7765 i915_gem_object_put(pctx);
7766}
7767
7768static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7769{
7770 dev_priv->gt_pm.rps.gpll_ref_freq =
7771 vlv_get_cck_clock(dev_priv, "GPLL ref",
7772 CCK_GPLL_CLOCK_CONTROL,
7773 dev_priv->czclk_freq);
7774
7775 DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7776 dev_priv->gt_pm.rps.gpll_ref_freq);
7777}
7778
7779static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7780{
7781 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7782 u32 val;
7783
7784 valleyview_setup_pctx(dev_priv);
7785
7786 vlv_iosf_sb_get(dev_priv,
7787 BIT(VLV_IOSF_SB_PUNIT) |
7788 BIT(VLV_IOSF_SB_NC) |
7789 BIT(VLV_IOSF_SB_CCK));
7790
7791 vlv_init_gpll_ref_freq(dev_priv);
7792
7793 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7794 switch ((val >> 6) & 3) {
7795 case 0:
7796 case 1:
7797 dev_priv->mem_freq = 800;
7798 break;
7799 case 2:
7800 dev_priv->mem_freq = 1066;
7801 break;
7802 case 3:
7803 dev_priv->mem_freq = 1333;
7804 break;
7805 }
7806 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7807
7808 rps->max_freq = valleyview_rps_max_freq(dev_priv);
7809 rps->rp0_freq = rps->max_freq;
7810 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7811 intel_gpu_freq(dev_priv, rps->max_freq),
7812 rps->max_freq);
7813
7814 rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7815 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7816 intel_gpu_freq(dev_priv, rps->efficient_freq),
7817 rps->efficient_freq);
7818
7819 rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7820 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7821 intel_gpu_freq(dev_priv, rps->rp1_freq),
7822 rps->rp1_freq);
7823
7824 rps->min_freq = valleyview_rps_min_freq(dev_priv);
7825 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7826 intel_gpu_freq(dev_priv, rps->min_freq),
7827 rps->min_freq);
7828
7829 vlv_iosf_sb_put(dev_priv,
7830 BIT(VLV_IOSF_SB_PUNIT) |
7831 BIT(VLV_IOSF_SB_NC) |
7832 BIT(VLV_IOSF_SB_CCK));
7833}
7834
7835static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7836{
7837 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7838 u32 val;
7839
7840 cherryview_setup_pctx(dev_priv);
7841
7842 vlv_iosf_sb_get(dev_priv,
7843 BIT(VLV_IOSF_SB_PUNIT) |
7844 BIT(VLV_IOSF_SB_NC) |
7845 BIT(VLV_IOSF_SB_CCK));
7846
7847 vlv_init_gpll_ref_freq(dev_priv);
7848
7849 val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7850
7851 switch ((val >> 2) & 0x7) {
7852 case 3:
7853 dev_priv->mem_freq = 2000;
7854 break;
7855 default:
7856 dev_priv->mem_freq = 1600;
7857 break;
7858 }
7859 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7860
7861 rps->max_freq = cherryview_rps_max_freq(dev_priv);
7862 rps->rp0_freq = rps->max_freq;
7863 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7864 intel_gpu_freq(dev_priv, rps->max_freq),
7865 rps->max_freq);
7866
7867 rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7868 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7869 intel_gpu_freq(dev_priv, rps->efficient_freq),
7870 rps->efficient_freq);
7871
7872 rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7873 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7874 intel_gpu_freq(dev_priv, rps->rp1_freq),
7875 rps->rp1_freq);
7876
7877 rps->min_freq = cherryview_rps_min_freq(dev_priv);
7878 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7879 intel_gpu_freq(dev_priv, rps->min_freq),
7880 rps->min_freq);
7881
7882 vlv_iosf_sb_put(dev_priv,
7883 BIT(VLV_IOSF_SB_PUNIT) |
7884 BIT(VLV_IOSF_SB_NC) |
7885 BIT(VLV_IOSF_SB_CCK));
7886
7887 WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7888 rps->min_freq) & 1,
7889 "Odd GPU freq values\n");
7890}
7891
7892static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7893{
7894 valleyview_cleanup_pctx(dev_priv);
7895}
7896
7897static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7898{
7899 struct intel_engine_cs *engine;
7900 enum intel_engine_id id;
7901 u32 gtfifodbg, rc6_mode, pcbr;
7902
7903 gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7904 GT_FIFO_FREE_ENTRIES_CHV);
7905 if (gtfifodbg) {
7906 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7907 gtfifodbg);
7908 I915_WRITE(GTFIFODBG, gtfifodbg);
7909 }
7910
7911 cherryview_check_pctx(dev_priv);
7912
7913
7914
7915 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7916
7917
7918 I915_WRITE(GEN6_RC_CONTROL, 0);
7919
7920
7921 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7922 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7923 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7924
7925 for_each_engine(engine, dev_priv, id)
7926 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7927 I915_WRITE(GEN6_RC_SLEEP, 0);
7928
7929
7930 I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7931
7932
7933 I915_WRITE(VLV_COUNTER_CONTROL,
7934 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7935 VLV_MEDIA_RC6_COUNT_EN |
7936 VLV_RENDER_RC6_COUNT_EN));
7937
7938
7939 pcbr = I915_READ(VLV_PCBR);
7940
7941
7942 rc6_mode = 0;
7943 if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7944 rc6_mode = GEN7_RC_CTL_TO_MODE;
7945 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7946
7947 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7948}
7949
7950static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7951{
7952 u32 val;
7953
7954 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
7955
7956
7957 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7958 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7959 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7960 I915_WRITE(GEN6_RP_UP_EI, 66000);
7961 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7962
7963 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7964
7965
7966 I915_WRITE(GEN6_RP_CONTROL,
7967 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7968 GEN6_RP_MEDIA_IS_GFX |
7969 GEN6_RP_ENABLE |
7970 GEN6_RP_UP_BUSY_AVG |
7971 GEN6_RP_DOWN_IDLE_AVG);
7972
7973
7974 vlv_punit_get(dev_priv);
7975
7976 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
7977 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7978
7979 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7980
7981 vlv_punit_put(dev_priv);
7982
7983
7984 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7985
7986 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7987 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7988
7989 reset_rps(dev_priv, valleyview_set_rps);
7990
7991 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
7992}
7993
7994static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7995{
7996 struct intel_engine_cs *engine;
7997 enum intel_engine_id id;
7998 u32 gtfifodbg;
7999
8000 valleyview_check_pctx(dev_priv);
8001
8002 gtfifodbg = I915_READ(GTFIFODBG);
8003 if (gtfifodbg) {
8004 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
8005 gtfifodbg);
8006 I915_WRITE(GTFIFODBG, gtfifodbg);
8007 }
8008
8009 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
8010
8011
8012 I915_WRITE(GEN6_RC_CONTROL, 0);
8013
8014 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
8015 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
8016 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
8017
8018 for_each_engine(engine, dev_priv, id)
8019 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
8020
8021 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
8022
8023
8024 I915_WRITE(VLV_COUNTER_CONTROL,
8025 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
8026 VLV_MEDIA_RC0_COUNT_EN |
8027 VLV_RENDER_RC0_COUNT_EN |
8028 VLV_MEDIA_RC6_COUNT_EN |
8029 VLV_RENDER_RC6_COUNT_EN));
8030
8031 I915_WRITE(GEN6_RC_CONTROL,
8032 GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
8033
8034 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
8035}
8036
8037static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
8038{
8039 u32 val;
8040
8041 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
8042
8043 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
8044 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
8045 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
8046 I915_WRITE(GEN6_RP_UP_EI, 66000);
8047 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
8048
8049 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
8050
8051 I915_WRITE(GEN6_RP_CONTROL,
8052 GEN6_RP_MEDIA_TURBO |
8053 GEN6_RP_MEDIA_HW_NORMAL_MODE |
8054 GEN6_RP_MEDIA_IS_GFX |
8055 GEN6_RP_ENABLE |
8056 GEN6_RP_UP_BUSY_AVG |
8057 GEN6_RP_DOWN_IDLE_CONT);
8058
8059 vlv_punit_get(dev_priv);
8060
8061
8062 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
8063 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
8064
8065 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
8066
8067 vlv_punit_put(dev_priv);
8068
8069
8070 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
8071
8072 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
8073 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
8074
8075 reset_rps(dev_priv, valleyview_set_rps);
8076
8077 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
8078}
8079
8080static unsigned long intel_pxfreq(u32 vidfreq)
8081{
8082 unsigned long freq;
8083 int div = (vidfreq & 0x3f0000) >> 16;
8084 int post = (vidfreq & 0x3000) >> 12;
8085 int pre = (vidfreq & 0x7);
8086
8087 if (!pre)
8088 return 0;
8089
8090 freq = ((div * 133333) / ((1<<post) * pre));
8091
8092 return freq;
8093}
8094
8095static const struct cparams {
8096 u16 i;
8097 u16 t;
8098 u16 m;
8099 u16 c;
8100} cparams[] = {
8101 { 1, 1333, 301, 28664 },
8102 { 1, 1066, 294, 24460 },
8103 { 1, 800, 294, 25192 },
8104 { 0, 1333, 276, 27605 },
8105 { 0, 1066, 276, 27605 },
8106 { 0, 800, 231, 23784 },
8107};
8108
8109static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
8110{
8111 u64 total_count, diff, ret;
8112 u32 count1, count2, count3, m = 0, c = 0;
8113 unsigned long now = jiffies_to_msecs(jiffies), diff1;
8114 int i;
8115
8116 lockdep_assert_held(&mchdev_lock);
8117
8118 diff1 = now - dev_priv->ips.last_time1;
8119
8120
8121
8122
8123
8124
8125 if (diff1 <= 10)
8126 return dev_priv->ips.chipset_power;
8127
8128 count1 = I915_READ(DMIEC);
8129 count2 = I915_READ(DDREC);
8130 count3 = I915_READ(CSIEC);
8131
8132 total_count = count1 + count2 + count3;
8133
8134
8135 if (total_count < dev_priv->ips.last_count1) {
8136 diff = ~0UL - dev_priv->ips.last_count1;
8137 diff += total_count;
8138 } else {
8139 diff = total_count - dev_priv->ips.last_count1;
8140 }
8141
8142 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
8143 if (cparams[i].i == dev_priv->ips.c_m &&
8144 cparams[i].t == dev_priv->ips.r_t) {
8145 m = cparams[i].m;
8146 c = cparams[i].c;
8147 break;
8148 }
8149 }
8150
8151 diff = div_u64(diff, diff1);
8152 ret = ((m * diff) + c);
8153 ret = div_u64(ret, 10);
8154
8155 dev_priv->ips.last_count1 = total_count;
8156 dev_priv->ips.last_time1 = now;
8157
8158 dev_priv->ips.chipset_power = ret;
8159
8160 return ret;
8161}
8162
8163unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
8164{
8165 intel_wakeref_t wakeref;
8166 unsigned long val = 0;
8167
8168 if (!IS_GEN(dev_priv, 5))
8169 return 0;
8170
8171 with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
8172 spin_lock_irq(&mchdev_lock);
8173 val = __i915_chipset_val(dev_priv);
8174 spin_unlock_irq(&mchdev_lock);
8175 }
8176
8177 return val;
8178}
8179
8180unsigned long i915_mch_val(struct drm_i915_private *i915)
8181{
8182 unsigned long m, x, b;
8183 u32 tsfs;
8184
8185 tsfs = intel_uncore_read(&i915->uncore, TSFS);
8186
8187 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
8188 x = intel_uncore_read8(&i915->uncore, TR1);
8189
8190 b = tsfs & TSFS_INTR_MASK;
8191
8192 return ((m * x) / 127) - b;
8193}
8194
8195static int _pxvid_to_vd(u8 pxvid)
8196{
8197 if (pxvid == 0)
8198 return 0;
8199
8200 if (pxvid >= 8 && pxvid < 31)
8201 pxvid = 31;
8202
8203 return (pxvid + 2) * 125;
8204}
8205
8206static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
8207{
8208 const int vd = _pxvid_to_vd(pxvid);
8209 const int vm = vd - 1125;
8210
8211 if (INTEL_INFO(dev_priv)->is_mobile)
8212 return vm > 0 ? vm : 0;
8213
8214 return vd;
8215}
8216
8217static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
8218{
8219 u64 now, diff, diffms;
8220 u32 count;
8221
8222 lockdep_assert_held(&mchdev_lock);
8223
8224 now = ktime_get_raw_ns();
8225 diffms = now - dev_priv->ips.last_time2;
8226 do_div(diffms, NSEC_PER_MSEC);
8227
8228
8229 if (!diffms)
8230 return;
8231
8232 count = I915_READ(GFXEC);
8233
8234 if (count < dev_priv->ips.last_count2) {
8235 diff = ~0UL - dev_priv->ips.last_count2;
8236 diff += count;
8237 } else {
8238 diff = count - dev_priv->ips.last_count2;
8239 }
8240
8241 dev_priv->ips.last_count2 = count;
8242 dev_priv->ips.last_time2 = now;
8243
8244
8245 diff = diff * 1181;
8246 diff = div_u64(diff, diffms * 10);
8247 dev_priv->ips.gfx_power = diff;
8248}
8249
8250void i915_update_gfx_val(struct drm_i915_private *dev_priv)
8251{
8252 intel_wakeref_t wakeref;
8253
8254 if (!IS_GEN(dev_priv, 5))
8255 return;
8256
8257 with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
8258 spin_lock_irq(&mchdev_lock);
8259 __i915_update_gfx_val(dev_priv);
8260 spin_unlock_irq(&mchdev_lock);
8261 }
8262}
8263
8264static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
8265{
8266 unsigned long t, corr, state1, corr2, state2;
8267 u32 pxvid, ext_v;
8268
8269 lockdep_assert_held(&mchdev_lock);
8270
8271 pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
8272 pxvid = (pxvid >> 24) & 0x7f;
8273 ext_v = pvid_to_extvid(dev_priv, pxvid);
8274
8275 state1 = ext_v;
8276
8277 t = i915_mch_val(dev_priv);
8278
8279
8280
8281
8282 if (t > 80)
8283 corr = ((t * 2349) + 135940);
8284 else if (t >= 50)
8285 corr = ((t * 964) + 29317);
8286 else
8287 corr = ((t * 301) + 1004);
8288
8289 corr = corr * ((150142 * state1) / 10000 - 78642);
8290 corr /= 100000;
8291 corr2 = (corr * dev_priv->ips.corr);
8292
8293 state2 = (corr2 * state1) / 10000;
8294 state2 /= 100;
8295
8296 __i915_update_gfx_val(dev_priv);
8297
8298 return dev_priv->ips.gfx_power + state2;
8299}
8300
8301unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
8302{
8303 intel_wakeref_t wakeref;
8304 unsigned long val = 0;
8305
8306 if (!IS_GEN(dev_priv, 5))
8307 return 0;
8308
8309 with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
8310 spin_lock_irq(&mchdev_lock);
8311 val = __i915_gfx_val(dev_priv);
8312 spin_unlock_irq(&mchdev_lock);
8313 }
8314
8315 return val;
8316}
8317
8318static struct drm_i915_private __rcu *i915_mch_dev;
8319
8320static struct drm_i915_private *mchdev_get(void)
8321{
8322 struct drm_i915_private *i915;
8323
8324 rcu_read_lock();
8325 i915 = rcu_dereference(i915_mch_dev);
8326 if (!kref_get_unless_zero(&i915->drm.ref))
8327 i915 = NULL;
8328 rcu_read_unlock();
8329
8330 return i915;
8331}
8332
8333
8334
8335
8336
8337
8338
8339unsigned long i915_read_mch_val(void)
8340{
8341 struct drm_i915_private *i915;
8342 unsigned long chipset_val = 0;
8343 unsigned long graphics_val = 0;
8344 intel_wakeref_t wakeref;
8345
8346 i915 = mchdev_get();
8347 if (!i915)
8348 return 0;
8349
8350 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
8351 spin_lock_irq(&mchdev_lock);
8352 chipset_val = __i915_chipset_val(i915);
8353 graphics_val = __i915_gfx_val(i915);
8354 spin_unlock_irq(&mchdev_lock);
8355 }
8356
8357 drm_dev_put(&i915->drm);
8358 return chipset_val + graphics_val;
8359}
8360EXPORT_SYMBOL_GPL(i915_read_mch_val);
8361
8362
8363
8364
8365
8366
8367bool i915_gpu_raise(void)
8368{
8369 struct drm_i915_private *i915;
8370
8371 i915 = mchdev_get();
8372 if (!i915)
8373 return false;
8374
8375 spin_lock_irq(&mchdev_lock);
8376 if (i915->ips.max_delay > i915->ips.fmax)
8377 i915->ips.max_delay--;
8378 spin_unlock_irq(&mchdev_lock);
8379
8380 drm_dev_put(&i915->drm);
8381 return true;
8382}
8383EXPORT_SYMBOL_GPL(i915_gpu_raise);
8384
8385
8386
8387
8388
8389
8390
8391bool i915_gpu_lower(void)
8392{
8393 struct drm_i915_private *i915;
8394
8395 i915 = mchdev_get();
8396 if (!i915)
8397 return false;
8398
8399 spin_lock_irq(&mchdev_lock);
8400 if (i915->ips.max_delay < i915->ips.min_delay)
8401 i915->ips.max_delay++;
8402 spin_unlock_irq(&mchdev_lock);
8403
8404 drm_dev_put(&i915->drm);
8405 return true;
8406}
8407EXPORT_SYMBOL_GPL(i915_gpu_lower);
8408
8409
8410
8411
8412
8413
8414bool i915_gpu_busy(void)
8415{
8416 struct drm_i915_private *i915;
8417 bool ret;
8418
8419 i915 = mchdev_get();
8420 if (!i915)
8421 return false;
8422
8423 ret = i915->gt.awake;
8424
8425 drm_dev_put(&i915->drm);
8426 return ret;
8427}
8428EXPORT_SYMBOL_GPL(i915_gpu_busy);
8429
8430
8431
8432
8433
8434
8435
8436bool i915_gpu_turbo_disable(void)
8437{
8438 struct drm_i915_private *i915;
8439 bool ret;
8440
8441 i915 = mchdev_get();
8442 if (!i915)
8443 return false;
8444
8445 spin_lock_irq(&mchdev_lock);
8446 i915->ips.max_delay = i915->ips.fstart;
8447 ret = ironlake_set_drps(i915, i915->ips.fstart);
8448 spin_unlock_irq(&mchdev_lock);
8449
8450 drm_dev_put(&i915->drm);
8451 return ret;
8452}
8453EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463static void
8464ips_ping_for_i915_load(void)
8465{
8466 void (*link)(void);
8467
8468 link = symbol_get(ips_link_to_i915_driver);
8469 if (link) {
8470 link();
8471 symbol_put(ips_link_to_i915_driver);
8472 }
8473}
8474
8475void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
8476{
8477
8478
8479 rcu_assign_pointer(i915_mch_dev, dev_priv);
8480
8481 ips_ping_for_i915_load();
8482}
8483
8484void intel_gpu_ips_teardown(void)
8485{
8486 rcu_assign_pointer(i915_mch_dev, NULL);
8487}
8488
8489static void intel_init_emon(struct drm_i915_private *dev_priv)
8490{
8491 u32 lcfuse;
8492 u8 pxw[16];
8493 int i;
8494
8495
8496 I915_WRITE(ECR, 0);
8497 POSTING_READ(ECR);
8498
8499
8500 I915_WRITE(SDEW, 0x15040d00);
8501 I915_WRITE(CSIEW0, 0x007f0000);
8502 I915_WRITE(CSIEW1, 0x1e220004);
8503 I915_WRITE(CSIEW2, 0x04000004);
8504
8505 for (i = 0; i < 5; i++)
8506 I915_WRITE(PEW(i), 0);
8507 for (i = 0; i < 3; i++)
8508 I915_WRITE(DEW(i), 0);
8509
8510
8511 for (i = 0; i < 16; i++) {
8512 u32 pxvidfreq = I915_READ(PXVFREQ(i));
8513 unsigned long freq = intel_pxfreq(pxvidfreq);
8514 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8515 PXVFREQ_PX_SHIFT;
8516 unsigned long val;
8517
8518 val = vid * vid;
8519 val *= (freq / 1000);
8520 val *= 255;
8521 val /= (127*127*900);
8522 if (val > 0xff)
8523 DRM_ERROR("bad pxval: %ld\n", val);
8524 pxw[i] = val;
8525 }
8526
8527 pxw[14] = 0;
8528 pxw[15] = 0;
8529
8530 for (i = 0; i < 4; i++) {
8531 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8532 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8533 I915_WRITE(PXW(i), val);
8534 }
8535
8536
8537 I915_WRITE(OGW0, 0);
8538 I915_WRITE(OGW1, 0);
8539 I915_WRITE(EG0, 0x00007f00);
8540 I915_WRITE(EG1, 0x0000000e);
8541 I915_WRITE(EG2, 0x000e0000);
8542 I915_WRITE(EG3, 0x68000300);
8543 I915_WRITE(EG4, 0x42000000);
8544 I915_WRITE(EG5, 0x00140031);
8545 I915_WRITE(EG6, 0);
8546 I915_WRITE(EG7, 0);
8547
8548 for (i = 0; i < 8; i++)
8549 I915_WRITE(PXWL(i), 0);
8550
8551
8552 I915_WRITE(ECR, 0x80000019);
8553
8554 lcfuse = I915_READ(LCFUSE02);
8555
8556 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8557}
8558
8559void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8560{
8561 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8562
8563
8564
8565
8566
8567 if (!sanitize_rc6(dev_priv)) {
8568 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
8569 pm_runtime_get(&dev_priv->drm.pdev->dev);
8570 }
8571
8572
8573 if (IS_CHERRYVIEW(dev_priv))
8574 cherryview_init_gt_powersave(dev_priv);
8575 else if (IS_VALLEYVIEW(dev_priv))
8576 valleyview_init_gt_powersave(dev_priv);
8577 else if (INTEL_GEN(dev_priv) >= 6)
8578 gen6_init_rps_frequencies(dev_priv);
8579
8580
8581 rps->max_freq_softlimit = rps->max_freq;
8582 rps->min_freq_softlimit = rps->min_freq;
8583
8584
8585 if (IS_GEN(dev_priv, 6) ||
8586 IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
8587 u32 params = 0;
8588
8589 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS,
8590 ¶ms, NULL);
8591 if (params & BIT(31)) {
8592 DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
8593 (rps->max_freq & 0xff) * 50,
8594 (params & 0xff) * 50);
8595 rps->max_freq = params & 0xff;
8596 }
8597 }
8598
8599
8600 rps->boost_freq = rps->max_freq;
8601 rps->idle_freq = rps->min_freq;
8602 rps->cur_freq = rps->idle_freq;
8603}
8604
8605void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8606{
8607 if (IS_VALLEYVIEW(dev_priv))
8608 valleyview_cleanup_gt_powersave(dev_priv);
8609
8610 if (!HAS_RC6(dev_priv))
8611 pm_runtime_put(&dev_priv->drm.pdev->dev);
8612}
8613
8614void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8615{
8616 dev_priv->gt_pm.rps.enabled = true;
8617 dev_priv->gt_pm.rc6.enabled = true;
8618 intel_disable_gt_powersave(dev_priv);
8619
8620 if (INTEL_GEN(dev_priv) >= 11)
8621 gen11_reset_rps_interrupts(dev_priv);
8622 else if (INTEL_GEN(dev_priv) >= 6)
8623 gen6_reset_rps_interrupts(dev_priv);
8624}
8625
8626static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8627{
8628 lockdep_assert_held(&i915->gt_pm.rps.lock);
8629
8630 if (!i915->gt_pm.llc_pstate.enabled)
8631 return;
8632
8633
8634
8635 i915->gt_pm.llc_pstate.enabled = false;
8636}
8637
8638static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8639{
8640 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8641
8642 if (!dev_priv->gt_pm.rc6.enabled)
8643 return;
8644
8645 if (INTEL_GEN(dev_priv) >= 9)
8646 gen9_disable_rc6(dev_priv);
8647 else if (IS_CHERRYVIEW(dev_priv))
8648 cherryview_disable_rc6(dev_priv);
8649 else if (IS_VALLEYVIEW(dev_priv))
8650 valleyview_disable_rc6(dev_priv);
8651 else if (INTEL_GEN(dev_priv) >= 6)
8652 gen6_disable_rc6(dev_priv);
8653
8654 dev_priv->gt_pm.rc6.enabled = false;
8655}
8656
8657static void intel_disable_rps(struct drm_i915_private *dev_priv)
8658{
8659 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8660
8661 if (!dev_priv->gt_pm.rps.enabled)
8662 return;
8663
8664 if (INTEL_GEN(dev_priv) >= 9)
8665 gen9_disable_rps(dev_priv);
8666 else if (IS_CHERRYVIEW(dev_priv))
8667 cherryview_disable_rps(dev_priv);
8668 else if (IS_VALLEYVIEW(dev_priv))
8669 valleyview_disable_rps(dev_priv);
8670 else if (INTEL_GEN(dev_priv) >= 6)
8671 gen6_disable_rps(dev_priv);
8672 else if (IS_IRONLAKE_M(dev_priv))
8673 ironlake_disable_drps(dev_priv);
8674
8675 dev_priv->gt_pm.rps.enabled = false;
8676}
8677
8678void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8679{
8680 mutex_lock(&dev_priv->gt_pm.rps.lock);
8681
8682 intel_disable_rc6(dev_priv);
8683 intel_disable_rps(dev_priv);
8684 if (HAS_LLC(dev_priv))
8685 intel_disable_llc_pstate(dev_priv);
8686
8687 mutex_unlock(&dev_priv->gt_pm.rps.lock);
8688}
8689
8690static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8691{
8692 lockdep_assert_held(&i915->gt_pm.rps.lock);
8693
8694 if (i915->gt_pm.llc_pstate.enabled)
8695 return;
8696
8697 gen6_update_ring_freq(i915);
8698
8699 i915->gt_pm.llc_pstate.enabled = true;
8700}
8701
8702static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8703{
8704 lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
8705
8706 if (dev_priv->gt_pm.rc6.enabled)
8707 return;
8708
8709 if (IS_CHERRYVIEW(dev_priv))
8710 cherryview_enable_rc6(dev_priv);
8711 else if (IS_VALLEYVIEW(dev_priv))
8712 valleyview_enable_rc6(dev_priv);
8713 else if (INTEL_GEN(dev_priv) >= 11)
8714 gen11_enable_rc6(dev_priv);
8715 else if (INTEL_GEN(dev_priv) >= 9)
8716 gen9_enable_rc6(dev_priv);
8717 else if (IS_BROADWELL(dev_priv))
8718 gen8_enable_rc6(dev_priv);
8719 else if (INTEL_GEN(dev_priv) >= 6)
8720 gen6_enable_rc6(dev_priv);
8721
8722 dev_priv->gt_pm.rc6.enabled = true;
8723}
8724
8725static void intel_enable_rps(struct drm_i915_private *dev_priv)
8726{
8727 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8728
8729 lockdep_assert_held(&rps->lock);
8730
8731 if (rps->enabled)
8732 return;
8733
8734 if (IS_CHERRYVIEW(dev_priv)) {
8735 cherryview_enable_rps(dev_priv);
8736 } else if (IS_VALLEYVIEW(dev_priv)) {
8737 valleyview_enable_rps(dev_priv);
8738 } else if (INTEL_GEN(dev_priv) >= 9) {
8739 gen9_enable_rps(dev_priv);
8740 } else if (IS_BROADWELL(dev_priv)) {
8741 gen8_enable_rps(dev_priv);
8742 } else if (INTEL_GEN(dev_priv) >= 6) {
8743 gen6_enable_rps(dev_priv);
8744 } else if (IS_IRONLAKE_M(dev_priv)) {
8745 ironlake_enable_drps(dev_priv);
8746 intel_init_emon(dev_priv);
8747 }
8748
8749 WARN_ON(rps->max_freq < rps->min_freq);
8750 WARN_ON(rps->idle_freq > rps->max_freq);
8751
8752 WARN_ON(rps->efficient_freq < rps->min_freq);
8753 WARN_ON(rps->efficient_freq > rps->max_freq);
8754
8755 rps->enabled = true;
8756}
8757
8758void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8759{
8760
8761 if (intel_vgpu_active(dev_priv))
8762 return;
8763
8764 mutex_lock(&dev_priv->gt_pm.rps.lock);
8765
8766 if (HAS_RC6(dev_priv))
8767 intel_enable_rc6(dev_priv);
8768 if (HAS_RPS(dev_priv))
8769 intel_enable_rps(dev_priv);
8770 if (HAS_LLC(dev_priv))
8771 intel_enable_llc_pstate(dev_priv);
8772
8773 mutex_unlock(&dev_priv->gt_pm.rps.lock);
8774}
8775
8776static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8777{
8778
8779
8780
8781
8782
8783 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8784}
8785
8786static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8787{
8788 enum pipe pipe;
8789
8790 for_each_pipe(dev_priv, pipe) {
8791 I915_WRITE(DSPCNTR(pipe),
8792 I915_READ(DSPCNTR(pipe)) |
8793 DISPPLANE_TRICKLE_FEED_DISABLE);
8794
8795 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8796 POSTING_READ(DSPSURF(pipe));
8797 }
8798}
8799
8800static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8801{
8802 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8803
8804
8805
8806
8807
8808 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8809 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8810 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8811
8812 I915_WRITE(PCH_3DCGDIS0,
8813 MARIUNIT_CLOCK_GATE_DISABLE |
8814 SVSMUNIT_CLOCK_GATE_DISABLE);
8815 I915_WRITE(PCH_3DCGDIS1,
8816 VFMUNIT_CLOCK_GATE_DISABLE);
8817
8818
8819
8820
8821
8822
8823
8824
8825 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8826 (I915_READ(ILK_DISPLAY_CHICKEN2) |
8827 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8828 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8829 I915_WRITE(DISP_ARB_CTL,
8830 (I915_READ(DISP_ARB_CTL) |
8831 DISP_FBC_WM_DIS));
8832
8833
8834
8835
8836
8837
8838
8839
8840 if (IS_IRONLAKE_M(dev_priv)) {
8841
8842 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8843 I915_READ(ILK_DISPLAY_CHICKEN1) |
8844 ILK_FBCQ_DIS);
8845 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8846 I915_READ(ILK_DISPLAY_CHICKEN2) |
8847 ILK_DPARB_GATE);
8848 }
8849
8850 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8851
8852 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8853 I915_READ(ILK_DISPLAY_CHICKEN2) |
8854 ILK_ELPIN_409_SELECT);
8855 I915_WRITE(_3D_CHICKEN2,
8856 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8857 _3D_CHICKEN2_WM_READ_PIPELINED);
8858
8859
8860 I915_WRITE(CACHE_MODE_0,
8861 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8862
8863
8864 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8865
8866 g4x_disable_trickle_feed(dev_priv);
8867
8868 ibx_init_clock_gating(dev_priv);
8869}
8870
8871static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8872{
8873 int pipe;
8874 u32 val;
8875
8876
8877
8878
8879
8880
8881 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8882 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8883 PCH_CPUNIT_CLOCK_GATE_DISABLE);
8884 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8885 DPLS_EDP_PPS_FIX_DIS);
8886
8887
8888
8889 for_each_pipe(dev_priv, pipe) {
8890 val = I915_READ(TRANS_CHICKEN2(pipe));
8891 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8892 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8893 if (dev_priv->vbt.fdi_rx_polarity_inverted)
8894 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8895 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
8896 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
8897 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
8898 I915_WRITE(TRANS_CHICKEN2(pipe), val);
8899 }
8900
8901 for_each_pipe(dev_priv, pipe) {
8902 I915_WRITE(TRANS_CHICKEN1(pipe),
8903 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8904 }
8905}
8906
8907static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
8908{
8909 u32 tmp;
8910
8911 tmp = I915_READ(MCH_SSKPD);
8912 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
8913 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
8914 tmp);
8915}
8916
8917static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
8918{
8919 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8920
8921 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8922
8923 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8924 I915_READ(ILK_DISPLAY_CHICKEN2) |
8925 ILK_ELPIN_409_SELECT);
8926
8927
8928 I915_WRITE(_3D_CHICKEN,
8929 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
8930
8931
8932 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942 I915_WRITE(GEN6_GT_MODE,
8943 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8944
8945 I915_WRITE(CACHE_MODE_0,
8946 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
8947
8948 I915_WRITE(GEN6_UCGCTL1,
8949 I915_READ(GEN6_UCGCTL1) |
8950 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
8951 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8952
8953
8954
8955
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966 I915_WRITE(GEN6_UCGCTL2,
8967 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8968 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8969
8970
8971 I915_WRITE(_3D_CHICKEN3,
8972 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
8973
8974
8975
8976
8977
8978
8979 I915_WRITE(_3D_CHICKEN3,
8980 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
8981
8982
8983
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8994 I915_READ(ILK_DISPLAY_CHICKEN1) |
8995 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8996 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8997 I915_READ(ILK_DISPLAY_CHICKEN2) |
8998 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8999 I915_WRITE(ILK_DSPCLK_GATE_D,
9000 I915_READ(ILK_DSPCLK_GATE_D) |
9001 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
9002 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
9003
9004 g4x_disable_trickle_feed(dev_priv);
9005
9006 cpt_init_clock_gating(dev_priv);
9007
9008 gen6_check_mch_setup(dev_priv);
9009}
9010
9011static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
9012{
9013 u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
9014
9015
9016
9017
9018
9019
9020
9021 reg &= ~GEN7_FF_SCHED_MASK;
9022 reg |= GEN7_FF_TS_SCHED_HW;
9023 reg |= GEN7_FF_VS_SCHED_HW;
9024 reg |= GEN7_FF_DS_SCHED_HW;
9025
9026 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
9027}
9028
9029static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
9030{
9031
9032
9033
9034
9035 if (HAS_PCH_LPT_LP(dev_priv))
9036 I915_WRITE(SOUTH_DSPCLK_GATE_D,
9037 I915_READ(SOUTH_DSPCLK_GATE_D) |
9038 PCH_LP_PARTITION_LEVEL_DISABLE);
9039
9040
9041 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
9042 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
9043 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
9044}
9045
9046static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
9047{
9048 if (HAS_PCH_LPT_LP(dev_priv)) {
9049 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9050
9051 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9052 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9053 }
9054}
9055
9056static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
9057 int general_prio_credits,
9058 int high_prio_credits)
9059{
9060 u32 misccpctl;
9061 u32 val;
9062
9063
9064 misccpctl = I915_READ(GEN7_MISCCPCTL);
9065 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
9066
9067 val = I915_READ(GEN8_L3SQCREG1);
9068 val &= ~L3_PRIO_CREDITS_MASK;
9069 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
9070 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
9071 I915_WRITE(GEN8_L3SQCREG1, val);
9072
9073
9074
9075
9076
9077 POSTING_READ(GEN8_L3SQCREG1);
9078 udelay(1);
9079 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
9080}
9081
9082static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
9083{
9084
9085 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
9086 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
9087
9088
9089 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS,
9090 _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE));
9091}
9092
9093static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
9094{
9095 if (!HAS_PCH_CNP(dev_priv))
9096 return;
9097
9098
9099 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
9100 CNP_PWM_CGE_GATING_DISABLE);
9101}
9102
9103static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
9104{
9105 u32 val;
9106 cnp_init_clock_gating(dev_priv);
9107
9108
9109 I915_WRITE(_3D_CHICKEN3,
9110 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
9111
9112
9113 I915_WRITE(GEN8_CHICKEN_DCPR_1,
9114 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
9115
9116
9117 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
9118 DISP_FBC_MEMORY_WAKE);
9119
9120 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
9121
9122 val |= RCCUNIT_CLKGATE_DIS;
9123
9124 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
9125 val |= SARBUNIT_CLKGATE_DIS;
9126 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
9127
9128
9129 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
9130 val |= GWUNIT_CLKGATE_DIS;
9131 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
9132
9133
9134
9135 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
9136 val |= VFUNIT_CLKGATE_DIS;
9137 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
9138}
9139
9140static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
9141{
9142 cnp_init_clock_gating(dev_priv);
9143 gen9_init_clock_gating(dev_priv);
9144
9145
9146 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
9147 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
9148}
9149
9150static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
9151{
9152 gen9_init_clock_gating(dev_priv);
9153
9154
9155 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
9156 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9157 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9158
9159
9160 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
9161 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9162 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
9163
9164
9165 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
9166 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
9167}
9168
9169static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
9170{
9171 gen9_init_clock_gating(dev_priv);
9172
9173
9174 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
9175 FBC_LLC_FULLY_OPEN);
9176
9177
9178 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
9179 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
9180}
9181
9182static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
9183{
9184
9185 bool can_use_gtt_cache = !HAS_PAGE_SIZES(dev_priv,
9186 I915_GTT_PAGE_SIZE_2M);
9187 enum pipe pipe;
9188
9189
9190 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
9191
9192
9193 I915_WRITE(CHICKEN_PAR1_1,
9194 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
9195
9196
9197 for_each_pipe(dev_priv, pipe) {
9198 I915_WRITE(CHICKEN_PIPESL_1(pipe),
9199 I915_READ(CHICKEN_PIPESL_1(pipe)) |
9200 BDW_DPRS_MASK_VBLANK_SRD);
9201 }
9202
9203
9204
9205 I915_WRITE(GEN7_FF_THREAD_MODE,
9206 I915_READ(GEN7_FF_THREAD_MODE) &
9207 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9208
9209 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9210 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9211
9212
9213 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9214 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9215
9216
9217 gen8_set_l3sqc_credits(dev_priv, 30, 2);
9218
9219
9220 I915_WRITE(HSW_GTT_CACHE_EN, can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
9221
9222
9223 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
9224 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
9225
9226 lpt_init_clock_gating(dev_priv);
9227
9228
9229
9230
9231
9232
9233 I915_WRITE(GEN6_UCGCTL1,
9234 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
9235}
9236
9237static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
9238{
9239
9240 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
9241 I915_WRITE(HSW_ROW_CHICKEN3,
9242 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
9243
9244
9245 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9246 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9247 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9248
9249
9250 I915_WRITE(GEN7_FF_THREAD_MODE,
9251 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
9252
9253
9254 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9255
9256
9257 I915_WRITE(CACHE_MODE_0_GEN7,
9258 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9259
9260
9261 I915_WRITE(CACHE_MODE_1,
9262 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9263
9264
9265
9266
9267
9268
9269
9270
9271
9272 I915_WRITE(GEN7_GT_MODE,
9273 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9274
9275
9276 I915_WRITE(HALF_SLICE_CHICKEN3,
9277 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
9278
9279
9280 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
9281
9282 lpt_init_clock_gating(dev_priv);
9283}
9284
9285static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
9286{
9287 u32 snpcr;
9288
9289 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
9290
9291
9292 I915_WRITE(_3D_CHICKEN3,
9293 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9294
9295
9296 I915_WRITE(IVB_CHICKEN3,
9297 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9298 CHICKEN3_DGMG_DONE_FIX_DISABLE);
9299
9300
9301 if (IS_IVB_GT1(dev_priv))
9302 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9303 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9304
9305
9306 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9307
9308
9309 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
9310 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
9311
9312
9313 I915_WRITE(GEN7_L3CNTLREG1,
9314 GEN7_WA_FOR_GEN7_L3_CONTROL);
9315 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
9316 GEN7_WA_L3_CHICKEN_MODE);
9317 if (IS_IVB_GT1(dev_priv))
9318 I915_WRITE(GEN7_ROW_CHICKEN2,
9319 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9320 else {
9321
9322 I915_WRITE(GEN7_ROW_CHICKEN2,
9323 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9324 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
9325 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9326 }
9327
9328
9329 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9330 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9331
9332
9333
9334
9335
9336 I915_WRITE(GEN6_UCGCTL2,
9337 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9338
9339
9340 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9341 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9342 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9343
9344 g4x_disable_trickle_feed(dev_priv);
9345
9346 gen7_setup_fixed_func_scheduler(dev_priv);
9347
9348 if (0) {
9349
9350 I915_WRITE(CACHE_MODE_0_GEN7,
9351 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9352 }
9353
9354
9355 I915_WRITE(CACHE_MODE_1,
9356 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9357
9358
9359
9360
9361
9362
9363
9364
9365
9366 I915_WRITE(GEN7_GT_MODE,
9367 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9368
9369 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
9370 snpcr &= ~GEN6_MBC_SNPCR_MASK;
9371 snpcr |= GEN6_MBC_SNPCR_MED;
9372 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
9373
9374 if (!HAS_PCH_NOP(dev_priv))
9375 cpt_init_clock_gating(dev_priv);
9376
9377 gen6_check_mch_setup(dev_priv);
9378}
9379
9380static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
9381{
9382
9383 I915_WRITE(_3D_CHICKEN3,
9384 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9385
9386
9387 I915_WRITE(IVB_CHICKEN3,
9388 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9389 CHICKEN3_DGMG_DONE_FIX_DISABLE);
9390
9391
9392
9393 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9394 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
9395 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9396
9397
9398 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9399
9400
9401 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9402 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9403
9404
9405 I915_WRITE(GEN7_ROW_CHICKEN2,
9406 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9407
9408
9409 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9410 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9411 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9412
9413 gen7_setup_fixed_func_scheduler(dev_priv);
9414
9415
9416
9417
9418
9419 I915_WRITE(GEN6_UCGCTL2,
9420 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9421
9422
9423
9424
9425 I915_WRITE(GEN7_UCGCTL4,
9426 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
9427
9428
9429
9430
9431
9432 I915_WRITE(CACHE_MODE_1,
9433 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443 I915_WRITE(GEN7_GT_MODE,
9444 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9445
9446
9447
9448
9449
9450 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
9451
9452
9453
9454
9455
9456
9457 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
9458}
9459
9460static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
9461{
9462
9463
9464 I915_WRITE(GEN7_FF_THREAD_MODE,
9465 I915_READ(GEN7_FF_THREAD_MODE) &
9466 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9467
9468
9469 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9470 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9471
9472
9473 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9474 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9475
9476
9477 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9478 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9479
9480
9481
9482
9483
9484
9485 gen8_set_l3sqc_credits(dev_priv, 38, 2);
9486
9487
9488
9489
9490
9491 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
9492}
9493
9494static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
9495{
9496 u32 dspclk_gate;
9497
9498 I915_WRITE(RENCLK_GATE_D1, 0);
9499 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
9500 GS_UNIT_CLOCK_GATE_DISABLE |
9501 CL_UNIT_CLOCK_GATE_DISABLE);
9502 I915_WRITE(RAMCLK_GATE_D, 0);
9503 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
9504 OVRUNIT_CLOCK_GATE_DISABLE |
9505 OVCUNIT_CLOCK_GATE_DISABLE;
9506 if (IS_GM45(dev_priv))
9507 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
9508 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
9509
9510
9511 I915_WRITE(CACHE_MODE_0,
9512 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
9513
9514
9515 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9516
9517 g4x_disable_trickle_feed(dev_priv);
9518}
9519
9520static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
9521{
9522 struct intel_uncore *uncore = &dev_priv->uncore;
9523
9524 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
9525 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
9526 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
9527 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
9528 intel_uncore_write16(uncore, DEUC, 0);
9529 intel_uncore_write(uncore,
9530 MI_ARB_STATE,
9531 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9532
9533
9534 intel_uncore_write(uncore,
9535 CACHE_MODE_0,
9536 _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9537}
9538
9539static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
9540{
9541 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
9542 I965_RCC_CLOCK_GATE_DISABLE |
9543 I965_RCPB_CLOCK_GATE_DISABLE |
9544 I965_ISC_CLOCK_GATE_DISABLE |
9545 I965_FBC_CLOCK_GATE_DISABLE);
9546 I915_WRITE(RENCLK_GATE_D2, 0);
9547 I915_WRITE(MI_ARB_STATE,
9548 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9549
9550
9551 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9552}
9553
9554static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
9555{
9556 u32 dstate = I915_READ(D_STATE);
9557
9558 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
9559 DSTATE_DOT_CLOCK_GATING;
9560 I915_WRITE(D_STATE, dstate);
9561
9562 if (IS_PINEVIEW(dev_priv))
9563 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
9564
9565
9566 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
9567
9568
9569 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
9570
9571
9572 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
9573
9574 I915_WRITE(MI_ARB_STATE,
9575 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9576}
9577
9578static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
9579{
9580 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
9581
9582
9583 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
9584 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
9585
9586 I915_WRITE(MEM_MODE,
9587 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
9588}
9589
9590static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
9591{
9592 I915_WRITE(MEM_MODE,
9593 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
9594 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
9595}
9596
9597void intel_init_clock_gating(struct drm_i915_private *dev_priv)
9598{
9599 dev_priv->display.init_clock_gating(dev_priv);
9600}
9601
9602void intel_suspend_hw(struct drm_i915_private *dev_priv)
9603{
9604 if (HAS_PCH_LPT(dev_priv))
9605 lpt_suspend_hw(dev_priv);
9606}
9607
9608static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
9609{
9610 DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
9611}
9612
9613
9614
9615
9616
9617
9618
9619
9620
9621
9622void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
9623{
9624 if (IS_GEN(dev_priv, 11))
9625 dev_priv->display.init_clock_gating = icl_init_clock_gating;
9626 else if (IS_CANNONLAKE(dev_priv))
9627 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
9628 else if (IS_COFFEELAKE(dev_priv))
9629 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
9630 else if (IS_SKYLAKE(dev_priv))
9631 dev_priv->display.init_clock_gating = skl_init_clock_gating;
9632 else if (IS_KABYLAKE(dev_priv))
9633 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
9634 else if (IS_BROXTON(dev_priv))
9635 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
9636 else if (IS_GEMINILAKE(dev_priv))
9637 dev_priv->display.init_clock_gating = glk_init_clock_gating;
9638 else if (IS_BROADWELL(dev_priv))
9639 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
9640 else if (IS_CHERRYVIEW(dev_priv))
9641 dev_priv->display.init_clock_gating = chv_init_clock_gating;
9642 else if (IS_HASWELL(dev_priv))
9643 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
9644 else if (IS_IVYBRIDGE(dev_priv))
9645 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
9646 else if (IS_VALLEYVIEW(dev_priv))
9647 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
9648 else if (IS_GEN(dev_priv, 6))
9649 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9650 else if (IS_GEN(dev_priv, 5))
9651 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
9652 else if (IS_G4X(dev_priv))
9653 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9654 else if (IS_I965GM(dev_priv))
9655 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
9656 else if (IS_I965G(dev_priv))
9657 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
9658 else if (IS_GEN(dev_priv, 3))
9659 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9660 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
9661 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9662 else if (IS_GEN(dev_priv, 2))
9663 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9664 else {
9665 MISSING_CASE(INTEL_DEVID(dev_priv));
9666 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9667 }
9668}
9669
9670
9671void intel_init_pm(struct drm_i915_private *dev_priv)
9672{
9673
9674 if (IS_PINEVIEW(dev_priv))
9675 i915_pineview_get_mem_freq(dev_priv);
9676 else if (IS_GEN(dev_priv, 5))
9677 i915_ironlake_get_mem_freq(dev_priv);
9678
9679
9680 if (INTEL_GEN(dev_priv) >= 9) {
9681 skl_setup_wm_latency(dev_priv);
9682 dev_priv->display.initial_watermarks = skl_initial_wm;
9683 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
9684 dev_priv->display.compute_global_watermarks = skl_compute_wm;
9685 } else if (HAS_PCH_SPLIT(dev_priv)) {
9686 ilk_setup_wm_latency(dev_priv);
9687
9688 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
9689 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9690 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
9691 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9692 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9693 dev_priv->display.compute_intermediate_wm =
9694 ilk_compute_intermediate_wm;
9695 dev_priv->display.initial_watermarks =
9696 ilk_initial_watermarks;
9697 dev_priv->display.optimize_watermarks =
9698 ilk_optimize_watermarks;
9699 } else {
9700 DRM_DEBUG_KMS("Failed to read display plane latency. "
9701 "Disable CxSR\n");
9702 }
9703 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9704 vlv_setup_wm_latency(dev_priv);
9705 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9706 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9707 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9708 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9709 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9710 } else if (IS_G4X(dev_priv)) {
9711 g4x_setup_wm_latency(dev_priv);
9712 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9713 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9714 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9715 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9716 } else if (IS_PINEVIEW(dev_priv)) {
9717 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
9718 dev_priv->is_ddr3,
9719 dev_priv->fsb_freq,
9720 dev_priv->mem_freq)) {
9721 DRM_INFO("failed to find known CxSR latency "
9722 "(found ddr%s fsb freq %d, mem freq %d), "
9723 "disabling CxSR\n",
9724 (dev_priv->is_ddr3 == 1) ? "3" : "2",
9725 dev_priv->fsb_freq, dev_priv->mem_freq);
9726
9727 intel_set_memory_cxsr(dev_priv, false);
9728 dev_priv->display.update_wm = NULL;
9729 } else
9730 dev_priv->display.update_wm = pineview_update_wm;
9731 } else if (IS_GEN(dev_priv, 4)) {
9732 dev_priv->display.update_wm = i965_update_wm;
9733 } else if (IS_GEN(dev_priv, 3)) {
9734 dev_priv->display.update_wm = i9xx_update_wm;
9735 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9736 } else if (IS_GEN(dev_priv, 2)) {
9737 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9738 dev_priv->display.update_wm = i845_update_wm;
9739 dev_priv->display.get_fifo_size = i845_get_fifo_size;
9740 } else {
9741 dev_priv->display.update_wm = i9xx_update_wm;
9742 dev_priv->display.get_fifo_size = i830_get_fifo_size;
9743 }
9744 } else {
9745 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9746 }
9747}
9748
9749static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9750{
9751 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9752
9753
9754
9755
9756
9757 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9758}
9759
9760static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9761{
9762 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9763
9764 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9765}
9766
9767static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9768{
9769 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9770
9771
9772
9773
9774
9775 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9776}
9777
9778static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9779{
9780 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9781
9782
9783 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9784}
9785
9786int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9787{
9788 if (INTEL_GEN(dev_priv) >= 9)
9789 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9790 GEN9_FREQ_SCALER);
9791 else if (IS_CHERRYVIEW(dev_priv))
9792 return chv_gpu_freq(dev_priv, val);
9793 else if (IS_VALLEYVIEW(dev_priv))
9794 return byt_gpu_freq(dev_priv, val);
9795 else
9796 return val * GT_FREQUENCY_MULTIPLIER;
9797}
9798
9799int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9800{
9801 if (INTEL_GEN(dev_priv) >= 9)
9802 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9803 GT_FREQUENCY_MULTIPLIER);
9804 else if (IS_CHERRYVIEW(dev_priv))
9805 return chv_freq_opcode(dev_priv, val);
9806 else if (IS_VALLEYVIEW(dev_priv))
9807 return byt_freq_opcode(dev_priv, val);
9808 else
9809 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9810}
9811
9812void intel_pm_setup(struct drm_i915_private *dev_priv)
9813{
9814 mutex_init(&dev_priv->gt_pm.rps.lock);
9815 mutex_init(&dev_priv->gt_pm.rps.power.mutex);
9816
9817 atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9818
9819 dev_priv->runtime_pm.suspended = false;
9820 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9821}
9822
9823static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9824 const i915_reg_t reg)
9825{
9826 u32 lower, upper, tmp;
9827 int loop = 2;
9828
9829
9830
9831
9832
9833 lockdep_assert_held(&dev_priv->uncore.lock);
9834
9835
9836
9837
9838
9839
9840
9841
9842
9843
9844
9845 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9846 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9847 upper = I915_READ_FW(reg);
9848 do {
9849 tmp = upper;
9850
9851 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9852 _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9853 lower = I915_READ_FW(reg);
9854
9855 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9856 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9857 upper = I915_READ_FW(reg);
9858 } while (upper != tmp && --loop);
9859
9860
9861
9862
9863
9864
9865
9866 return lower | (u64)upper << 8;
9867}
9868
9869u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9870 const i915_reg_t reg)
9871{
9872 struct intel_uncore *uncore = &dev_priv->uncore;
9873 u64 time_hw, prev_hw, overflow_hw;
9874 unsigned int fw_domains;
9875 unsigned long flags;
9876 unsigned int i;
9877 u32 mul, div;
9878
9879 if (!HAS_RC6(dev_priv))
9880 return 0;
9881
9882
9883
9884
9885
9886
9887
9888
9889 i = (i915_mmio_reg_offset(reg) -
9890 i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
9891 if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
9892 return 0;
9893
9894 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
9895
9896 spin_lock_irqsave(&uncore->lock, flags);
9897 intel_uncore_forcewake_get__locked(uncore, fw_domains);
9898
9899
9900 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9901 mul = 1000000;
9902 div = dev_priv->czclk_freq;
9903 overflow_hw = BIT_ULL(40);
9904 time_hw = vlv_residency_raw(dev_priv, reg);
9905 } else {
9906
9907 if (IS_GEN9_LP(dev_priv)) {
9908 mul = 10000;
9909 div = 12;
9910 } else {
9911 mul = 1280;
9912 div = 1;
9913 }
9914
9915 overflow_hw = BIT_ULL(32);
9916 time_hw = intel_uncore_read_fw(uncore, reg);
9917 }
9918
9919
9920
9921
9922
9923
9924
9925 prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
9926 dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
9927
9928
9929 if (time_hw >= prev_hw)
9930 time_hw -= prev_hw;
9931 else
9932 time_hw += overflow_hw - prev_hw;
9933
9934
9935 time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
9936 dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
9937
9938 intel_uncore_forcewake_put__locked(uncore, fw_domains);
9939 spin_unlock_irqrestore(&uncore->lock, flags);
9940
9941 return mul_u64_u32_div(time_hw, mul, div);
9942}
9943
9944u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
9945 i915_reg_t reg)
9946{
9947 return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(dev_priv, reg), 1000);
9948}
9949
9950u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
9951{
9952 u32 cagf;
9953
9954 if (INTEL_GEN(dev_priv) >= 9)
9955 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
9956 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
9957 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
9958 else
9959 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
9960
9961 return cagf;
9962}
9963