1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/cpufreq.h>
29#include <drm/drm_plane_helper.h>
30#include "i915_drv.h"
31#include "intel_drv.h"
32#include "../../../platform/x86/intel_ips.h"
33#include <linux/module.h>
34#include <drm/drm_atomic_helper.h>
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
57{
58 if (HAS_LLC(dev_priv)) {
59
60
61
62
63
64
65
66 I915_WRITE(CHICKEN_PAR1_1,
67 I915_READ(CHICKEN_PAR1_1) |
68 SKL_DE_COMPRESSED_HASH_MODE);
69 }
70
71
72 I915_WRITE(CHICKEN_PAR1_1,
73 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
74
75
76 I915_WRITE(GEN8_CHICKEN_DCPR_1,
77 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
78
79
80
81 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
82 DISP_FBC_WM_DIS |
83 DISP_FBC_MEMORY_WAKE);
84
85
86 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
87 ILK_DPFC_DISABLE_DUMMY0);
88
89 if (IS_SKYLAKE(dev_priv)) {
90
91 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
92 & ~GEN7_DOP_CLOCK_GATE_ENABLE);
93 }
94}
95
96static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
97{
98 gen9_init_clock_gating(dev_priv);
99
100
101 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
102 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
103
104
105
106
107
108 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
109 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
110
111
112
113
114
115 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
116 PWM1_GATING_DIS | PWM2_GATING_DIS);
117}
118
119static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
120{
121 gen9_init_clock_gating(dev_priv);
122
123
124
125
126
127
128 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
129 PWM1_GATING_DIS | PWM2_GATING_DIS);
130
131
132 if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
133 u32 val = I915_READ(CHICKEN_MISC_2);
134 val &= ~(GLK_CL0_PWR_DOWN |
135 GLK_CL1_PWR_DOWN |
136 GLK_CL2_PWR_DOWN);
137 I915_WRITE(CHICKEN_MISC_2, val);
138 }
139
140}
141
142static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
143{
144 u32 tmp;
145
146 tmp = I915_READ(CLKCFG);
147
148 switch (tmp & CLKCFG_FSB_MASK) {
149 case CLKCFG_FSB_533:
150 dev_priv->fsb_freq = 533;
151 break;
152 case CLKCFG_FSB_800:
153 dev_priv->fsb_freq = 800;
154 break;
155 case CLKCFG_FSB_667:
156 dev_priv->fsb_freq = 667;
157 break;
158 case CLKCFG_FSB_400:
159 dev_priv->fsb_freq = 400;
160 break;
161 }
162
163 switch (tmp & CLKCFG_MEM_MASK) {
164 case CLKCFG_MEM_533:
165 dev_priv->mem_freq = 533;
166 break;
167 case CLKCFG_MEM_667:
168 dev_priv->mem_freq = 667;
169 break;
170 case CLKCFG_MEM_800:
171 dev_priv->mem_freq = 800;
172 break;
173 }
174
175
176 tmp = I915_READ(CSHRDDR3CTL);
177 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
178}
179
180static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
181{
182 u16 ddrpll, csipll;
183
184 ddrpll = I915_READ16(DDRMPLL1);
185 csipll = I915_READ16(CSIPLL0);
186
187 switch (ddrpll & 0xff) {
188 case 0xc:
189 dev_priv->mem_freq = 800;
190 break;
191 case 0x10:
192 dev_priv->mem_freq = 1066;
193 break;
194 case 0x14:
195 dev_priv->mem_freq = 1333;
196 break;
197 case 0x18:
198 dev_priv->mem_freq = 1600;
199 break;
200 default:
201 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
202 ddrpll & 0xff);
203 dev_priv->mem_freq = 0;
204 break;
205 }
206
207 dev_priv->ips.r_t = dev_priv->mem_freq;
208
209 switch (csipll & 0x3ff) {
210 case 0x00c:
211 dev_priv->fsb_freq = 3200;
212 break;
213 case 0x00e:
214 dev_priv->fsb_freq = 3733;
215 break;
216 case 0x010:
217 dev_priv->fsb_freq = 4266;
218 break;
219 case 0x012:
220 dev_priv->fsb_freq = 4800;
221 break;
222 case 0x014:
223 dev_priv->fsb_freq = 5333;
224 break;
225 case 0x016:
226 dev_priv->fsb_freq = 5866;
227 break;
228 case 0x018:
229 dev_priv->fsb_freq = 6400;
230 break;
231 default:
232 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
233 csipll & 0x3ff);
234 dev_priv->fsb_freq = 0;
235 break;
236 }
237
238 if (dev_priv->fsb_freq == 3200) {
239 dev_priv->ips.c_m = 0;
240 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
241 dev_priv->ips.c_m = 1;
242 } else {
243 dev_priv->ips.c_m = 2;
244 }
245}
246
247static const struct cxsr_latency cxsr_latency_table[] = {
248 {1, 0, 800, 400, 3382, 33382, 3983, 33983},
249 {1, 0, 800, 667, 3354, 33354, 3807, 33807},
250 {1, 0, 800, 800, 3347, 33347, 3763, 33763},
251 {1, 1, 800, 667, 6420, 36420, 6873, 36873},
252 {1, 1, 800, 800, 5902, 35902, 6318, 36318},
253
254 {1, 0, 667, 400, 3400, 33400, 4021, 34021},
255 {1, 0, 667, 667, 3372, 33372, 3845, 33845},
256 {1, 0, 667, 800, 3386, 33386, 3822, 33822},
257 {1, 1, 667, 667, 6438, 36438, 6911, 36911},
258 {1, 1, 667, 800, 5941, 35941, 6377, 36377},
259
260 {1, 0, 400, 400, 3472, 33472, 4173, 34173},
261 {1, 0, 400, 667, 3443, 33443, 3996, 33996},
262 {1, 0, 400, 800, 3430, 33430, 3946, 33946},
263 {1, 1, 400, 667, 6509, 36509, 7062, 37062},
264 {1, 1, 400, 800, 5985, 35985, 6501, 36501},
265
266 {0, 0, 800, 400, 3438, 33438, 4065, 34065},
267 {0, 0, 800, 667, 3410, 33410, 3889, 33889},
268 {0, 0, 800, 800, 3403, 33403, 3845, 33845},
269 {0, 1, 800, 667, 6476, 36476, 6955, 36955},
270 {0, 1, 800, 800, 5958, 35958, 6400, 36400},
271
272 {0, 0, 667, 400, 3456, 33456, 4103, 34106},
273 {0, 0, 667, 667, 3428, 33428, 3927, 33927},
274 {0, 0, 667, 800, 3443, 33443, 3905, 33905},
275 {0, 1, 667, 667, 6494, 36494, 6993, 36993},
276 {0, 1, 667, 800, 5998, 35998, 6460, 36460},
277
278 {0, 0, 400, 400, 3528, 33528, 4255, 34255},
279 {0, 0, 400, 667, 3500, 33500, 4079, 34079},
280 {0, 0, 400, 800, 3487, 33487, 4029, 34029},
281 {0, 1, 400, 667, 6566, 36566, 7145, 37145},
282 {0, 1, 400, 800, 6042, 36042, 6584, 36584},
283};
284
285static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
286 bool is_ddr3,
287 int fsb,
288 int mem)
289{
290 const struct cxsr_latency *latency;
291 int i;
292
293 if (fsb == 0 || mem == 0)
294 return NULL;
295
296 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
297 latency = &cxsr_latency_table[i];
298 if (is_desktop == latency->is_desktop &&
299 is_ddr3 == latency->is_ddr3 &&
300 fsb == latency->fsb_freq && mem == latency->mem_freq)
301 return latency;
302 }
303
304 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
305
306 return NULL;
307}
308
309static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
310{
311 u32 val;
312
313 mutex_lock(&dev_priv->pcu_lock);
314
315 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
316 if (enable)
317 val &= ~FORCE_DDR_HIGH_FREQ;
318 else
319 val |= FORCE_DDR_HIGH_FREQ;
320 val &= ~FORCE_DDR_LOW_FREQ;
321 val |= FORCE_DDR_FREQ_REQ_ACK;
322 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
323
324 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
325 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
326 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
327
328 mutex_unlock(&dev_priv->pcu_lock);
329}
330
331static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
332{
333 u32 val;
334
335 mutex_lock(&dev_priv->pcu_lock);
336
337 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
338 if (enable)
339 val |= DSP_MAXFIFO_PM5_ENABLE;
340 else
341 val &= ~DSP_MAXFIFO_PM5_ENABLE;
342 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
343
344 mutex_unlock(&dev_priv->pcu_lock);
345}
346
347#define FW_WM(value, plane) \
348 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
349
350static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
351{
352 bool was_enabled;
353 u32 val;
354
355 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
356 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
357 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
358 POSTING_READ(FW_BLC_SELF_VLV);
359 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
360 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
361 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
362 POSTING_READ(FW_BLC_SELF);
363 } else if (IS_PINEVIEW(dev_priv)) {
364 val = I915_READ(DSPFW3);
365 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
366 if (enable)
367 val |= PINEVIEW_SELF_REFRESH_EN;
368 else
369 val &= ~PINEVIEW_SELF_REFRESH_EN;
370 I915_WRITE(DSPFW3, val);
371 POSTING_READ(DSPFW3);
372 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
373 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
374 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
375 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
376 I915_WRITE(FW_BLC_SELF, val);
377 POSTING_READ(FW_BLC_SELF);
378 } else if (IS_I915GM(dev_priv)) {
379
380
381
382
383
384 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
385 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
386 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
387 I915_WRITE(INSTPM, val);
388 POSTING_READ(INSTPM);
389 } else {
390 return false;
391 }
392
393 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
394
395 DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
396 enableddisabled(enable),
397 enableddisabled(was_enabled));
398
399 return was_enabled;
400}
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
440{
441 bool ret;
442
443 mutex_lock(&dev_priv->wm.wm_mutex);
444 ret = _intel_set_memory_cxsr(dev_priv, enable);
445 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
446 dev_priv->wm.vlv.cxsr = enable;
447 else if (IS_G4X(dev_priv))
448 dev_priv->wm.g4x.cxsr = enable;
449 mutex_unlock(&dev_priv->wm.wm_mutex);
450
451 return ret;
452}
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468static const int pessimal_latency_ns = 5000;
469
470#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
471 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
472
473static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
474{
475 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
476 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
477 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
478 enum pipe pipe = crtc->pipe;
479 int sprite0_start, sprite1_start;
480
481 switch (pipe) {
482 uint32_t dsparb, dsparb2, dsparb3;
483 case PIPE_A:
484 dsparb = I915_READ(DSPARB);
485 dsparb2 = I915_READ(DSPARB2);
486 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
487 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
488 break;
489 case PIPE_B:
490 dsparb = I915_READ(DSPARB);
491 dsparb2 = I915_READ(DSPARB2);
492 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
493 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
494 break;
495 case PIPE_C:
496 dsparb2 = I915_READ(DSPARB2);
497 dsparb3 = I915_READ(DSPARB3);
498 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
499 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
500 break;
501 default:
502 MISSING_CASE(pipe);
503 return;
504 }
505
506 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
507 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
508 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
509 fifo_state->plane[PLANE_CURSOR] = 63;
510}
511
512static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
513 enum i9xx_plane_id i9xx_plane)
514{
515 uint32_t dsparb = I915_READ(DSPARB);
516 int size;
517
518 size = dsparb & 0x7f;
519 if (i9xx_plane == PLANE_B)
520 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
521
522 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
523 dsparb, plane_name(i9xx_plane), size);
524
525 return size;
526}
527
528static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
529 enum i9xx_plane_id i9xx_plane)
530{
531 uint32_t dsparb = I915_READ(DSPARB);
532 int size;
533
534 size = dsparb & 0x1ff;
535 if (i9xx_plane == PLANE_B)
536 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
537 size >>= 1;
538
539 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
540 dsparb, plane_name(i9xx_plane), size);
541
542 return size;
543}
544
545static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
546 enum i9xx_plane_id i9xx_plane)
547{
548 uint32_t dsparb = I915_READ(DSPARB);
549 int size;
550
551 size = dsparb & 0x7f;
552 size >>= 2;
553
554 DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
555 dsparb, plane_name(i9xx_plane), size);
556
557 return size;
558}
559
560
561static const struct intel_watermark_params pineview_display_wm = {
562 .fifo_size = PINEVIEW_DISPLAY_FIFO,
563 .max_wm = PINEVIEW_MAX_WM,
564 .default_wm = PINEVIEW_DFT_WM,
565 .guard_size = PINEVIEW_GUARD_WM,
566 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
567};
568static const struct intel_watermark_params pineview_display_hplloff_wm = {
569 .fifo_size = PINEVIEW_DISPLAY_FIFO,
570 .max_wm = PINEVIEW_MAX_WM,
571 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
572 .guard_size = PINEVIEW_GUARD_WM,
573 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
574};
575static const struct intel_watermark_params pineview_cursor_wm = {
576 .fifo_size = PINEVIEW_CURSOR_FIFO,
577 .max_wm = PINEVIEW_CURSOR_MAX_WM,
578 .default_wm = PINEVIEW_CURSOR_DFT_WM,
579 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
580 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
581};
582static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
583 .fifo_size = PINEVIEW_CURSOR_FIFO,
584 .max_wm = PINEVIEW_CURSOR_MAX_WM,
585 .default_wm = PINEVIEW_CURSOR_DFT_WM,
586 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
587 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
588};
589static const struct intel_watermark_params i965_cursor_wm_info = {
590 .fifo_size = I965_CURSOR_FIFO,
591 .max_wm = I965_CURSOR_MAX_WM,
592 .default_wm = I965_CURSOR_DFT_WM,
593 .guard_size = 2,
594 .cacheline_size = I915_FIFO_LINE_SIZE,
595};
596static const struct intel_watermark_params i945_wm_info = {
597 .fifo_size = I945_FIFO_SIZE,
598 .max_wm = I915_MAX_WM,
599 .default_wm = 1,
600 .guard_size = 2,
601 .cacheline_size = I915_FIFO_LINE_SIZE,
602};
603static const struct intel_watermark_params i915_wm_info = {
604 .fifo_size = I915_FIFO_SIZE,
605 .max_wm = I915_MAX_WM,
606 .default_wm = 1,
607 .guard_size = 2,
608 .cacheline_size = I915_FIFO_LINE_SIZE,
609};
610static const struct intel_watermark_params i830_a_wm_info = {
611 .fifo_size = I855GM_FIFO_SIZE,
612 .max_wm = I915_MAX_WM,
613 .default_wm = 1,
614 .guard_size = 2,
615 .cacheline_size = I830_FIFO_LINE_SIZE,
616};
617static const struct intel_watermark_params i830_bc_wm_info = {
618 .fifo_size = I855GM_FIFO_SIZE,
619 .max_wm = I915_MAX_WM/2,
620 .default_wm = 1,
621 .guard_size = 2,
622 .cacheline_size = I830_FIFO_LINE_SIZE,
623};
624static const struct intel_watermark_params i845_wm_info = {
625 .fifo_size = I830_FIFO_SIZE,
626 .max_wm = I915_MAX_WM,
627 .default_wm = 1,
628 .guard_size = 2,
629 .cacheline_size = I830_FIFO_LINE_SIZE,
630};
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665static unsigned int intel_wm_method1(unsigned int pixel_rate,
666 unsigned int cpp,
667 unsigned int latency)
668{
669 uint64_t ret;
670
671 ret = (uint64_t) pixel_rate * cpp * latency;
672 ret = DIV_ROUND_UP_ULL(ret, 10000);
673
674 return ret;
675}
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707static unsigned int intel_wm_method2(unsigned int pixel_rate,
708 unsigned int htotal,
709 unsigned int width,
710 unsigned int cpp,
711 unsigned int latency)
712{
713 unsigned int ret;
714
715
716
717
718
719 if (WARN_ON_ONCE(htotal == 0))
720 htotal = 1;
721
722 ret = (latency * pixel_rate) / (htotal * 10000);
723 ret = (ret + 1) * width * cpp;
724
725 return ret;
726}
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747static unsigned int intel_calculate_wm(int pixel_rate,
748 const struct intel_watermark_params *wm,
749 int fifo_size, int cpp,
750 unsigned int latency_ns)
751{
752 int entries, wm_size;
753
754
755
756
757
758
759
760 entries = intel_wm_method1(pixel_rate, cpp,
761 latency_ns / 100);
762 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
763 wm->guard_size;
764 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
765
766 wm_size = fifo_size - entries;
767 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
768
769
770 if (wm_size > wm->max_wm)
771 wm_size = wm->max_wm;
772 if (wm_size <= 0)
773 wm_size = wm->default_wm;
774
775
776
777
778
779
780
781
782 if (wm_size <= 8)
783 wm_size = 8;
784
785 return wm_size;
786}
787
788static bool is_disabling(int old, int new, int threshold)
789{
790 return old >= threshold && new < threshold;
791}
792
793static bool is_enabling(int old, int new, int threshold)
794{
795 return old < threshold && new >= threshold;
796}
797
798static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
799{
800 return dev_priv->wm.max_level + 1;
801}
802
803static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
804 const struct intel_plane_state *plane_state)
805{
806 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
807
808
809 if (!crtc_state->base.active)
810 return false;
811
812
813
814
815
816
817
818
819
820 if (plane->id == PLANE_CURSOR)
821 return plane_state->base.fb != NULL;
822 else
823 return plane_state->base.visible;
824}
825
826static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
827{
828 struct intel_crtc *crtc, *enabled = NULL;
829
830 for_each_intel_crtc(&dev_priv->drm, crtc) {
831 if (intel_crtc_active(crtc)) {
832 if (enabled)
833 return NULL;
834 enabled = crtc;
835 }
836 }
837
838 return enabled;
839}
840
841static void pineview_update_wm(struct intel_crtc *unused_crtc)
842{
843 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
844 struct intel_crtc *crtc;
845 const struct cxsr_latency *latency;
846 u32 reg;
847 unsigned int wm;
848
849 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
850 dev_priv->is_ddr3,
851 dev_priv->fsb_freq,
852 dev_priv->mem_freq);
853 if (!latency) {
854 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
855 intel_set_memory_cxsr(dev_priv, false);
856 return;
857 }
858
859 crtc = single_enabled_crtc(dev_priv);
860 if (crtc) {
861 const struct drm_display_mode *adjusted_mode =
862 &crtc->config->base.adjusted_mode;
863 const struct drm_framebuffer *fb =
864 crtc->base.primary->state->fb;
865 int cpp = fb->format->cpp[0];
866 int clock = adjusted_mode->crtc_clock;
867
868
869 wm = intel_calculate_wm(clock, &pineview_display_wm,
870 pineview_display_wm.fifo_size,
871 cpp, latency->display_sr);
872 reg = I915_READ(DSPFW1);
873 reg &= ~DSPFW_SR_MASK;
874 reg |= FW_WM(wm, SR);
875 I915_WRITE(DSPFW1, reg);
876 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
877
878
879 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
880 pineview_display_wm.fifo_size,
881 4, latency->cursor_sr);
882 reg = I915_READ(DSPFW3);
883 reg &= ~DSPFW_CURSOR_SR_MASK;
884 reg |= FW_WM(wm, CURSOR_SR);
885 I915_WRITE(DSPFW3, reg);
886
887
888 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
889 pineview_display_hplloff_wm.fifo_size,
890 cpp, latency->display_hpll_disable);
891 reg = I915_READ(DSPFW3);
892 reg &= ~DSPFW_HPLL_SR_MASK;
893 reg |= FW_WM(wm, HPLL_SR);
894 I915_WRITE(DSPFW3, reg);
895
896
897 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
898 pineview_display_hplloff_wm.fifo_size,
899 4, latency->cursor_hpll_disable);
900 reg = I915_READ(DSPFW3);
901 reg &= ~DSPFW_HPLL_CURSOR_MASK;
902 reg |= FW_WM(wm, HPLL_CURSOR);
903 I915_WRITE(DSPFW3, reg);
904 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
905
906 intel_set_memory_cxsr(dev_priv, true);
907 } else {
908 intel_set_memory_cxsr(dev_priv, false);
909 }
910}
911
912
913
914
915
916
917
918
919
920
921
922static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
923{
924 int tlb_miss = fifo_size * 64 - width * cpp * 8;
925
926 return max(0, tlb_miss);
927}
928
929static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
930 const struct g4x_wm_values *wm)
931{
932 enum pipe pipe;
933
934 for_each_pipe(dev_priv, pipe)
935 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
936
937 I915_WRITE(DSPFW1,
938 FW_WM(wm->sr.plane, SR) |
939 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
940 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
941 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
942 I915_WRITE(DSPFW2,
943 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
944 FW_WM(wm->sr.fbc, FBC_SR) |
945 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
946 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
947 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
948 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
949 I915_WRITE(DSPFW3,
950 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
951 FW_WM(wm->sr.cursor, CURSOR_SR) |
952 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
953 FW_WM(wm->hpll.plane, HPLL_SR));
954
955 POSTING_READ(DSPFW1);
956}
957
958#define FW_WM_VLV(value, plane) \
959 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
960
961static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
962 const struct vlv_wm_values *wm)
963{
964 enum pipe pipe;
965
966 for_each_pipe(dev_priv, pipe) {
967 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
968
969 I915_WRITE(VLV_DDL(pipe),
970 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
971 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
972 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
973 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
974 }
975
976
977
978
979
980
981 I915_WRITE(DSPHOWM, 0);
982 I915_WRITE(DSPHOWM1, 0);
983 I915_WRITE(DSPFW4, 0);
984 I915_WRITE(DSPFW5, 0);
985 I915_WRITE(DSPFW6, 0);
986
987 I915_WRITE(DSPFW1,
988 FW_WM(wm->sr.plane, SR) |
989 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
990 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
991 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
992 I915_WRITE(DSPFW2,
993 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
994 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
995 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
996 I915_WRITE(DSPFW3,
997 FW_WM(wm->sr.cursor, CURSOR_SR));
998
999 if (IS_CHERRYVIEW(dev_priv)) {
1000 I915_WRITE(DSPFW7_CHV,
1001 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1002 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1003 I915_WRITE(DSPFW8_CHV,
1004 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1005 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1006 I915_WRITE(DSPFW9_CHV,
1007 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1008 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1009 I915_WRITE(DSPHOWM,
1010 FW_WM(wm->sr.plane >> 9, SR_HI) |
1011 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1012 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1013 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1014 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1015 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1016 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1017 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1018 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1019 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1020 } else {
1021 I915_WRITE(DSPFW7,
1022 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1023 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1024 I915_WRITE(DSPHOWM,
1025 FW_WM(wm->sr.plane >> 9, SR_HI) |
1026 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1027 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1028 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1029 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1030 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1031 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1032 }
1033
1034 POSTING_READ(DSPFW1);
1035}
1036
1037#undef FW_WM_VLV
1038
1039static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1040{
1041
1042 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1043 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1044 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1045
1046 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1047}
1048
1049static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1050{
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065 switch (plane_id) {
1066 case PLANE_CURSOR:
1067 return 63;
1068 case PLANE_PRIMARY:
1069 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1070 case PLANE_SPRITE0:
1071 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1072 default:
1073 MISSING_CASE(plane_id);
1074 return 0;
1075 }
1076}
1077
1078static int g4x_fbc_fifo_size(int level)
1079{
1080 switch (level) {
1081 case G4X_WM_LEVEL_SR:
1082 return 7;
1083 case G4X_WM_LEVEL_HPLL:
1084 return 15;
1085 default:
1086 MISSING_CASE(level);
1087 return 0;
1088 }
1089}
1090
1091static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1092 const struct intel_plane_state *plane_state,
1093 int level)
1094{
1095 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1096 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1097 const struct drm_display_mode *adjusted_mode =
1098 &crtc_state->base.adjusted_mode;
1099 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1100 unsigned int clock, htotal, cpp, width, wm;
1101
1102 if (latency == 0)
1103 return USHRT_MAX;
1104
1105 if (!intel_wm_plane_visible(crtc_state, plane_state))
1106 return 0;
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1120 level != G4X_WM_LEVEL_NORMAL)
1121 cpp = 4;
1122 else
1123 cpp = plane_state->base.fb->format->cpp[0];
1124
1125 clock = adjusted_mode->crtc_clock;
1126 htotal = adjusted_mode->crtc_htotal;
1127
1128 if (plane->id == PLANE_CURSOR)
1129 width = plane_state->base.crtc_w;
1130 else
1131 width = drm_rect_width(&plane_state->base.dst);
1132
1133 if (plane->id == PLANE_CURSOR) {
1134 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1135 } else if (plane->id == PLANE_PRIMARY &&
1136 level == G4X_WM_LEVEL_NORMAL) {
1137 wm = intel_wm_method1(clock, cpp, latency);
1138 } else {
1139 unsigned int small, large;
1140
1141 small = intel_wm_method1(clock, cpp, latency);
1142 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1143
1144 wm = min(small, large);
1145 }
1146
1147 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1148 width, cpp);
1149
1150 wm = DIV_ROUND_UP(wm, 64) + 2;
1151
1152 return min_t(unsigned int, wm, USHRT_MAX);
1153}
1154
1155static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1156 int level, enum plane_id plane_id, u16 value)
1157{
1158 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1159 bool dirty = false;
1160
1161 for (; level < intel_wm_num_levels(dev_priv); level++) {
1162 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1163
1164 dirty |= raw->plane[plane_id] != value;
1165 raw->plane[plane_id] = value;
1166 }
1167
1168 return dirty;
1169}
1170
1171static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1172 int level, u16 value)
1173{
1174 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1175 bool dirty = false;
1176
1177
1178 level = max(level, G4X_WM_LEVEL_SR);
1179
1180 for (; level < intel_wm_num_levels(dev_priv); level++) {
1181 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1182
1183 dirty |= raw->fbc != value;
1184 raw->fbc = value;
1185 }
1186
1187 return dirty;
1188}
1189
1190static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1191 const struct intel_plane_state *pstate,
1192 uint32_t pri_val);
1193
1194static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1195 const struct intel_plane_state *plane_state)
1196{
1197 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1198 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1199 enum plane_id plane_id = plane->id;
1200 bool dirty = false;
1201 int level;
1202
1203 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1204 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1205 if (plane_id == PLANE_PRIMARY)
1206 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1207 goto out;
1208 }
1209
1210 for (level = 0; level < num_levels; level++) {
1211 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1212 int wm, max_wm;
1213
1214 wm = g4x_compute_wm(crtc_state, plane_state, level);
1215 max_wm = g4x_plane_fifo_size(plane_id, level);
1216
1217 if (wm > max_wm)
1218 break;
1219
1220 dirty |= raw->plane[plane_id] != wm;
1221 raw->plane[plane_id] = wm;
1222
1223 if (plane_id != PLANE_PRIMARY ||
1224 level == G4X_WM_LEVEL_NORMAL)
1225 continue;
1226
1227 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1228 raw->plane[plane_id]);
1229 max_wm = g4x_fbc_fifo_size(level);
1230
1231
1232
1233
1234
1235 if (wm > max_wm)
1236 wm = USHRT_MAX;
1237
1238 dirty |= raw->fbc != wm;
1239 raw->fbc = wm;
1240 }
1241
1242
1243 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1244
1245 if (plane_id == PLANE_PRIMARY)
1246 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1247
1248 out:
1249 if (dirty) {
1250 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1251 plane->base.name,
1252 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1253 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1254 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1255
1256 if (plane_id == PLANE_PRIMARY)
1257 DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1258 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1259 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1260 }
1261
1262 return dirty;
1263}
1264
1265static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1266 enum plane_id plane_id, int level)
1267{
1268 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1269
1270 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1271}
1272
1273static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1274 int level)
1275{
1276 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1277
1278 if (level > dev_priv->wm.max_level)
1279 return false;
1280
1281 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1282 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1283 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1284}
1285
1286
1287static void g4x_invalidate_wms(struct intel_crtc *crtc,
1288 struct g4x_wm_state *wm_state, int level)
1289{
1290 if (level <= G4X_WM_LEVEL_NORMAL) {
1291 enum plane_id plane_id;
1292
1293 for_each_plane_id_on_crtc(crtc, plane_id)
1294 wm_state->wm.plane[plane_id] = USHRT_MAX;
1295 }
1296
1297 if (level <= G4X_WM_LEVEL_SR) {
1298 wm_state->cxsr = false;
1299 wm_state->sr.cursor = USHRT_MAX;
1300 wm_state->sr.plane = USHRT_MAX;
1301 wm_state->sr.fbc = USHRT_MAX;
1302 }
1303
1304 if (level <= G4X_WM_LEVEL_HPLL) {
1305 wm_state->hpll_en = false;
1306 wm_state->hpll.cursor = USHRT_MAX;
1307 wm_state->hpll.plane = USHRT_MAX;
1308 wm_state->hpll.fbc = USHRT_MAX;
1309 }
1310}
1311
1312static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1313{
1314 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1315 struct intel_atomic_state *state =
1316 to_intel_atomic_state(crtc_state->base.state);
1317 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1318 int num_active_planes = hweight32(crtc_state->active_planes &
1319 ~BIT(PLANE_CURSOR));
1320 const struct g4x_pipe_wm *raw;
1321 const struct intel_plane_state *old_plane_state;
1322 const struct intel_plane_state *new_plane_state;
1323 struct intel_plane *plane;
1324 enum plane_id plane_id;
1325 int i, level;
1326 unsigned int dirty = 0;
1327
1328 for_each_oldnew_intel_plane_in_state(state, plane,
1329 old_plane_state,
1330 new_plane_state, i) {
1331 if (new_plane_state->base.crtc != &crtc->base &&
1332 old_plane_state->base.crtc != &crtc->base)
1333 continue;
1334
1335 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1336 dirty |= BIT(plane->id);
1337 }
1338
1339 if (!dirty)
1340 return 0;
1341
1342 level = G4X_WM_LEVEL_NORMAL;
1343 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1344 goto out;
1345
1346 raw = &crtc_state->wm.g4x.raw[level];
1347 for_each_plane_id_on_crtc(crtc, plane_id)
1348 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1349
1350 level = G4X_WM_LEVEL_SR;
1351
1352 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1353 goto out;
1354
1355 raw = &crtc_state->wm.g4x.raw[level];
1356 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1357 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1358 wm_state->sr.fbc = raw->fbc;
1359
1360 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1361
1362 level = G4X_WM_LEVEL_HPLL;
1363
1364 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1365 goto out;
1366
1367 raw = &crtc_state->wm.g4x.raw[level];
1368 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1369 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1370 wm_state->hpll.fbc = raw->fbc;
1371
1372 wm_state->hpll_en = wm_state->cxsr;
1373
1374 level++;
1375
1376 out:
1377 if (level == G4X_WM_LEVEL_NORMAL)
1378 return -EINVAL;
1379
1380
1381 g4x_invalidate_wms(crtc, wm_state, level);
1382
1383
1384
1385
1386
1387
1388
1389 wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1390
1391 if (level >= G4X_WM_LEVEL_SR &&
1392 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1393 wm_state->fbc_en = false;
1394 else if (level >= G4X_WM_LEVEL_HPLL &&
1395 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1396 wm_state->fbc_en = false;
1397
1398 return 0;
1399}
1400
1401static int g4x_compute_intermediate_wm(struct drm_device *dev,
1402 struct intel_crtc *crtc,
1403 struct intel_crtc_state *new_crtc_state)
1404{
1405 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1406 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1407 struct intel_atomic_state *intel_state =
1408 to_intel_atomic_state(new_crtc_state->base.state);
1409 const struct intel_crtc_state *old_crtc_state =
1410 intel_atomic_get_old_crtc_state(intel_state, crtc);
1411 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1412 enum plane_id plane_id;
1413
1414 if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1415 *intermediate = *optimal;
1416
1417 intermediate->cxsr = false;
1418 intermediate->hpll_en = false;
1419 goto out;
1420 }
1421
1422 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1423 !new_crtc_state->disable_cxsr;
1424 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1425 !new_crtc_state->disable_cxsr;
1426 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1427
1428 for_each_plane_id_on_crtc(crtc, plane_id) {
1429 intermediate->wm.plane[plane_id] =
1430 max(optimal->wm.plane[plane_id],
1431 active->wm.plane[plane_id]);
1432
1433 WARN_ON(intermediate->wm.plane[plane_id] >
1434 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1435 }
1436
1437 intermediate->sr.plane = max(optimal->sr.plane,
1438 active->sr.plane);
1439 intermediate->sr.cursor = max(optimal->sr.cursor,
1440 active->sr.cursor);
1441 intermediate->sr.fbc = max(optimal->sr.fbc,
1442 active->sr.fbc);
1443
1444 intermediate->hpll.plane = max(optimal->hpll.plane,
1445 active->hpll.plane);
1446 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1447 active->hpll.cursor);
1448 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1449 active->hpll.fbc);
1450
1451 WARN_ON((intermediate->sr.plane >
1452 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1453 intermediate->sr.cursor >
1454 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1455 intermediate->cxsr);
1456 WARN_ON((intermediate->sr.plane >
1457 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1458 intermediate->sr.cursor >
1459 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1460 intermediate->hpll_en);
1461
1462 WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1463 intermediate->fbc_en && intermediate->cxsr);
1464 WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1465 intermediate->fbc_en && intermediate->hpll_en);
1466
1467out:
1468
1469
1470
1471
1472 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1473 new_crtc_state->wm.need_postvbl_update = true;
1474
1475 return 0;
1476}
1477
1478static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1479 struct g4x_wm_values *wm)
1480{
1481 struct intel_crtc *crtc;
1482 int num_active_crtcs = 0;
1483
1484 wm->cxsr = true;
1485 wm->hpll_en = true;
1486 wm->fbc_en = true;
1487
1488 for_each_intel_crtc(&dev_priv->drm, crtc) {
1489 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1490
1491 if (!crtc->active)
1492 continue;
1493
1494 if (!wm_state->cxsr)
1495 wm->cxsr = false;
1496 if (!wm_state->hpll_en)
1497 wm->hpll_en = false;
1498 if (!wm_state->fbc_en)
1499 wm->fbc_en = false;
1500
1501 num_active_crtcs++;
1502 }
1503
1504 if (num_active_crtcs != 1) {
1505 wm->cxsr = false;
1506 wm->hpll_en = false;
1507 wm->fbc_en = false;
1508 }
1509
1510 for_each_intel_crtc(&dev_priv->drm, crtc) {
1511 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1512 enum pipe pipe = crtc->pipe;
1513
1514 wm->pipe[pipe] = wm_state->wm;
1515 if (crtc->active && wm->cxsr)
1516 wm->sr = wm_state->sr;
1517 if (crtc->active && wm->hpll_en)
1518 wm->hpll = wm_state->hpll;
1519 }
1520}
1521
1522static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1523{
1524 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1525 struct g4x_wm_values new_wm = {};
1526
1527 g4x_merge_wm(dev_priv, &new_wm);
1528
1529 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1530 return;
1531
1532 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1533 _intel_set_memory_cxsr(dev_priv, false);
1534
1535 g4x_write_wm_values(dev_priv, &new_wm);
1536
1537 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1538 _intel_set_memory_cxsr(dev_priv, true);
1539
1540 *old_wm = new_wm;
1541}
1542
1543static void g4x_initial_watermarks(struct intel_atomic_state *state,
1544 struct intel_crtc_state *crtc_state)
1545{
1546 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1547 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1548
1549 mutex_lock(&dev_priv->wm.wm_mutex);
1550 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1551 g4x_program_watermarks(dev_priv);
1552 mutex_unlock(&dev_priv->wm.wm_mutex);
1553}
1554
1555static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1556 struct intel_crtc_state *crtc_state)
1557{
1558 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1559 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1560
1561 if (!crtc_state->wm.need_postvbl_update)
1562 return;
1563
1564 mutex_lock(&dev_priv->wm.wm_mutex);
1565 intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1566 g4x_program_watermarks(dev_priv);
1567 mutex_unlock(&dev_priv->wm.wm_mutex);
1568}
1569
1570
1571static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1572 unsigned int htotal,
1573 unsigned int width,
1574 unsigned int cpp,
1575 unsigned int latency)
1576{
1577 unsigned int ret;
1578
1579 ret = intel_wm_method2(pixel_rate, htotal,
1580 width, cpp, latency);
1581 ret = DIV_ROUND_UP(ret, 64);
1582
1583 return ret;
1584}
1585
1586static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1587{
1588
1589 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1590
1591 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1592
1593 if (IS_CHERRYVIEW(dev_priv)) {
1594 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1595 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1596
1597 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1598 }
1599}
1600
1601static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1602 const struct intel_plane_state *plane_state,
1603 int level)
1604{
1605 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1606 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1607 const struct drm_display_mode *adjusted_mode =
1608 &crtc_state->base.adjusted_mode;
1609 unsigned int clock, htotal, cpp, width, wm;
1610
1611 if (dev_priv->wm.pri_latency[level] == 0)
1612 return USHRT_MAX;
1613
1614 if (!intel_wm_plane_visible(crtc_state, plane_state))
1615 return 0;
1616
1617 cpp = plane_state->base.fb->format->cpp[0];
1618 clock = adjusted_mode->crtc_clock;
1619 htotal = adjusted_mode->crtc_htotal;
1620 width = crtc_state->pipe_src_w;
1621
1622 if (plane->id == PLANE_CURSOR) {
1623
1624
1625
1626
1627
1628
1629 wm = 63;
1630 } else {
1631 wm = vlv_wm_method2(clock, htotal, width, cpp,
1632 dev_priv->wm.pri_latency[level] * 10);
1633 }
1634
1635 return min_t(unsigned int, wm, USHRT_MAX);
1636}
1637
1638static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1639{
1640 return (active_planes & (BIT(PLANE_SPRITE0) |
1641 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1642}
1643
1644static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1645{
1646 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1647 const struct g4x_pipe_wm *raw =
1648 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1649 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1650 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1651 int num_active_planes = hweight32(active_planes);
1652 const int fifo_size = 511;
1653 int fifo_extra, fifo_left = fifo_size;
1654 int sprite0_fifo_extra = 0;
1655 unsigned int total_rate;
1656 enum plane_id plane_id;
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666 if (vlv_need_sprite0_fifo_workaround(active_planes))
1667 sprite0_fifo_extra = 1;
1668
1669 total_rate = raw->plane[PLANE_PRIMARY] +
1670 raw->plane[PLANE_SPRITE0] +
1671 raw->plane[PLANE_SPRITE1] +
1672 sprite0_fifo_extra;
1673
1674 if (total_rate > fifo_size)
1675 return -EINVAL;
1676
1677 if (total_rate == 0)
1678 total_rate = 1;
1679
1680 for_each_plane_id_on_crtc(crtc, plane_id) {
1681 unsigned int rate;
1682
1683 if ((active_planes & BIT(plane_id)) == 0) {
1684 fifo_state->plane[plane_id] = 0;
1685 continue;
1686 }
1687
1688 rate = raw->plane[plane_id];
1689 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1690 fifo_left -= fifo_state->plane[plane_id];
1691 }
1692
1693 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1694 fifo_left -= sprite0_fifo_extra;
1695
1696 fifo_state->plane[PLANE_CURSOR] = 63;
1697
1698 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1699
1700
1701 for_each_plane_id_on_crtc(crtc, plane_id) {
1702 int plane_extra;
1703
1704 if (fifo_left == 0)
1705 break;
1706
1707 if ((active_planes & BIT(plane_id)) == 0)
1708 continue;
1709
1710 plane_extra = min(fifo_extra, fifo_left);
1711 fifo_state->plane[plane_id] += plane_extra;
1712 fifo_left -= plane_extra;
1713 }
1714
1715 WARN_ON(active_planes != 0 && fifo_left != 0);
1716
1717
1718 if (active_planes == 0) {
1719 WARN_ON(fifo_left != fifo_size);
1720 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1721 }
1722
1723 return 0;
1724}
1725
1726
1727static void vlv_invalidate_wms(struct intel_crtc *crtc,
1728 struct vlv_wm_state *wm_state, int level)
1729{
1730 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1731
1732 for (; level < intel_wm_num_levels(dev_priv); level++) {
1733 enum plane_id plane_id;
1734
1735 for_each_plane_id_on_crtc(crtc, plane_id)
1736 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1737
1738 wm_state->sr[level].cursor = USHRT_MAX;
1739 wm_state->sr[level].plane = USHRT_MAX;
1740 }
1741}
1742
1743static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1744{
1745 if (wm > fifo_size)
1746 return USHRT_MAX;
1747 else
1748 return fifo_size - wm;
1749}
1750
1751
1752
1753
1754
1755static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1756 int level, enum plane_id plane_id, u16 value)
1757{
1758 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1759 int num_levels = intel_wm_num_levels(dev_priv);
1760 bool dirty = false;
1761
1762 for (; level < num_levels; level++) {
1763 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1764
1765 dirty |= raw->plane[plane_id] != value;
1766 raw->plane[plane_id] = value;
1767 }
1768
1769 return dirty;
1770}
1771
1772static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1773 const struct intel_plane_state *plane_state)
1774{
1775 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1776 enum plane_id plane_id = plane->id;
1777 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1778 int level;
1779 bool dirty = false;
1780
1781 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1782 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1783 goto out;
1784 }
1785
1786 for (level = 0; level < num_levels; level++) {
1787 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1788 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1789 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1790
1791 if (wm > max_wm)
1792 break;
1793
1794 dirty |= raw->plane[plane_id] != wm;
1795 raw->plane[plane_id] = wm;
1796 }
1797
1798
1799 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1800
1801out:
1802 if (dirty)
1803 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1804 plane->base.name,
1805 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1806 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1807 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1808
1809 return dirty;
1810}
1811
1812static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1813 enum plane_id plane_id, int level)
1814{
1815 const struct g4x_pipe_wm *raw =
1816 &crtc_state->wm.vlv.raw[level];
1817 const struct vlv_fifo_state *fifo_state =
1818 &crtc_state->wm.vlv.fifo_state;
1819
1820 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1821}
1822
1823static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1824{
1825 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1826 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1827 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1828 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1829}
1830
1831static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1832{
1833 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1834 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1835 struct intel_atomic_state *state =
1836 to_intel_atomic_state(crtc_state->base.state);
1837 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1838 const struct vlv_fifo_state *fifo_state =
1839 &crtc_state->wm.vlv.fifo_state;
1840 int num_active_planes = hweight32(crtc_state->active_planes &
1841 ~BIT(PLANE_CURSOR));
1842 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1843 const struct intel_plane_state *old_plane_state;
1844 const struct intel_plane_state *new_plane_state;
1845 struct intel_plane *plane;
1846 enum plane_id plane_id;
1847 int level, ret, i;
1848 unsigned int dirty = 0;
1849
1850 for_each_oldnew_intel_plane_in_state(state, plane,
1851 old_plane_state,
1852 new_plane_state, i) {
1853 if (new_plane_state->base.crtc != &crtc->base &&
1854 old_plane_state->base.crtc != &crtc->base)
1855 continue;
1856
1857 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1858 dirty |= BIT(plane->id);
1859 }
1860
1861
1862
1863
1864
1865
1866
1867 if (needs_modeset)
1868 crtc_state->fifo_changed = true;
1869
1870 if (!dirty)
1871 return 0;
1872
1873
1874 if (dirty & ~BIT(PLANE_CURSOR)) {
1875 const struct intel_crtc_state *old_crtc_state =
1876 intel_atomic_get_old_crtc_state(state, crtc);
1877 const struct vlv_fifo_state *old_fifo_state =
1878 &old_crtc_state->wm.vlv.fifo_state;
1879
1880 ret = vlv_compute_fifo(crtc_state);
1881 if (ret)
1882 return ret;
1883
1884 if (needs_modeset ||
1885 memcmp(old_fifo_state, fifo_state,
1886 sizeof(*fifo_state)) != 0)
1887 crtc_state->fifo_changed = true;
1888 }
1889
1890
1891 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1892
1893
1894
1895
1896
1897 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1898
1899 for (level = 0; level < wm_state->num_levels; level++) {
1900 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1901 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1902
1903 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1904 break;
1905
1906 for_each_plane_id_on_crtc(crtc, plane_id) {
1907 wm_state->wm[level].plane[plane_id] =
1908 vlv_invert_wm_value(raw->plane[plane_id],
1909 fifo_state->plane[plane_id]);
1910 }
1911
1912 wm_state->sr[level].plane =
1913 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1914 raw->plane[PLANE_SPRITE0],
1915 raw->plane[PLANE_SPRITE1]),
1916 sr_fifo_size);
1917
1918 wm_state->sr[level].cursor =
1919 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1920 63);
1921 }
1922
1923 if (level == 0)
1924 return -EINVAL;
1925
1926
1927 wm_state->num_levels = level;
1928
1929
1930 vlv_invalidate_wms(crtc, wm_state, level);
1931
1932 return 0;
1933}
1934
1935#define VLV_FIFO(plane, value) \
1936 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1937
1938static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1939 struct intel_crtc_state *crtc_state)
1940{
1941 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1942 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1943 const struct vlv_fifo_state *fifo_state =
1944 &crtc_state->wm.vlv.fifo_state;
1945 int sprite0_start, sprite1_start, fifo_size;
1946
1947 if (!crtc_state->fifo_changed)
1948 return;
1949
1950 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1951 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1952 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1953
1954 WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1955 WARN_ON(fifo_size != 511);
1956
1957 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968 spin_lock(&dev_priv->uncore.lock);
1969
1970 switch (crtc->pipe) {
1971 uint32_t dsparb, dsparb2, dsparb3;
1972 case PIPE_A:
1973 dsparb = I915_READ_FW(DSPARB);
1974 dsparb2 = I915_READ_FW(DSPARB2);
1975
1976 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1977 VLV_FIFO(SPRITEB, 0xff));
1978 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1979 VLV_FIFO(SPRITEB, sprite1_start));
1980
1981 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1982 VLV_FIFO(SPRITEB_HI, 0x1));
1983 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1984 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1985
1986 I915_WRITE_FW(DSPARB, dsparb);
1987 I915_WRITE_FW(DSPARB2, dsparb2);
1988 break;
1989 case PIPE_B:
1990 dsparb = I915_READ_FW(DSPARB);
1991 dsparb2 = I915_READ_FW(DSPARB2);
1992
1993 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1994 VLV_FIFO(SPRITED, 0xff));
1995 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1996 VLV_FIFO(SPRITED, sprite1_start));
1997
1998 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1999 VLV_FIFO(SPRITED_HI, 0xff));
2000 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2001 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2002
2003 I915_WRITE_FW(DSPARB, dsparb);
2004 I915_WRITE_FW(DSPARB2, dsparb2);
2005 break;
2006 case PIPE_C:
2007 dsparb3 = I915_READ_FW(DSPARB3);
2008 dsparb2 = I915_READ_FW(DSPARB2);
2009
2010 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2011 VLV_FIFO(SPRITEF, 0xff));
2012 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2013 VLV_FIFO(SPRITEF, sprite1_start));
2014
2015 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2016 VLV_FIFO(SPRITEF_HI, 0xff));
2017 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2018 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2019
2020 I915_WRITE_FW(DSPARB3, dsparb3);
2021 I915_WRITE_FW(DSPARB2, dsparb2);
2022 break;
2023 default:
2024 break;
2025 }
2026
2027 POSTING_READ_FW(DSPARB);
2028
2029 spin_unlock(&dev_priv->uncore.lock);
2030}
2031
2032#undef VLV_FIFO
2033
2034static int vlv_compute_intermediate_wm(struct drm_device *dev,
2035 struct intel_crtc *crtc,
2036 struct intel_crtc_state *new_crtc_state)
2037{
2038 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2039 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2040 struct intel_atomic_state *intel_state =
2041 to_intel_atomic_state(new_crtc_state->base.state);
2042 const struct intel_crtc_state *old_crtc_state =
2043 intel_atomic_get_old_crtc_state(intel_state, crtc);
2044 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2045 int level;
2046
2047 if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2048 *intermediate = *optimal;
2049
2050 intermediate->cxsr = false;
2051 goto out;
2052 }
2053
2054 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2055 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2056 !new_crtc_state->disable_cxsr;
2057
2058 for (level = 0; level < intermediate->num_levels; level++) {
2059 enum plane_id plane_id;
2060
2061 for_each_plane_id_on_crtc(crtc, plane_id) {
2062 intermediate->wm[level].plane[plane_id] =
2063 min(optimal->wm[level].plane[plane_id],
2064 active->wm[level].plane[plane_id]);
2065 }
2066
2067 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2068 active->sr[level].plane);
2069 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2070 active->sr[level].cursor);
2071 }
2072
2073 vlv_invalidate_wms(crtc, intermediate, level);
2074
2075out:
2076
2077
2078
2079
2080 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2081 new_crtc_state->wm.need_postvbl_update = true;
2082
2083 return 0;
2084}
2085
2086static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2087 struct vlv_wm_values *wm)
2088{
2089 struct intel_crtc *crtc;
2090 int num_active_crtcs = 0;
2091
2092 wm->level = dev_priv->wm.max_level;
2093 wm->cxsr = true;
2094
2095 for_each_intel_crtc(&dev_priv->drm, crtc) {
2096 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2097
2098 if (!crtc->active)
2099 continue;
2100
2101 if (!wm_state->cxsr)
2102 wm->cxsr = false;
2103
2104 num_active_crtcs++;
2105 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2106 }
2107
2108 if (num_active_crtcs != 1)
2109 wm->cxsr = false;
2110
2111 if (num_active_crtcs > 1)
2112 wm->level = VLV_WM_LEVEL_PM2;
2113
2114 for_each_intel_crtc(&dev_priv->drm, crtc) {
2115 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2116 enum pipe pipe = crtc->pipe;
2117
2118 wm->pipe[pipe] = wm_state->wm[wm->level];
2119 if (crtc->active && wm->cxsr)
2120 wm->sr = wm_state->sr[wm->level];
2121
2122 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2123 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2124 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2125 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2126 }
2127}
2128
2129static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2130{
2131 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2132 struct vlv_wm_values new_wm = {};
2133
2134 vlv_merge_wm(dev_priv, &new_wm);
2135
2136 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2137 return;
2138
2139 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2140 chv_set_memory_dvfs(dev_priv, false);
2141
2142 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2143 chv_set_memory_pm5(dev_priv, false);
2144
2145 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2146 _intel_set_memory_cxsr(dev_priv, false);
2147
2148 vlv_write_wm_values(dev_priv, &new_wm);
2149
2150 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2151 _intel_set_memory_cxsr(dev_priv, true);
2152
2153 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2154 chv_set_memory_pm5(dev_priv, true);
2155
2156 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2157 chv_set_memory_dvfs(dev_priv, true);
2158
2159 *old_wm = new_wm;
2160}
2161
2162static void vlv_initial_watermarks(struct intel_atomic_state *state,
2163 struct intel_crtc_state *crtc_state)
2164{
2165 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2166 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2167
2168 mutex_lock(&dev_priv->wm.wm_mutex);
2169 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2170 vlv_program_watermarks(dev_priv);
2171 mutex_unlock(&dev_priv->wm.wm_mutex);
2172}
2173
2174static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2175 struct intel_crtc_state *crtc_state)
2176{
2177 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2178 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2179
2180 if (!crtc_state->wm.need_postvbl_update)
2181 return;
2182
2183 mutex_lock(&dev_priv->wm.wm_mutex);
2184 intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2185 vlv_program_watermarks(dev_priv);
2186 mutex_unlock(&dev_priv->wm.wm_mutex);
2187}
2188
2189static void i965_update_wm(struct intel_crtc *unused_crtc)
2190{
2191 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2192 struct intel_crtc *crtc;
2193 int srwm = 1;
2194 int cursor_sr = 16;
2195 bool cxsr_enabled;
2196
2197
2198 crtc = single_enabled_crtc(dev_priv);
2199 if (crtc) {
2200
2201 static const int sr_latency_ns = 12000;
2202 const struct drm_display_mode *adjusted_mode =
2203 &crtc->config->base.adjusted_mode;
2204 const struct drm_framebuffer *fb =
2205 crtc->base.primary->state->fb;
2206 int clock = adjusted_mode->crtc_clock;
2207 int htotal = adjusted_mode->crtc_htotal;
2208 int hdisplay = crtc->config->pipe_src_w;
2209 int cpp = fb->format->cpp[0];
2210 int entries;
2211
2212 entries = intel_wm_method2(clock, htotal,
2213 hdisplay, cpp, sr_latency_ns / 100);
2214 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2215 srwm = I965_FIFO_SIZE - entries;
2216 if (srwm < 0)
2217 srwm = 1;
2218 srwm &= 0x1ff;
2219 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2220 entries, srwm);
2221
2222 entries = intel_wm_method2(clock, htotal,
2223 crtc->base.cursor->state->crtc_w, 4,
2224 sr_latency_ns / 100);
2225 entries = DIV_ROUND_UP(entries,
2226 i965_cursor_wm_info.cacheline_size) +
2227 i965_cursor_wm_info.guard_size;
2228
2229 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2230 if (cursor_sr > i965_cursor_wm_info.max_wm)
2231 cursor_sr = i965_cursor_wm_info.max_wm;
2232
2233 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2234 "cursor %d\n", srwm, cursor_sr);
2235
2236 cxsr_enabled = true;
2237 } else {
2238 cxsr_enabled = false;
2239
2240 intel_set_memory_cxsr(dev_priv, false);
2241 }
2242
2243 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2244 srwm);
2245
2246
2247 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2248 FW_WM(8, CURSORB) |
2249 FW_WM(8, PLANEB) |
2250 FW_WM(8, PLANEA));
2251 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2252 FW_WM(8, PLANEC_OLD));
2253
2254 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2255
2256 if (cxsr_enabled)
2257 intel_set_memory_cxsr(dev_priv, true);
2258}
2259
2260#undef FW_WM
2261
2262static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2263{
2264 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2265 const struct intel_watermark_params *wm_info;
2266 uint32_t fwater_lo;
2267 uint32_t fwater_hi;
2268 int cwm, srwm = 1;
2269 int fifo_size;
2270 int planea_wm, planeb_wm;
2271 struct intel_crtc *crtc, *enabled = NULL;
2272
2273 if (IS_I945GM(dev_priv))
2274 wm_info = &i945_wm_info;
2275 else if (!IS_GEN2(dev_priv))
2276 wm_info = &i915_wm_info;
2277 else
2278 wm_info = &i830_a_wm_info;
2279
2280 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2281 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2282 if (intel_crtc_active(crtc)) {
2283 const struct drm_display_mode *adjusted_mode =
2284 &crtc->config->base.adjusted_mode;
2285 const struct drm_framebuffer *fb =
2286 crtc->base.primary->state->fb;
2287 int cpp;
2288
2289 if (IS_GEN2(dev_priv))
2290 cpp = 4;
2291 else
2292 cpp = fb->format->cpp[0];
2293
2294 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2295 wm_info, fifo_size, cpp,
2296 pessimal_latency_ns);
2297 enabled = crtc;
2298 } else {
2299 planea_wm = fifo_size - wm_info->guard_size;
2300 if (planea_wm > (long)wm_info->max_wm)
2301 planea_wm = wm_info->max_wm;
2302 }
2303
2304 if (IS_GEN2(dev_priv))
2305 wm_info = &i830_bc_wm_info;
2306
2307 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2308 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2309 if (intel_crtc_active(crtc)) {
2310 const struct drm_display_mode *adjusted_mode =
2311 &crtc->config->base.adjusted_mode;
2312 const struct drm_framebuffer *fb =
2313 crtc->base.primary->state->fb;
2314 int cpp;
2315
2316 if (IS_GEN2(dev_priv))
2317 cpp = 4;
2318 else
2319 cpp = fb->format->cpp[0];
2320
2321 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2322 wm_info, fifo_size, cpp,
2323 pessimal_latency_ns);
2324 if (enabled == NULL)
2325 enabled = crtc;
2326 else
2327 enabled = NULL;
2328 } else {
2329 planeb_wm = fifo_size - wm_info->guard_size;
2330 if (planeb_wm > (long)wm_info->max_wm)
2331 planeb_wm = wm_info->max_wm;
2332 }
2333
2334 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2335
2336 if (IS_I915GM(dev_priv) && enabled) {
2337 struct drm_i915_gem_object *obj;
2338
2339 obj = intel_fb_obj(enabled->base.primary->state->fb);
2340
2341
2342 if (!i915_gem_object_is_tiled(obj))
2343 enabled = NULL;
2344 }
2345
2346
2347
2348
2349 cwm = 2;
2350
2351
2352 intel_set_memory_cxsr(dev_priv, false);
2353
2354
2355 if (HAS_FW_BLC(dev_priv) && enabled) {
2356
2357 static const int sr_latency_ns = 6000;
2358 const struct drm_display_mode *adjusted_mode =
2359 &enabled->config->base.adjusted_mode;
2360 const struct drm_framebuffer *fb =
2361 enabled->base.primary->state->fb;
2362 int clock = adjusted_mode->crtc_clock;
2363 int htotal = adjusted_mode->crtc_htotal;
2364 int hdisplay = enabled->config->pipe_src_w;
2365 int cpp;
2366 int entries;
2367
2368 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2369 cpp = 4;
2370 else
2371 cpp = fb->format->cpp[0];
2372
2373 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2374 sr_latency_ns / 100);
2375 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2376 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2377 srwm = wm_info->fifo_size - entries;
2378 if (srwm < 0)
2379 srwm = 1;
2380
2381 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2382 I915_WRITE(FW_BLC_SELF,
2383 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2384 else
2385 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2386 }
2387
2388 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2389 planea_wm, planeb_wm, cwm, srwm);
2390
2391 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2392 fwater_hi = (cwm & 0x1f);
2393
2394
2395 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2396 fwater_hi = fwater_hi | (1 << 8);
2397
2398 I915_WRITE(FW_BLC, fwater_lo);
2399 I915_WRITE(FW_BLC2, fwater_hi);
2400
2401 if (enabled)
2402 intel_set_memory_cxsr(dev_priv, true);
2403}
2404
2405static void i845_update_wm(struct intel_crtc *unused_crtc)
2406{
2407 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2408 struct intel_crtc *crtc;
2409 const struct drm_display_mode *adjusted_mode;
2410 uint32_t fwater_lo;
2411 int planea_wm;
2412
2413 crtc = single_enabled_crtc(dev_priv);
2414 if (crtc == NULL)
2415 return;
2416
2417 adjusted_mode = &crtc->config->base.adjusted_mode;
2418 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2419 &i845_wm_info,
2420 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2421 4, pessimal_latency_ns);
2422 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2423 fwater_lo |= (3<<8) | planea_wm;
2424
2425 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2426
2427 I915_WRITE(FW_BLC, fwater_lo);
2428}
2429
2430
2431static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2432 unsigned int cpp,
2433 unsigned int latency)
2434{
2435 unsigned int ret;
2436
2437 ret = intel_wm_method1(pixel_rate, cpp, latency);
2438 ret = DIV_ROUND_UP(ret, 64) + 2;
2439
2440 return ret;
2441}
2442
2443
2444static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2445 unsigned int htotal,
2446 unsigned int width,
2447 unsigned int cpp,
2448 unsigned int latency)
2449{
2450 unsigned int ret;
2451
2452 ret = intel_wm_method2(pixel_rate, htotal,
2453 width, cpp, latency);
2454 ret = DIV_ROUND_UP(ret, 64) + 2;
2455
2456 return ret;
2457}
2458
2459static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
2460 uint8_t cpp)
2461{
2462
2463
2464
2465
2466
2467
2468 if (WARN_ON(!cpp))
2469 return 0;
2470 if (WARN_ON(!horiz_pixels))
2471 return 0;
2472
2473 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2474}
2475
2476struct ilk_wm_maximums {
2477 uint16_t pri;
2478 uint16_t spr;
2479 uint16_t cur;
2480 uint16_t fbc;
2481};
2482
2483
2484
2485
2486
2487static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
2488 const struct intel_plane_state *pstate,
2489 uint32_t mem_value,
2490 bool is_lp)
2491{
2492 uint32_t method1, method2;
2493 int cpp;
2494
2495 if (!intel_wm_plane_visible(cstate, pstate))
2496 return 0;
2497
2498 cpp = pstate->base.fb->format->cpp[0];
2499
2500 method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2501
2502 if (!is_lp)
2503 return method1;
2504
2505 method2 = ilk_wm_method2(cstate->pixel_rate,
2506 cstate->base.adjusted_mode.crtc_htotal,
2507 drm_rect_width(&pstate->base.dst),
2508 cpp, mem_value);
2509
2510 return min(method1, method2);
2511}
2512
2513
2514
2515
2516
2517static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2518 const struct intel_plane_state *pstate,
2519 uint32_t mem_value)
2520{
2521 uint32_t method1, method2;
2522 int cpp;
2523
2524 if (!intel_wm_plane_visible(cstate, pstate))
2525 return 0;
2526
2527 cpp = pstate->base.fb->format->cpp[0];
2528
2529 method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2530 method2 = ilk_wm_method2(cstate->pixel_rate,
2531 cstate->base.adjusted_mode.crtc_htotal,
2532 drm_rect_width(&pstate->base.dst),
2533 cpp, mem_value);
2534 return min(method1, method2);
2535}
2536
2537
2538
2539
2540
2541static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2542 const struct intel_plane_state *pstate,
2543 uint32_t mem_value)
2544{
2545 int cpp;
2546
2547 if (!intel_wm_plane_visible(cstate, pstate))
2548 return 0;
2549
2550 cpp = pstate->base.fb->format->cpp[0];
2551
2552 return ilk_wm_method2(cstate->pixel_rate,
2553 cstate->base.adjusted_mode.crtc_htotal,
2554 pstate->base.crtc_w, cpp, mem_value);
2555}
2556
2557
2558static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2559 const struct intel_plane_state *pstate,
2560 uint32_t pri_val)
2561{
2562 int cpp;
2563
2564 if (!intel_wm_plane_visible(cstate, pstate))
2565 return 0;
2566
2567 cpp = pstate->base.fb->format->cpp[0];
2568
2569 return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2570}
2571
2572static unsigned int
2573ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2574{
2575 if (INTEL_GEN(dev_priv) >= 8)
2576 return 3072;
2577 else if (INTEL_GEN(dev_priv) >= 7)
2578 return 768;
2579 else
2580 return 512;
2581}
2582
2583static unsigned int
2584ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2585 int level, bool is_sprite)
2586{
2587 if (INTEL_GEN(dev_priv) >= 8)
2588
2589 return level == 0 ? 255 : 2047;
2590 else if (INTEL_GEN(dev_priv) >= 7)
2591
2592 return level == 0 ? 127 : 1023;
2593 else if (!is_sprite)
2594
2595 return level == 0 ? 127 : 511;
2596 else
2597
2598 return level == 0 ? 63 : 255;
2599}
2600
2601static unsigned int
2602ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2603{
2604 if (INTEL_GEN(dev_priv) >= 7)
2605 return level == 0 ? 63 : 255;
2606 else
2607 return level == 0 ? 31 : 63;
2608}
2609
2610static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2611{
2612 if (INTEL_GEN(dev_priv) >= 8)
2613 return 31;
2614 else
2615 return 15;
2616}
2617
2618
2619static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2620 int level,
2621 const struct intel_wm_config *config,
2622 enum intel_ddb_partitioning ddb_partitioning,
2623 bool is_sprite)
2624{
2625 struct drm_i915_private *dev_priv = to_i915(dev);
2626 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2627
2628
2629 if (is_sprite && !config->sprites_enabled)
2630 return 0;
2631
2632
2633 if (level == 0 || config->num_pipes_active > 1) {
2634 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2635
2636
2637
2638
2639
2640
2641 if (INTEL_GEN(dev_priv) <= 6)
2642 fifo_size /= 2;
2643 }
2644
2645 if (config->sprites_enabled) {
2646
2647 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2648 if (is_sprite)
2649 fifo_size *= 5;
2650 fifo_size /= 6;
2651 } else {
2652 fifo_size /= 2;
2653 }
2654 }
2655
2656
2657 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2658}
2659
2660
2661static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2662 int level,
2663 const struct intel_wm_config *config)
2664{
2665
2666 if (level > 0 && config->num_pipes_active > 1)
2667 return 64;
2668
2669
2670 return ilk_cursor_wm_reg_max(to_i915(dev), level);
2671}
2672
2673static void ilk_compute_wm_maximums(const struct drm_device *dev,
2674 int level,
2675 const struct intel_wm_config *config,
2676 enum intel_ddb_partitioning ddb_partitioning,
2677 struct ilk_wm_maximums *max)
2678{
2679 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2680 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2681 max->cur = ilk_cursor_wm_max(dev, level, config);
2682 max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
2683}
2684
2685static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2686 int level,
2687 struct ilk_wm_maximums *max)
2688{
2689 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2690 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2691 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2692 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2693}
2694
2695static bool ilk_validate_wm_level(int level,
2696 const struct ilk_wm_maximums *max,
2697 struct intel_wm_level *result)
2698{
2699 bool ret;
2700
2701
2702 if (!result->enable)
2703 return false;
2704
2705 result->enable = result->pri_val <= max->pri &&
2706 result->spr_val <= max->spr &&
2707 result->cur_val <= max->cur;
2708
2709 ret = result->enable;
2710
2711
2712
2713
2714
2715
2716 if (level == 0 && !result->enable) {
2717 if (result->pri_val > max->pri)
2718 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2719 level, result->pri_val, max->pri);
2720 if (result->spr_val > max->spr)
2721 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2722 level, result->spr_val, max->spr);
2723 if (result->cur_val > max->cur)
2724 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2725 level, result->cur_val, max->cur);
2726
2727 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2728 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2729 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2730 result->enable = true;
2731 }
2732
2733 return ret;
2734}
2735
2736static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2737 const struct intel_crtc *intel_crtc,
2738 int level,
2739 struct intel_crtc_state *cstate,
2740 const struct intel_plane_state *pristate,
2741 const struct intel_plane_state *sprstate,
2742 const struct intel_plane_state *curstate,
2743 struct intel_wm_level *result)
2744{
2745 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2746 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2747 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2748
2749
2750 if (level > 0) {
2751 pri_latency *= 5;
2752 spr_latency *= 5;
2753 cur_latency *= 5;
2754 }
2755
2756 if (pristate) {
2757 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2758 pri_latency, level);
2759 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2760 }
2761
2762 if (sprstate)
2763 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2764
2765 if (curstate)
2766 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2767
2768 result->enable = true;
2769}
2770
2771static uint32_t
2772hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2773{
2774 const struct intel_atomic_state *intel_state =
2775 to_intel_atomic_state(cstate->base.state);
2776 const struct drm_display_mode *adjusted_mode =
2777 &cstate->base.adjusted_mode;
2778 u32 linetime, ips_linetime;
2779
2780 if (!cstate->base.active)
2781 return 0;
2782 if (WARN_ON(adjusted_mode->crtc_clock == 0))
2783 return 0;
2784 if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2785 return 0;
2786
2787
2788
2789
2790 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2791 adjusted_mode->crtc_clock);
2792 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2793 intel_state->cdclk.logical.cdclk);
2794
2795 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2796 PIPE_WM_LINETIME_TIME(linetime);
2797}
2798
2799static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2800 uint16_t wm[8])
2801{
2802 if (INTEL_GEN(dev_priv) >= 9) {
2803 uint32_t val;
2804 int ret, i;
2805 int level, max_level = ilk_wm_max_level(dev_priv);
2806
2807
2808 val = 0;
2809 mutex_lock(&dev_priv->pcu_lock);
2810 ret = sandybridge_pcode_read(dev_priv,
2811 GEN9_PCODE_READ_MEM_LATENCY,
2812 &val);
2813 mutex_unlock(&dev_priv->pcu_lock);
2814
2815 if (ret) {
2816 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2817 return;
2818 }
2819
2820 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2821 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2822 GEN9_MEM_LATENCY_LEVEL_MASK;
2823 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2824 GEN9_MEM_LATENCY_LEVEL_MASK;
2825 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2826 GEN9_MEM_LATENCY_LEVEL_MASK;
2827
2828
2829 val = 1;
2830 mutex_lock(&dev_priv->pcu_lock);
2831 ret = sandybridge_pcode_read(dev_priv,
2832 GEN9_PCODE_READ_MEM_LATENCY,
2833 &val);
2834 mutex_unlock(&dev_priv->pcu_lock);
2835 if (ret) {
2836 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2837 return;
2838 }
2839
2840 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2841 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2842 GEN9_MEM_LATENCY_LEVEL_MASK;
2843 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2844 GEN9_MEM_LATENCY_LEVEL_MASK;
2845 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2846 GEN9_MEM_LATENCY_LEVEL_MASK;
2847
2848
2849
2850
2851
2852
2853 for (level = 1; level <= max_level; level++) {
2854 if (wm[level] == 0) {
2855 for (i = level + 1; i <= max_level; i++)
2856 wm[i] = 0;
2857 break;
2858 }
2859 }
2860
2861
2862
2863
2864
2865
2866
2867
2868 if (wm[0] == 0) {
2869 wm[0] += 2;
2870 for (level = 1; level <= max_level; level++) {
2871 if (wm[level] == 0)
2872 break;
2873 wm[level] += 2;
2874 }
2875 }
2876
2877 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2878 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2879
2880 wm[0] = (sskpd >> 56) & 0xFF;
2881 if (wm[0] == 0)
2882 wm[0] = sskpd & 0xF;
2883 wm[1] = (sskpd >> 4) & 0xFF;
2884 wm[2] = (sskpd >> 12) & 0xFF;
2885 wm[3] = (sskpd >> 20) & 0x1FF;
2886 wm[4] = (sskpd >> 32) & 0x1FF;
2887 } else if (INTEL_GEN(dev_priv) >= 6) {
2888 uint32_t sskpd = I915_READ(MCH_SSKPD);
2889
2890 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2891 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2892 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2893 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2894 } else if (INTEL_GEN(dev_priv) >= 5) {
2895 uint32_t mltr = I915_READ(MLTR_ILK);
2896
2897
2898 wm[0] = 7;
2899 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2900 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2901 } else {
2902 MISSING_CASE(INTEL_DEVID(dev_priv));
2903 }
2904}
2905
2906static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2907 uint16_t wm[5])
2908{
2909
2910 if (IS_GEN5(dev_priv))
2911 wm[0] = 13;
2912}
2913
2914static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2915 uint16_t wm[5])
2916{
2917
2918 if (IS_GEN5(dev_priv))
2919 wm[0] = 13;
2920}
2921
2922int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2923{
2924
2925 if (INTEL_GEN(dev_priv) >= 9)
2926 return 7;
2927 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2928 return 4;
2929 else if (INTEL_GEN(dev_priv) >= 6)
2930 return 3;
2931 else
2932 return 2;
2933}
2934
2935static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2936 const char *name,
2937 const uint16_t wm[8])
2938{
2939 int level, max_level = ilk_wm_max_level(dev_priv);
2940
2941 for (level = 0; level <= max_level; level++) {
2942 unsigned int latency = wm[level];
2943
2944 if (latency == 0) {
2945 DRM_ERROR("%s WM%d latency not provided\n",
2946 name, level);
2947 continue;
2948 }
2949
2950
2951
2952
2953
2954 if (INTEL_GEN(dev_priv) >= 9)
2955 latency *= 10;
2956 else if (level > 0)
2957 latency *= 5;
2958
2959 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2960 name, level, wm[level],
2961 latency / 10, latency % 10);
2962 }
2963}
2964
2965static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2966 uint16_t wm[5], uint16_t min)
2967{
2968 int level, max_level = ilk_wm_max_level(dev_priv);
2969
2970 if (wm[0] >= min)
2971 return false;
2972
2973 wm[0] = max(wm[0], min);
2974 for (level = 1; level <= max_level; level++)
2975 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2976
2977 return true;
2978}
2979
2980static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2981{
2982 bool changed;
2983
2984
2985
2986
2987
2988 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2989 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2990 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2991
2992 if (!changed)
2993 return;
2994
2995 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2996 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
2997 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
2998 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
2999}
3000
3001static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3002{
3003 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3004
3005 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3006 sizeof(dev_priv->wm.pri_latency));
3007 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3008 sizeof(dev_priv->wm.pri_latency));
3009
3010 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3011 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3012
3013 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3014 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3015 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3016
3017 if (IS_GEN6(dev_priv))
3018 snb_wm_latency_quirk(dev_priv);
3019}
3020
3021static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3022{
3023 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3024 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3025}
3026
3027static bool ilk_validate_pipe_wm(struct drm_device *dev,
3028 struct intel_pipe_wm *pipe_wm)
3029{
3030
3031 const struct intel_wm_config config = {
3032 .num_pipes_active = 1,
3033 .sprites_enabled = pipe_wm->sprites_enabled,
3034 .sprites_scaled = pipe_wm->sprites_scaled,
3035 };
3036 struct ilk_wm_maximums max;
3037
3038
3039 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
3040
3041
3042 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3043 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3044 return false;
3045 }
3046
3047 return true;
3048}
3049
3050
3051static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
3052{
3053 struct drm_atomic_state *state = cstate->base.state;
3054 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3055 struct intel_pipe_wm *pipe_wm;
3056 struct drm_device *dev = state->dev;
3057 const struct drm_i915_private *dev_priv = to_i915(dev);
3058 struct drm_plane *plane;
3059 const struct drm_plane_state *plane_state;
3060 const struct intel_plane_state *pristate = NULL;
3061 const struct intel_plane_state *sprstate = NULL;
3062 const struct intel_plane_state *curstate = NULL;
3063 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3064 struct ilk_wm_maximums max;
3065
3066 pipe_wm = &cstate->wm.ilk.optimal;
3067
3068 drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &cstate->base) {
3069 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3070
3071 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3072 pristate = ps;
3073 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3074 sprstate = ps;
3075 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3076 curstate = ps;
3077 }
3078
3079 pipe_wm->pipe_enabled = cstate->base.active;
3080 if (sprstate) {
3081 pipe_wm->sprites_enabled = sprstate->base.visible;
3082 pipe_wm->sprites_scaled = sprstate->base.visible &&
3083 (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3084 drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3085 }
3086
3087 usable_level = max_level;
3088
3089
3090 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3091 usable_level = 1;
3092
3093
3094 if (pipe_wm->sprites_scaled)
3095 usable_level = 0;
3096
3097 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3098 ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
3099 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3100
3101 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3102 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
3103
3104 if (!ilk_validate_pipe_wm(dev, pipe_wm))
3105 return -EINVAL;
3106
3107 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3108
3109 for (level = 1; level <= usable_level; level++) {
3110 struct intel_wm_level *wm = &pipe_wm->wm[level];
3111
3112 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
3113 pristate, sprstate, curstate, wm);
3114
3115
3116
3117
3118
3119
3120 if (!ilk_validate_wm_level(level, &max, wm)) {
3121 memset(wm, 0, sizeof(*wm));
3122 break;
3123 }
3124 }
3125
3126 return 0;
3127}
3128
3129
3130
3131
3132
3133
3134static int ilk_compute_intermediate_wm(struct drm_device *dev,
3135 struct intel_crtc *intel_crtc,
3136 struct intel_crtc_state *newstate)
3137{
3138 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3139 struct intel_atomic_state *intel_state =
3140 to_intel_atomic_state(newstate->base.state);
3141 const struct intel_crtc_state *oldstate =
3142 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3143 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3144 int level, max_level = ilk_wm_max_level(to_i915(dev));
3145
3146
3147
3148
3149
3150
3151 *a = newstate->wm.ilk.optimal;
3152 if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base))
3153 return 0;
3154
3155 a->pipe_enabled |= b->pipe_enabled;
3156 a->sprites_enabled |= b->sprites_enabled;
3157 a->sprites_scaled |= b->sprites_scaled;
3158
3159 for (level = 0; level <= max_level; level++) {
3160 struct intel_wm_level *a_wm = &a->wm[level];
3161 const struct intel_wm_level *b_wm = &b->wm[level];
3162
3163 a_wm->enable &= b_wm->enable;
3164 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3165 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3166 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3167 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3168 }
3169
3170
3171
3172
3173
3174
3175
3176 if (!ilk_validate_pipe_wm(dev, a))
3177 return -EINVAL;
3178
3179
3180
3181
3182
3183 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3184 newstate->wm.need_postvbl_update = true;
3185
3186 return 0;
3187}
3188
3189
3190
3191
3192static void ilk_merge_wm_level(struct drm_device *dev,
3193 int level,
3194 struct intel_wm_level *ret_wm)
3195{
3196 const struct intel_crtc *intel_crtc;
3197
3198 ret_wm->enable = true;
3199
3200 for_each_intel_crtc(dev, intel_crtc) {
3201 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3202 const struct intel_wm_level *wm = &active->wm[level];
3203
3204 if (!active->pipe_enabled)
3205 continue;
3206
3207
3208
3209
3210
3211
3212 if (!wm->enable)
3213 ret_wm->enable = false;
3214
3215 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3216 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3217 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3218 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3219 }
3220}
3221
3222
3223
3224
3225static void ilk_wm_merge(struct drm_device *dev,
3226 const struct intel_wm_config *config,
3227 const struct ilk_wm_maximums *max,
3228 struct intel_pipe_wm *merged)
3229{
3230 struct drm_i915_private *dev_priv = to_i915(dev);
3231 int level, max_level = ilk_wm_max_level(dev_priv);
3232 int last_enabled_level = max_level;
3233
3234
3235 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3236 config->num_pipes_active > 1)
3237 last_enabled_level = 0;
3238
3239
3240 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3241
3242
3243 for (level = 1; level <= max_level; level++) {
3244 struct intel_wm_level *wm = &merged->wm[level];
3245
3246 ilk_merge_wm_level(dev, level, wm);
3247
3248 if (level > last_enabled_level)
3249 wm->enable = false;
3250 else if (!ilk_validate_wm_level(level, max, wm))
3251
3252 last_enabled_level = level - 1;
3253
3254
3255
3256
3257
3258 if (wm->fbc_val > max->fbc) {
3259 if (wm->enable)
3260 merged->fbc_wm_enabled = false;
3261 wm->fbc_val = 0;
3262 }
3263 }
3264
3265
3266
3267
3268
3269
3270
3271 if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
3272 intel_fbc_is_active(dev_priv)) {
3273 for (level = 2; level <= max_level; level++) {
3274 struct intel_wm_level *wm = &merged->wm[level];
3275
3276 wm->enable = false;
3277 }
3278 }
3279}
3280
3281static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3282{
3283
3284 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3285}
3286
3287
3288static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
3289{
3290 struct drm_i915_private *dev_priv = to_i915(dev);
3291
3292 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3293 return 2 * level;
3294 else
3295 return dev_priv->wm.pri_latency[level];
3296}
3297
3298static void ilk_compute_wm_results(struct drm_device *dev,
3299 const struct intel_pipe_wm *merged,
3300 enum intel_ddb_partitioning partitioning,
3301 struct ilk_wm_values *results)
3302{
3303 struct drm_i915_private *dev_priv = to_i915(dev);
3304 struct intel_crtc *intel_crtc;
3305 int level, wm_lp;
3306
3307 results->enable_fbc_wm = merged->fbc_wm_enabled;
3308 results->partitioning = partitioning;
3309
3310
3311 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3312 const struct intel_wm_level *r;
3313
3314 level = ilk_wm_lp_to_level(wm_lp, merged);
3315
3316 r = &merged->wm[level];
3317
3318
3319
3320
3321
3322 results->wm_lp[wm_lp - 1] =
3323 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
3324 (r->pri_val << WM1_LP_SR_SHIFT) |
3325 r->cur_val;
3326
3327 if (r->enable)
3328 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3329
3330 if (INTEL_GEN(dev_priv) >= 8)
3331 results->wm_lp[wm_lp - 1] |=
3332 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3333 else
3334 results->wm_lp[wm_lp - 1] |=
3335 r->fbc_val << WM1_LP_FBC_SHIFT;
3336
3337
3338
3339
3340
3341 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3342 WARN_ON(wm_lp != 1);
3343 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3344 } else
3345 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3346 }
3347
3348
3349 for_each_intel_crtc(dev, intel_crtc) {
3350 enum pipe pipe = intel_crtc->pipe;
3351 const struct intel_wm_level *r =
3352 &intel_crtc->wm.active.ilk.wm[0];
3353
3354 if (WARN_ON(!r->enable))
3355 continue;
3356
3357 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3358
3359 results->wm_pipe[pipe] =
3360 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3361 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3362 r->cur_val;
3363 }
3364}
3365
3366
3367
3368static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
3369 struct intel_pipe_wm *r1,
3370 struct intel_pipe_wm *r2)
3371{
3372 int level, max_level = ilk_wm_max_level(to_i915(dev));
3373 int level1 = 0, level2 = 0;
3374
3375 for (level = 1; level <= max_level; level++) {
3376 if (r1->wm[level].enable)
3377 level1 = level;
3378 if (r2->wm[level].enable)
3379 level2 = level;
3380 }
3381
3382 if (level1 == level2) {
3383 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3384 return r2;
3385 else
3386 return r1;
3387 } else if (level1 > level2) {
3388 return r1;
3389 } else {
3390 return r2;
3391 }
3392}
3393
3394
3395#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3396#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3397#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3398#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3399#define WM_DIRTY_FBC (1 << 24)
3400#define WM_DIRTY_DDB (1 << 25)
3401
3402static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3403 const struct ilk_wm_values *old,
3404 const struct ilk_wm_values *new)
3405{
3406 unsigned int dirty = 0;
3407 enum pipe pipe;
3408 int wm_lp;
3409
3410 for_each_pipe(dev_priv, pipe) {
3411 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3412 dirty |= WM_DIRTY_LINETIME(pipe);
3413
3414 dirty |= WM_DIRTY_LP_ALL;
3415 }
3416
3417 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3418 dirty |= WM_DIRTY_PIPE(pipe);
3419
3420 dirty |= WM_DIRTY_LP_ALL;
3421 }
3422 }
3423
3424 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3425 dirty |= WM_DIRTY_FBC;
3426
3427 dirty |= WM_DIRTY_LP_ALL;
3428 }
3429
3430 if (old->partitioning != new->partitioning) {
3431 dirty |= WM_DIRTY_DDB;
3432
3433 dirty |= WM_DIRTY_LP_ALL;
3434 }
3435
3436
3437 if (dirty & WM_DIRTY_LP_ALL)
3438 return dirty;
3439
3440
3441 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3442 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3443 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3444 break;
3445 }
3446
3447
3448 for (; wm_lp <= 3; wm_lp++)
3449 dirty |= WM_DIRTY_LP(wm_lp);
3450
3451 return dirty;
3452}
3453
3454static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3455 unsigned int dirty)
3456{
3457 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3458 bool changed = false;
3459
3460 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3461 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3462 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3463 changed = true;
3464 }
3465 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3466 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3467 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3468 changed = true;
3469 }
3470 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3471 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3472 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3473 changed = true;
3474 }
3475
3476
3477
3478
3479
3480
3481 return changed;
3482}
3483
3484
3485
3486
3487
3488static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3489 struct ilk_wm_values *results)
3490{
3491 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3492 unsigned int dirty;
3493 uint32_t val;
3494
3495 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3496 if (!dirty)
3497 return;
3498
3499 _ilk_disable_lp_wm(dev_priv, dirty);
3500
3501 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3502 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3503 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3504 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3505 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3506 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3507
3508 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3509 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3510 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3511 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3512 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3513 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3514
3515 if (dirty & WM_DIRTY_DDB) {
3516 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3517 val = I915_READ(WM_MISC);
3518 if (results->partitioning == INTEL_DDB_PART_1_2)
3519 val &= ~WM_MISC_DATA_PARTITION_5_6;
3520 else
3521 val |= WM_MISC_DATA_PARTITION_5_6;
3522 I915_WRITE(WM_MISC, val);
3523 } else {
3524 val = I915_READ(DISP_ARB_CTL2);
3525 if (results->partitioning == INTEL_DDB_PART_1_2)
3526 val &= ~DISP_DATA_PARTITION_5_6;
3527 else
3528 val |= DISP_DATA_PARTITION_5_6;
3529 I915_WRITE(DISP_ARB_CTL2, val);
3530 }
3531 }
3532
3533 if (dirty & WM_DIRTY_FBC) {
3534 val = I915_READ(DISP_ARB_CTL);
3535 if (results->enable_fbc_wm)
3536 val &= ~DISP_FBC_WM_DIS;
3537 else
3538 val |= DISP_FBC_WM_DIS;
3539 I915_WRITE(DISP_ARB_CTL, val);
3540 }
3541
3542 if (dirty & WM_DIRTY_LP(1) &&
3543 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3544 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3545
3546 if (INTEL_GEN(dev_priv) >= 7) {
3547 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3548 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3549 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3550 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3551 }
3552
3553 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3554 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3555 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3556 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3557 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3558 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3559
3560 dev_priv->wm.hw = *results;
3561}
3562
3563bool ilk_disable_lp_wm(struct drm_device *dev)
3564{
3565 struct drm_i915_private *dev_priv = to_i915(dev);
3566
3567 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3568}
3569
3570static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3571{
3572 u8 enabled_slices;
3573
3574
3575 enabled_slices = 1;
3576
3577
3578 if (INTEL_GEN(dev_priv) < 11)
3579 return enabled_slices;
3580
3581 if (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3582 enabled_slices++;
3583
3584 return enabled_slices;
3585}
3586
3587
3588
3589
3590
3591static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
3592{
3593 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3594
3595 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
3596 return true;
3597
3598 return false;
3599}
3600
3601static bool
3602intel_has_sagv(struct drm_i915_private *dev_priv)
3603{
3604 if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
3605 IS_CANNONLAKE(dev_priv))
3606 return true;
3607
3608 if (IS_SKYLAKE(dev_priv) &&
3609 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
3610 return true;
3611
3612 return false;
3613}
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626int
3627intel_enable_sagv(struct drm_i915_private *dev_priv)
3628{
3629 int ret;
3630
3631 if (!intel_has_sagv(dev_priv))
3632 return 0;
3633
3634 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3635 return 0;
3636
3637 DRM_DEBUG_KMS("Enabling the SAGV\n");
3638 mutex_lock(&dev_priv->pcu_lock);
3639
3640 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3641 GEN9_SAGV_ENABLE);
3642
3643
3644 mutex_unlock(&dev_priv->pcu_lock);
3645
3646
3647
3648
3649
3650 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3651 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3652 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3653 return 0;
3654 } else if (ret < 0) {
3655 DRM_ERROR("Failed to enable the SAGV\n");
3656 return ret;
3657 }
3658
3659 dev_priv->sagv_status = I915_SAGV_ENABLED;
3660 return 0;
3661}
3662
3663int
3664intel_disable_sagv(struct drm_i915_private *dev_priv)
3665{
3666 int ret;
3667
3668 if (!intel_has_sagv(dev_priv))
3669 return 0;
3670
3671 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3672 return 0;
3673
3674 DRM_DEBUG_KMS("Disabling the SAGV\n");
3675 mutex_lock(&dev_priv->pcu_lock);
3676
3677
3678 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3679 GEN9_SAGV_DISABLE,
3680 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3681 1);
3682 mutex_unlock(&dev_priv->pcu_lock);
3683
3684
3685
3686
3687
3688 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3689 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3690 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3691 return 0;
3692 } else if (ret < 0) {
3693 DRM_ERROR("Failed to disable the SAGV (%d)\n", ret);
3694 return ret;
3695 }
3696
3697 dev_priv->sagv_status = I915_SAGV_DISABLED;
3698 return 0;
3699}
3700
3701bool intel_can_enable_sagv(struct drm_atomic_state *state)
3702{
3703 struct drm_device *dev = state->dev;
3704 struct drm_i915_private *dev_priv = to_i915(dev);
3705 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3706 struct intel_crtc *crtc;
3707 struct intel_plane *plane;
3708 struct intel_crtc_state *cstate;
3709 enum pipe pipe;
3710 int level, latency;
3711 int sagv_block_time_us;
3712
3713 if (!intel_has_sagv(dev_priv))
3714 return false;
3715
3716 if (IS_GEN9(dev_priv))
3717 sagv_block_time_us = 30;
3718 else if (IS_GEN10(dev_priv))
3719 sagv_block_time_us = 20;
3720 else
3721 sagv_block_time_us = 10;
3722
3723
3724
3725
3726
3727
3728
3729 if (hweight32(intel_state->active_crtcs) == 0)
3730 return true;
3731 else if (hweight32(intel_state->active_crtcs) > 1)
3732 return false;
3733
3734
3735 pipe = ffs(intel_state->active_crtcs) - 1;
3736 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3737 cstate = to_intel_crtc_state(crtc->base.state);
3738
3739 if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3740 return false;
3741
3742 for_each_intel_plane_on_crtc(dev, crtc, plane) {
3743 struct skl_plane_wm *wm =
3744 &cstate->wm.skl.optimal.planes[plane->id];
3745
3746
3747 if (!wm->wm[0].plane_en)
3748 continue;
3749
3750
3751 for (level = ilk_wm_max_level(dev_priv);
3752 !wm->wm[level].plane_en; --level)
3753 { }
3754
3755 latency = dev_priv->wm.skl_latency[level];
3756
3757 if (skl_needs_memory_bw_wa(intel_state) &&
3758 plane->base.state->fb->modifier ==
3759 I915_FORMAT_MOD_X_TILED)
3760 latency += 15;
3761
3762
3763
3764
3765
3766
3767 if (latency < sagv_block_time_us)
3768 return false;
3769 }
3770
3771 return true;
3772}
3773
3774static unsigned int intel_get_ddb_size(struct drm_i915_private *dev_priv,
3775 const struct intel_crtc_state *cstate,
3776 const unsigned int total_data_rate,
3777 const int num_active,
3778 struct skl_ddb_allocation *ddb)
3779{
3780 const struct drm_display_mode *adjusted_mode;
3781 u64 total_data_bw;
3782 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3783
3784 WARN_ON(ddb_size == 0);
3785
3786 if (INTEL_GEN(dev_priv) < 11)
3787 return ddb_size - 4;
3788
3789 adjusted_mode = &cstate->base.adjusted_mode;
3790 total_data_bw = (u64)total_data_rate * drm_mode_vrefresh(adjusted_mode);
3791
3792
3793
3794
3795 if (total_data_bw >= GBps(12) || num_active > 1) {
3796 ddb->enabled_slices = 2;
3797 } else {
3798 ddb->enabled_slices = 1;
3799 ddb_size /= 2;
3800 }
3801
3802 return ddb_size;
3803}
3804
3805static void
3806skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3807 const struct intel_crtc_state *cstate,
3808 const unsigned int total_data_rate,
3809 struct skl_ddb_allocation *ddb,
3810 struct skl_ddb_entry *alloc,
3811 int *num_active )
3812{
3813 struct drm_atomic_state *state = cstate->base.state;
3814 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3815 struct drm_i915_private *dev_priv = to_i915(dev);
3816 struct drm_crtc *for_crtc = cstate->base.crtc;
3817 unsigned int pipe_size, ddb_size;
3818 int nth_active_pipe;
3819
3820 if (WARN_ON(!state) || !cstate->base.active) {
3821 alloc->start = 0;
3822 alloc->end = 0;
3823 *num_active = hweight32(dev_priv->active_crtcs);
3824 return;
3825 }
3826
3827 if (intel_state->active_pipe_changes)
3828 *num_active = hweight32(intel_state->active_crtcs);
3829 else
3830 *num_active = hweight32(dev_priv->active_crtcs);
3831
3832 ddb_size = intel_get_ddb_size(dev_priv, cstate, total_data_rate,
3833 *num_active, ddb);
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843 if (!intel_state->active_pipe_changes) {
3844
3845
3846
3847
3848 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3849 return;
3850 }
3851
3852 nth_active_pipe = hweight32(intel_state->active_crtcs &
3853 (drm_crtc_mask(for_crtc) - 1));
3854 pipe_size = ddb_size / hweight32(intel_state->active_crtcs);
3855 alloc->start = nth_active_pipe * ddb_size / *num_active;
3856 alloc->end = alloc->start + pipe_size;
3857}
3858
3859static unsigned int skl_cursor_allocation(int num_active)
3860{
3861 if (num_active == 1)
3862 return 32;
3863
3864 return 8;
3865}
3866
3867static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3868 struct skl_ddb_entry *entry, u32 reg)
3869{
3870 u16 mask;
3871
3872 if (INTEL_GEN(dev_priv) >= 11)
3873 mask = ICL_DDB_ENTRY_MASK;
3874 else
3875 mask = SKL_DDB_ENTRY_MASK;
3876 entry->start = reg & mask;
3877 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & mask;
3878
3879 if (entry->end)
3880 entry->end += 1;
3881}
3882
3883static void
3884skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3885 const enum pipe pipe,
3886 const enum plane_id plane_id,
3887 struct skl_ddb_allocation *ddb )
3888{
3889 u32 val, val2 = 0;
3890 int fourcc, pixel_format;
3891
3892
3893 if (plane_id == PLANE_CURSOR) {
3894 val = I915_READ(CUR_BUF_CFG(pipe));
3895 skl_ddb_entry_init_from_hw(dev_priv,
3896 &ddb->plane[pipe][plane_id], val);
3897 return;
3898 }
3899
3900 val = I915_READ(PLANE_CTL(pipe, plane_id));
3901
3902
3903 if (!(val & PLANE_CTL_ENABLE))
3904 return;
3905
3906 pixel_format = val & PLANE_CTL_FORMAT_MASK;
3907 fourcc = skl_format_to_fourcc(pixel_format,
3908 val & PLANE_CTL_ORDER_RGBX,
3909 val & PLANE_CTL_ALPHA_MASK);
3910
3911 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
3912 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
3913
3914 if (fourcc == DRM_FORMAT_NV12) {
3915 skl_ddb_entry_init_from_hw(dev_priv,
3916 &ddb->plane[pipe][plane_id], val2);
3917 skl_ddb_entry_init_from_hw(dev_priv,
3918 &ddb->uv_plane[pipe][plane_id], val);
3919 } else {
3920 skl_ddb_entry_init_from_hw(dev_priv,
3921 &ddb->plane[pipe][plane_id], val);
3922 }
3923}
3924
3925void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
3926 struct skl_ddb_allocation *ddb )
3927{
3928 struct intel_crtc *crtc;
3929
3930 memset(ddb, 0, sizeof(*ddb));
3931
3932 ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
3933
3934 for_each_intel_crtc(&dev_priv->drm, crtc) {
3935 enum intel_display_power_domain power_domain;
3936 enum plane_id plane_id;
3937 enum pipe pipe = crtc->pipe;
3938
3939 power_domain = POWER_DOMAIN_PIPE(pipe);
3940 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3941 continue;
3942
3943 for_each_plane_id_on_crtc(crtc, plane_id)
3944 skl_ddb_get_hw_plane_state(dev_priv, pipe,
3945 plane_id, ddb);
3946
3947 intel_display_power_put(dev_priv, power_domain);
3948 }
3949}
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967static uint_fixed_16_16_t
3968skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
3969 const struct intel_plane_state *pstate)
3970{
3971 struct intel_plane *plane = to_intel_plane(pstate->base.plane);
3972 uint32_t src_w, src_h, dst_w, dst_h;
3973 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
3974 uint_fixed_16_16_t downscale_h, downscale_w;
3975
3976 if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
3977 return u32_to_fixed16(0);
3978
3979
3980 if (plane->id == PLANE_CURSOR) {
3981
3982
3983
3984
3985 src_w = pstate->base.src_w >> 16;
3986 src_h = pstate->base.src_h >> 16;
3987 dst_w = pstate->base.crtc_w;
3988 dst_h = pstate->base.crtc_h;
3989 } else {
3990
3991
3992
3993
3994
3995 src_w = drm_rect_width(&pstate->base.src) >> 16;
3996 src_h = drm_rect_height(&pstate->base.src) >> 16;
3997 dst_w = drm_rect_width(&pstate->base.dst);
3998 dst_h = drm_rect_height(&pstate->base.dst);
3999 }
4000
4001 fp_w_ratio = div_fixed16(src_w, dst_w);
4002 fp_h_ratio = div_fixed16(src_h, dst_h);
4003 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4004 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4005
4006 return mul_fixed16(downscale_w, downscale_h);
4007}
4008
4009static uint_fixed_16_16_t
4010skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
4011{
4012 uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
4013
4014 if (!crtc_state->base.enable)
4015 return pipe_downscale;
4016
4017 if (crtc_state->pch_pfit.enabled) {
4018 uint32_t src_w, src_h, dst_w, dst_h;
4019 uint32_t pfit_size = crtc_state->pch_pfit.size;
4020 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4021 uint_fixed_16_16_t downscale_h, downscale_w;
4022
4023 src_w = crtc_state->pipe_src_w;
4024 src_h = crtc_state->pipe_src_h;
4025 dst_w = pfit_size >> 16;
4026 dst_h = pfit_size & 0xffff;
4027
4028 if (!dst_w || !dst_h)
4029 return pipe_downscale;
4030
4031 fp_w_ratio = div_fixed16(src_w, dst_w);
4032 fp_h_ratio = div_fixed16(src_h, dst_h);
4033 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4034 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4035
4036 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
4037 }
4038
4039 return pipe_downscale;
4040}
4041
4042int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
4043 struct intel_crtc_state *cstate)
4044{
4045 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4046 struct drm_crtc_state *crtc_state = &cstate->base;
4047 struct drm_atomic_state *state = crtc_state->state;
4048 struct drm_plane *plane;
4049 const struct drm_plane_state *pstate;
4050 struct intel_plane_state *intel_pstate;
4051 int crtc_clock, dotclk;
4052 uint32_t pipe_max_pixel_rate;
4053 uint_fixed_16_16_t pipe_downscale;
4054 uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
4055
4056 if (!cstate->base.enable)
4057 return 0;
4058
4059 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4060 uint_fixed_16_16_t plane_downscale;
4061 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
4062 int bpp;
4063
4064 if (!intel_wm_plane_visible(cstate,
4065 to_intel_plane_state(pstate)))
4066 continue;
4067
4068 if (WARN_ON(!pstate->fb))
4069 return -EINVAL;
4070
4071 intel_pstate = to_intel_plane_state(pstate);
4072 plane_downscale = skl_plane_downscale_amount(cstate,
4073 intel_pstate);
4074 bpp = pstate->fb->format->cpp[0] * 8;
4075 if (bpp == 64)
4076 plane_downscale = mul_fixed16(plane_downscale,
4077 fp_9_div_8);
4078
4079 max_downscale = max_fixed16(plane_downscale, max_downscale);
4080 }
4081 pipe_downscale = skl_pipe_downscale_amount(cstate);
4082
4083 pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
4084
4085 crtc_clock = crtc_state->adjusted_mode.crtc_clock;
4086 dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
4087
4088 if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
4089 dotclk *= 2;
4090
4091 pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
4092
4093 if (pipe_max_pixel_rate < crtc_clock) {
4094 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
4095 return -EINVAL;
4096 }
4097
4098 return 0;
4099}
4100
4101static unsigned int
4102skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
4103 const struct drm_plane_state *pstate,
4104 const int plane)
4105{
4106 struct intel_plane *intel_plane = to_intel_plane(pstate->plane);
4107 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4108 uint32_t data_rate;
4109 uint32_t width = 0, height = 0;
4110 struct drm_framebuffer *fb;
4111 u32 format;
4112 uint_fixed_16_16_t down_scale_amount;
4113
4114 if (!intel_pstate->base.visible)
4115 return 0;
4116
4117 fb = pstate->fb;
4118 format = fb->format->format;
4119
4120 if (intel_plane->id == PLANE_CURSOR)
4121 return 0;
4122 if (plane == 1 && format != DRM_FORMAT_NV12)
4123 return 0;
4124
4125
4126
4127
4128
4129
4130 width = drm_rect_width(&intel_pstate->base.src) >> 16;
4131 height = drm_rect_height(&intel_pstate->base.src) >> 16;
4132
4133
4134 if (plane == 1 && format == DRM_FORMAT_NV12) {
4135 width /= 2;
4136 height /= 2;
4137 }
4138
4139 data_rate = width * height * fb->format->cpp[plane];
4140
4141 down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
4142
4143 return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4144}
4145
4146
4147
4148
4149
4150
4151static unsigned int
4152skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4153 unsigned int *plane_data_rate,
4154 unsigned int *uv_plane_data_rate)
4155{
4156 struct drm_crtc_state *cstate = &intel_cstate->base;
4157 struct drm_atomic_state *state = cstate->state;
4158 struct drm_plane *plane;
4159 const struct drm_plane_state *pstate;
4160 unsigned int total_data_rate = 0;
4161
4162 if (WARN_ON(!state))
4163 return 0;
4164
4165
4166 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4167 enum plane_id plane_id = to_intel_plane(plane)->id;
4168 unsigned int rate;
4169
4170
4171 rate = skl_plane_relative_data_rate(intel_cstate,
4172 pstate, 0);
4173 plane_data_rate[plane_id] = rate;
4174
4175 total_data_rate += rate;
4176
4177
4178 rate = skl_plane_relative_data_rate(intel_cstate,
4179 pstate, 1);
4180 uv_plane_data_rate[plane_id] = rate;
4181
4182 total_data_rate += rate;
4183 }
4184
4185 return total_data_rate;
4186}
4187
4188static uint16_t
4189skl_ddb_min_alloc(const struct drm_plane_state *pstate, const int plane)
4190{
4191 struct drm_framebuffer *fb = pstate->fb;
4192 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4193 uint32_t src_w, src_h;
4194 uint32_t min_scanlines = 8;
4195 uint8_t plane_bpp;
4196
4197 if (WARN_ON(!fb))
4198 return 0;
4199
4200
4201 if (plane == 1 && fb->format->format != DRM_FORMAT_NV12)
4202 return 0;
4203
4204
4205 if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
4206 fb->modifier != I915_FORMAT_MOD_Yf_TILED &&
4207 fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS &&
4208 fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS)
4209 return 8;
4210
4211
4212
4213
4214
4215
4216 src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
4217 src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
4218
4219
4220 if (plane == 1) {
4221 src_w /= 2;
4222 src_h /= 2;
4223 }
4224
4225 plane_bpp = fb->format->cpp[plane];
4226
4227 if (drm_rotation_90_or_270(pstate->rotation)) {
4228 switch (plane_bpp) {
4229 case 1:
4230 min_scanlines = 32;
4231 break;
4232 case 2:
4233 min_scanlines = 16;
4234 break;
4235 case 4:
4236 min_scanlines = 8;
4237 break;
4238 case 8:
4239 min_scanlines = 4;
4240 break;
4241 default:
4242 WARN(1, "Unsupported pixel depth %u for rotation",
4243 plane_bpp);
4244 min_scanlines = 32;
4245 }
4246 }
4247
4248 return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
4249}
4250
4251static void
4252skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
4253 uint16_t *minimum, uint16_t *uv_minimum)
4254{
4255 const struct drm_plane_state *pstate;
4256 struct drm_plane *plane;
4257
4258 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
4259 enum plane_id plane_id = to_intel_plane(plane)->id;
4260
4261 if (plane_id == PLANE_CURSOR)
4262 continue;
4263
4264 if (!pstate->visible)
4265 continue;
4266
4267 minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
4268 uv_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
4269 }
4270
4271 minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
4272}
4273
4274static int
4275skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
4276 struct skl_ddb_allocation *ddb )
4277{
4278 struct drm_atomic_state *state = cstate->base.state;
4279 struct drm_crtc *crtc = cstate->base.crtc;
4280 struct drm_device *dev = crtc->dev;
4281 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4282 enum pipe pipe = intel_crtc->pipe;
4283 struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
4284 uint16_t alloc_size, start;
4285 uint16_t minimum[I915_MAX_PLANES] = {};
4286 uint16_t uv_minimum[I915_MAX_PLANES] = {};
4287 unsigned int total_data_rate;
4288 enum plane_id plane_id;
4289 int num_active;
4290 unsigned int plane_data_rate[I915_MAX_PLANES] = {};
4291 unsigned int uv_plane_data_rate[I915_MAX_PLANES] = {};
4292 uint16_t total_min_blocks = 0;
4293
4294
4295 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
4296 memset(ddb->uv_plane[pipe], 0, sizeof(ddb->uv_plane[pipe]));
4297
4298 if (WARN_ON(!state))
4299 return 0;
4300
4301 if (!cstate->base.active) {
4302 alloc->start = alloc->end = 0;
4303 return 0;
4304 }
4305
4306 total_data_rate = skl_get_total_relative_data_rate(cstate,
4307 plane_data_rate,
4308 uv_plane_data_rate);
4309 skl_ddb_get_pipe_allocation_limits(dev, cstate, total_data_rate, ddb,
4310 alloc, &num_active);
4311 alloc_size = skl_ddb_entry_size(alloc);
4312 if (alloc_size == 0)
4313 return 0;
4314
4315 skl_ddb_calc_min(cstate, num_active, minimum, uv_minimum);
4316
4317
4318
4319
4320
4321
4322
4323 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4324 total_min_blocks += minimum[plane_id];
4325 total_min_blocks += uv_minimum[plane_id];
4326 }
4327
4328 if (total_min_blocks > alloc_size) {
4329 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4330 DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks,
4331 alloc_size);
4332 return -EINVAL;
4333 }
4334
4335 alloc_size -= total_min_blocks;
4336 ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
4337 ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
4338
4339
4340
4341
4342
4343
4344
4345 if (total_data_rate == 0)
4346 return 0;
4347
4348 start = alloc->start;
4349 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4350 unsigned int data_rate, uv_data_rate;
4351 uint16_t plane_blocks, uv_plane_blocks;
4352
4353 if (plane_id == PLANE_CURSOR)
4354 continue;
4355
4356 data_rate = plane_data_rate[plane_id];
4357
4358
4359
4360
4361
4362
4363 plane_blocks = minimum[plane_id];
4364 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
4365 total_data_rate);
4366
4367
4368 if (data_rate) {
4369 ddb->plane[pipe][plane_id].start = start;
4370 ddb->plane[pipe][plane_id].end = start + plane_blocks;
4371 }
4372
4373 start += plane_blocks;
4374
4375
4376 uv_data_rate = uv_plane_data_rate[plane_id];
4377
4378 uv_plane_blocks = uv_minimum[plane_id];
4379 uv_plane_blocks += div_u64((uint64_t)alloc_size * uv_data_rate,
4380 total_data_rate);
4381
4382 if (uv_data_rate) {
4383 ddb->uv_plane[pipe][plane_id].start = start;
4384 ddb->uv_plane[pipe][plane_id].end =
4385 start + uv_plane_blocks;
4386 }
4387
4388 start += uv_plane_blocks;
4389 }
4390
4391 return 0;
4392}
4393
4394
4395
4396
4397
4398
4399
4400static uint_fixed_16_16_t
4401skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate,
4402 uint8_t cpp, uint32_t latency, uint32_t dbuf_block_size)
4403{
4404 uint32_t wm_intermediate_val;
4405 uint_fixed_16_16_t ret;
4406
4407 if (latency == 0)
4408 return FP_16_16_MAX;
4409
4410 wm_intermediate_val = latency * pixel_rate * cpp;
4411 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4412
4413 if (INTEL_GEN(dev_priv) >= 10)
4414 ret = add_fixed16_u32(ret, 1);
4415
4416 return ret;
4417}
4418
4419static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
4420 uint32_t pipe_htotal,
4421 uint32_t latency,
4422 uint_fixed_16_16_t plane_blocks_per_line)
4423{
4424 uint32_t wm_intermediate_val;
4425 uint_fixed_16_16_t ret;
4426
4427 if (latency == 0)
4428 return FP_16_16_MAX;
4429
4430 wm_intermediate_val = latency * pixel_rate;
4431 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4432 pipe_htotal * 1000);
4433 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4434 return ret;
4435}
4436
4437static uint_fixed_16_16_t
4438intel_get_linetime_us(struct intel_crtc_state *cstate)
4439{
4440 uint32_t pixel_rate;
4441 uint32_t crtc_htotal;
4442 uint_fixed_16_16_t linetime_us;
4443
4444 if (!cstate->base.active)
4445 return u32_to_fixed16(0);
4446
4447 pixel_rate = cstate->pixel_rate;
4448
4449 if (WARN_ON(pixel_rate == 0))
4450 return u32_to_fixed16(0);
4451
4452 crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
4453 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4454
4455 return linetime_us;
4456}
4457
4458static uint32_t
4459skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
4460 const struct intel_plane_state *pstate)
4461{
4462 uint64_t adjusted_pixel_rate;
4463 uint_fixed_16_16_t downscale_amount;
4464
4465
4466 if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4467 return 0;
4468
4469
4470
4471
4472
4473 adjusted_pixel_rate = cstate->pixel_rate;
4474 downscale_amount = skl_plane_downscale_amount(cstate, pstate);
4475
4476 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4477 downscale_amount);
4478}
4479
4480static int
4481skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
4482 struct intel_crtc_state *cstate,
4483 const struct intel_plane_state *intel_pstate,
4484 struct skl_wm_params *wp, int plane_id)
4485{
4486 struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
4487 const struct drm_plane_state *pstate = &intel_pstate->base;
4488 const struct drm_framebuffer *fb = pstate->fb;
4489 uint32_t interm_pbpl;
4490 struct intel_atomic_state *state =
4491 to_intel_atomic_state(cstate->base.state);
4492 bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4493
4494 if (!intel_wm_plane_visible(cstate, intel_pstate))
4495 return 0;
4496
4497
4498 if (plane_id == 1 && fb->format->format != DRM_FORMAT_NV12) {
4499 DRM_DEBUG_KMS("Non NV12 format have single plane\n");
4500 return -EINVAL;
4501 }
4502
4503 wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
4504 fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
4505 fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4506 fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4507 wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
4508 wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4509 fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4510 wp->is_planar = fb->format->format == DRM_FORMAT_NV12;
4511
4512 if (plane->id == PLANE_CURSOR) {
4513 wp->width = intel_pstate->base.crtc_w;
4514 } else {
4515
4516
4517
4518
4519
4520 wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
4521 }
4522
4523 if (plane_id == 1 && wp->is_planar)
4524 wp->width /= 2;
4525
4526 wp->cpp = fb->format->cpp[plane_id];
4527 wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
4528 intel_pstate);
4529
4530 if (INTEL_GEN(dev_priv) >= 11 &&
4531 fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 8)
4532 wp->dbuf_block_size = 256;
4533 else
4534 wp->dbuf_block_size = 512;
4535
4536 if (drm_rotation_90_or_270(pstate->rotation)) {
4537
4538 switch (wp->cpp) {
4539 case 1:
4540 wp->y_min_scanlines = 16;
4541 break;
4542 case 2:
4543 wp->y_min_scanlines = 8;
4544 break;
4545 case 4:
4546 wp->y_min_scanlines = 4;
4547 break;
4548 default:
4549 MISSING_CASE(wp->cpp);
4550 return -EINVAL;
4551 }
4552 } else {
4553 wp->y_min_scanlines = 4;
4554 }
4555
4556 if (apply_memory_bw_wa)
4557 wp->y_min_scanlines *= 2;
4558
4559 wp->plane_bytes_per_line = wp->width * wp->cpp;
4560 if (wp->y_tiled) {
4561 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4562 wp->y_min_scanlines,
4563 wp->dbuf_block_size);
4564
4565 if (INTEL_GEN(dev_priv) >= 10)
4566 interm_pbpl++;
4567
4568 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4569 wp->y_min_scanlines);
4570 } else if (wp->x_tiled && IS_GEN9(dev_priv)) {
4571 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4572 wp->dbuf_block_size);
4573 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4574 } else {
4575 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4576 wp->dbuf_block_size) + 1;
4577 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4578 }
4579
4580 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4581 wp->plane_blocks_per_line);
4582 wp->linetime_us = fixed16_to_u32_round_up(
4583 intel_get_linetime_us(cstate));
4584
4585 return 0;
4586}
4587
4588static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
4589 struct intel_crtc_state *cstate,
4590 const struct intel_plane_state *intel_pstate,
4591 uint16_t ddb_allocation,
4592 int level,
4593 const struct skl_wm_params *wp,
4594 const struct skl_wm_level *result_prev,
4595 struct skl_wm_level *result )
4596{
4597 const struct drm_plane_state *pstate = &intel_pstate->base;
4598 uint32_t latency = dev_priv->wm.skl_latency[level];
4599 uint_fixed_16_16_t method1, method2;
4600 uint_fixed_16_16_t selected_result;
4601 uint32_t res_blocks, res_lines;
4602 struct intel_atomic_state *state =
4603 to_intel_atomic_state(cstate->base.state);
4604 bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4605 uint32_t min_disp_buf_needed;
4606
4607 if (latency == 0 ||
4608 !intel_wm_plane_visible(cstate, intel_pstate)) {
4609 result->plane_en = false;
4610 return 0;
4611 }
4612
4613
4614 if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
4615 IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) &&
4616 dev_priv->ipc_enabled)
4617 latency += 4;
4618
4619 if (apply_memory_bw_wa && wp->x_tiled)
4620 latency += 15;
4621
4622 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4623 wp->cpp, latency, wp->dbuf_block_size);
4624 method2 = skl_wm_method2(wp->plane_pixel_rate,
4625 cstate->base.adjusted_mode.crtc_htotal,
4626 latency,
4627 wp->plane_blocks_per_line);
4628
4629 if (wp->y_tiled) {
4630 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4631 } else {
4632 if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
4633 wp->dbuf_block_size < 1) &&
4634 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1))
4635 selected_result = method2;
4636 else if (ddb_allocation >=
4637 fixed16_to_u32_round_up(wp->plane_blocks_per_line))
4638 selected_result = min_fixed16(method1, method2);
4639 else if (latency >= wp->linetime_us)
4640 selected_result = min_fixed16(method1, method2);
4641 else
4642 selected_result = method1;
4643 }
4644
4645 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4646 res_lines = div_round_up_fixed16(selected_result,
4647 wp->plane_blocks_per_line);
4648
4649
4650 if (level == 0 && wp->rc_surface)
4651 res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
4652
4653
4654 if (level >= 1 && level <= 7) {
4655 if (wp->y_tiled) {
4656 res_blocks += fixed16_to_u32_round_up(
4657 wp->y_tile_minimum);
4658 res_lines += wp->y_min_scanlines;
4659 } else {
4660 res_blocks++;
4661 }
4662
4663
4664
4665
4666
4667
4668
4669 if (result_prev->plane_res_b > res_blocks)
4670 res_blocks = result_prev->plane_res_b;
4671 }
4672
4673 if (INTEL_GEN(dev_priv) >= 11) {
4674 if (wp->y_tiled) {
4675 uint32_t extra_lines;
4676 uint_fixed_16_16_t fp_min_disp_buf_needed;
4677
4678 if (res_lines % wp->y_min_scanlines == 0)
4679 extra_lines = wp->y_min_scanlines;
4680 else
4681 extra_lines = wp->y_min_scanlines * 2 -
4682 res_lines % wp->y_min_scanlines;
4683
4684 fp_min_disp_buf_needed = mul_u32_fixed16(res_lines +
4685 extra_lines,
4686 wp->plane_blocks_per_line);
4687 min_disp_buf_needed = fixed16_to_u32_round_up(
4688 fp_min_disp_buf_needed);
4689 } else {
4690 min_disp_buf_needed = DIV_ROUND_UP(res_blocks * 11, 10);
4691 }
4692 } else {
4693 min_disp_buf_needed = res_blocks;
4694 }
4695
4696 if ((level > 0 && res_lines > 31) ||
4697 res_blocks >= ddb_allocation ||
4698 min_disp_buf_needed >= ddb_allocation) {
4699 result->plane_en = false;
4700
4701
4702
4703
4704
4705 if (level) {
4706 return 0;
4707 } else {
4708 struct drm_plane *plane = pstate->plane;
4709
4710 DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
4711 DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n",
4712 plane->base.id, plane->name,
4713 res_blocks, ddb_allocation, res_lines);
4714 return -EINVAL;
4715 }
4716 }
4717
4718
4719
4720
4721
4722 if (wp->is_planar && level >= 1 &&
4723 (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
4724 IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))) {
4725 result->plane_en = false;
4726 return 0;
4727 }
4728
4729
4730 result->plane_res_b = res_blocks;
4731 result->plane_res_l = res_lines;
4732 result->plane_en = true;
4733
4734 return 0;
4735}
4736
4737static int
4738skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
4739 struct skl_ddb_allocation *ddb,
4740 struct intel_crtc_state *cstate,
4741 const struct intel_plane_state *intel_pstate,
4742 const struct skl_wm_params *wm_params,
4743 struct skl_plane_wm *wm,
4744 int plane_id)
4745{
4746 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4747 struct drm_plane *plane = intel_pstate->base.plane;
4748 struct intel_plane *intel_plane = to_intel_plane(plane);
4749 uint16_t ddb_blocks;
4750 enum pipe pipe = intel_crtc->pipe;
4751 int level, max_level = ilk_wm_max_level(dev_priv);
4752 enum plane_id intel_plane_id = intel_plane->id;
4753 int ret;
4754
4755 if (WARN_ON(!intel_pstate->base.fb))
4756 return -EINVAL;
4757
4758 ddb_blocks = plane_id ?
4759 skl_ddb_entry_size(&ddb->uv_plane[pipe][intel_plane_id]) :
4760 skl_ddb_entry_size(&ddb->plane[pipe][intel_plane_id]);
4761
4762 for (level = 0; level <= max_level; level++) {
4763 struct skl_wm_level *result = plane_id ? &wm->uv_wm[level] :
4764 &wm->wm[level];
4765 struct skl_wm_level *result_prev;
4766
4767 if (level)
4768 result_prev = plane_id ? &wm->uv_wm[level - 1] :
4769 &wm->wm[level - 1];
4770 else
4771 result_prev = plane_id ? &wm->uv_wm[0] : &wm->wm[0];
4772
4773 ret = skl_compute_plane_wm(dev_priv,
4774 cstate,
4775 intel_pstate,
4776 ddb_blocks,
4777 level,
4778 wm_params,
4779 result_prev,
4780 result);
4781 if (ret)
4782 return ret;
4783 }
4784
4785 if (intel_pstate->base.fb->format->format == DRM_FORMAT_NV12)
4786 wm->is_planar = true;
4787
4788 return 0;
4789}
4790
4791static uint32_t
4792skl_compute_linetime_wm(struct intel_crtc_state *cstate)
4793{
4794 struct drm_atomic_state *state = cstate->base.state;
4795 struct drm_i915_private *dev_priv = to_i915(state->dev);
4796 uint_fixed_16_16_t linetime_us;
4797 uint32_t linetime_wm;
4798
4799 linetime_us = intel_get_linetime_us(cstate);
4800
4801 if (is_fixed16_zero(linetime_us))
4802 return 0;
4803
4804 linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4805
4806
4807 if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) &&
4808 dev_priv->ipc_enabled)
4809 linetime_wm /= 2;
4810
4811 return linetime_wm;
4812}
4813
4814static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
4815 struct skl_wm_params *wp,
4816 struct skl_wm_level *wm_l0,
4817 uint16_t ddb_allocation,
4818 struct skl_wm_level *trans_wm )
4819{
4820 struct drm_device *dev = cstate->base.crtc->dev;
4821 const struct drm_i915_private *dev_priv = to_i915(dev);
4822 uint16_t trans_min, trans_y_tile_min;
4823 const uint16_t trans_amount = 10;
4824 uint16_t trans_offset_b, res_blocks;
4825
4826 if (!cstate->base.active)
4827 goto exit;
4828
4829
4830 if (INTEL_GEN(dev_priv) <= 9)
4831 goto exit;
4832
4833
4834 if (!dev_priv->ipc_enabled)
4835 goto exit;
4836
4837 trans_min = 0;
4838 if (INTEL_GEN(dev_priv) >= 10)
4839 trans_min = 4;
4840
4841 trans_offset_b = trans_min + trans_amount;
4842
4843 if (wp->y_tiled) {
4844 trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2,
4845 wp->y_tile_minimum);
4846 res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) +
4847 trans_offset_b;
4848 } else {
4849 res_blocks = wm_l0->plane_res_b + trans_offset_b;
4850
4851
4852 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4853 res_blocks += 1;
4854
4855 }
4856
4857 res_blocks += 1;
4858
4859 if (res_blocks < ddb_allocation) {
4860 trans_wm->plane_res_b = res_blocks;
4861 trans_wm->plane_en = true;
4862 return;
4863 }
4864
4865exit:
4866 trans_wm->plane_en = false;
4867}
4868
4869static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
4870 struct skl_ddb_allocation *ddb,
4871 struct skl_pipe_wm *pipe_wm)
4872{
4873 struct drm_device *dev = cstate->base.crtc->dev;
4874 struct drm_crtc_state *crtc_state = &cstate->base;
4875 const struct drm_i915_private *dev_priv = to_i915(dev);
4876 struct drm_plane *plane;
4877 const struct drm_plane_state *pstate;
4878 struct skl_plane_wm *wm;
4879 int ret;
4880
4881
4882
4883
4884
4885 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4886
4887 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4888 const struct intel_plane_state *intel_pstate =
4889 to_intel_plane_state(pstate);
4890 enum plane_id plane_id = to_intel_plane(plane)->id;
4891 struct skl_wm_params wm_params;
4892 enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe;
4893 uint16_t ddb_blocks;
4894
4895 wm = &pipe_wm->planes[plane_id];
4896 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
4897
4898 ret = skl_compute_plane_wm_params(dev_priv, cstate,
4899 intel_pstate, &wm_params, 0);
4900 if (ret)
4901 return ret;
4902
4903 ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4904 intel_pstate, &wm_params, wm, 0);
4905 if (ret)
4906 return ret;
4907
4908 skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
4909 ddb_blocks, &wm->trans_wm);
4910
4911
4912 if (wm_params.is_planar) {
4913 memset(&wm_params, 0, sizeof(struct skl_wm_params));
4914 wm->is_planar = true;
4915
4916 ret = skl_compute_plane_wm_params(dev_priv, cstate,
4917 intel_pstate,
4918 &wm_params, 1);
4919 if (ret)
4920 return ret;
4921
4922 ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4923 intel_pstate, &wm_params,
4924 wm, 1);
4925 if (ret)
4926 return ret;
4927 }
4928 }
4929
4930 pipe_wm->linetime = skl_compute_linetime_wm(cstate);
4931
4932 return 0;
4933}
4934
4935static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
4936 i915_reg_t reg,
4937 const struct skl_ddb_entry *entry)
4938{
4939 if (entry->end)
4940 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
4941 else
4942 I915_WRITE(reg, 0);
4943}
4944
4945static void skl_write_wm_level(struct drm_i915_private *dev_priv,
4946 i915_reg_t reg,
4947 const struct skl_wm_level *level)
4948{
4949 uint32_t val = 0;
4950
4951 if (level->plane_en) {
4952 val |= PLANE_WM_EN;
4953 val |= level->plane_res_b;
4954 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
4955 }
4956
4957 I915_WRITE(reg, val);
4958}
4959
4960static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
4961 const struct skl_plane_wm *wm,
4962 const struct skl_ddb_allocation *ddb,
4963 enum plane_id plane_id)
4964{
4965 struct drm_crtc *crtc = &intel_crtc->base;
4966 struct drm_device *dev = crtc->dev;
4967 struct drm_i915_private *dev_priv = to_i915(dev);
4968 int level, max_level = ilk_wm_max_level(dev_priv);
4969 enum pipe pipe = intel_crtc->pipe;
4970
4971 for (level = 0; level <= max_level; level++) {
4972 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
4973 &wm->wm[level]);
4974 }
4975 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
4976 &wm->trans_wm);
4977
4978 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
4979 &ddb->plane[pipe][plane_id]);
4980 if (INTEL_GEN(dev_priv) >= 11)
4981 return skl_ddb_entry_write(dev_priv,
4982 PLANE_BUF_CFG(pipe, plane_id),
4983 &ddb->plane[pipe][plane_id]);
4984 if (wm->is_planar) {
4985 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
4986 &ddb->uv_plane[pipe][plane_id]);
4987 skl_ddb_entry_write(dev_priv,
4988 PLANE_NV12_BUF_CFG(pipe, plane_id),
4989 &ddb->plane[pipe][plane_id]);
4990 } else {
4991 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
4992 &ddb->plane[pipe][plane_id]);
4993 I915_WRITE(PLANE_NV12_BUF_CFG(pipe, plane_id), 0x0);
4994 }
4995}
4996
4997static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
4998 const struct skl_plane_wm *wm,
4999 const struct skl_ddb_allocation *ddb)
5000{
5001 struct drm_crtc *crtc = &intel_crtc->base;
5002 struct drm_device *dev = crtc->dev;
5003 struct drm_i915_private *dev_priv = to_i915(dev);
5004 int level, max_level = ilk_wm_max_level(dev_priv);
5005 enum pipe pipe = intel_crtc->pipe;
5006
5007 for (level = 0; level <= max_level; level++) {
5008 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5009 &wm->wm[level]);
5010 }
5011 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5012
5013 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
5014 &ddb->plane[pipe][PLANE_CURSOR]);
5015}
5016
5017bool skl_wm_level_equals(const struct skl_wm_level *l1,
5018 const struct skl_wm_level *l2)
5019{
5020 if (l1->plane_en != l2->plane_en)
5021 return false;
5022
5023
5024 if (!l1->plane_en)
5025 return true;
5026
5027 return (l1->plane_res_l == l2->plane_res_l &&
5028 l1->plane_res_b == l2->plane_res_b);
5029}
5030
5031static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5032 const struct skl_ddb_entry *b)
5033{
5034 return a->start < b->end && b->start < a->end;
5035}
5036
5037bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv,
5038 const struct skl_ddb_entry **entries,
5039 const struct skl_ddb_entry *ddb,
5040 int ignore)
5041{
5042 enum pipe pipe;
5043
5044 for_each_pipe(dev_priv, pipe) {
5045 if (pipe != ignore && entries[pipe] &&
5046 skl_ddb_entries_overlap(ddb, entries[pipe]))
5047 return true;
5048 }
5049
5050 return false;
5051}
5052
5053static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
5054 const struct skl_pipe_wm *old_pipe_wm,
5055 struct skl_pipe_wm *pipe_wm,
5056 struct skl_ddb_allocation *ddb,
5057 bool *changed )
5058{
5059 struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
5060 int ret;
5061
5062 ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
5063 if (ret)
5064 return ret;
5065
5066 if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
5067 *changed = false;
5068 else
5069 *changed = true;
5070
5071 return 0;
5072}
5073
5074static uint32_t
5075pipes_modified(struct drm_atomic_state *state)
5076{
5077 struct drm_crtc *crtc;
5078 struct drm_crtc_state *cstate;
5079 uint32_t i, ret = 0;
5080
5081 for_each_new_crtc_in_state(state, crtc, cstate, i)
5082 ret |= drm_crtc_mask(crtc);
5083
5084 return ret;
5085}
5086
5087static int
5088skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
5089{
5090 struct drm_atomic_state *state = cstate->base.state;
5091 struct drm_device *dev = state->dev;
5092 struct drm_crtc *crtc = cstate->base.crtc;
5093 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5094 struct drm_i915_private *dev_priv = to_i915(dev);
5095 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5096 struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5097 struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
5098 struct drm_plane_state *plane_state;
5099 struct drm_plane *plane;
5100 enum pipe pipe = intel_crtc->pipe;
5101
5102 drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
5103 enum plane_id plane_id = to_intel_plane(plane)->id;
5104
5105 if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
5106 &new_ddb->plane[pipe][plane_id]) &&
5107 skl_ddb_entry_equal(&cur_ddb->uv_plane[pipe][plane_id],
5108 &new_ddb->uv_plane[pipe][plane_id]))
5109 continue;
5110
5111 plane_state = drm_atomic_get_plane_state(state, plane);
5112 if (IS_ERR(plane_state))
5113 return PTR_ERR(plane_state);
5114 }
5115
5116 return 0;
5117}
5118
5119static int
5120skl_compute_ddb(struct drm_atomic_state *state)
5121{
5122 const struct drm_i915_private *dev_priv = to_i915(state->dev);
5123 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5124 struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
5125 struct intel_crtc *crtc;
5126 struct intel_crtc_state *cstate;
5127 int ret, i;
5128
5129 memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5130
5131 for_each_new_intel_crtc_in_state(intel_state, crtc, cstate, i) {
5132 ret = skl_allocate_pipe_ddb(cstate, ddb);
5133 if (ret)
5134 return ret;
5135
5136 ret = skl_ddb_add_affected_planes(cstate);
5137 if (ret)
5138 return ret;
5139 }
5140
5141 return 0;
5142}
5143
5144static void
5145skl_copy_ddb_for_pipe(struct skl_ddb_values *dst,
5146 struct skl_ddb_values *src,
5147 enum pipe pipe)
5148{
5149 memcpy(dst->ddb.uv_plane[pipe], src->ddb.uv_plane[pipe],
5150 sizeof(dst->ddb.uv_plane[pipe]));
5151 memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
5152 sizeof(dst->ddb.plane[pipe]));
5153}
5154
5155static void
5156skl_print_wm_changes(const struct drm_atomic_state *state)
5157{
5158 const struct drm_device *dev = state->dev;
5159 const struct drm_i915_private *dev_priv = to_i915(dev);
5160 const struct intel_atomic_state *intel_state =
5161 to_intel_atomic_state(state);
5162 const struct drm_crtc *crtc;
5163 const struct drm_crtc_state *cstate;
5164 const struct intel_plane *intel_plane;
5165 const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
5166 const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5167 int i;
5168
5169 for_each_new_crtc_in_state(state, crtc, cstate, i) {
5170 const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5171 enum pipe pipe = intel_crtc->pipe;
5172
5173 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
5174 enum plane_id plane_id = intel_plane->id;
5175 const struct skl_ddb_entry *old, *new;
5176
5177 old = &old_ddb->plane[pipe][plane_id];
5178 new = &new_ddb->plane[pipe][plane_id];
5179
5180 if (skl_ddb_entry_equal(old, new))
5181 continue;
5182
5183 DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
5184 intel_plane->base.base.id,
5185 intel_plane->base.name,
5186 old->start, old->end,
5187 new->start, new->end);
5188 }
5189 }
5190}
5191
5192static int
5193skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed)
5194{
5195 struct drm_device *dev = state->dev;
5196 const struct drm_i915_private *dev_priv = to_i915(dev);
5197 const struct drm_crtc *crtc;
5198 const struct drm_crtc_state *cstate;
5199 struct intel_crtc *intel_crtc;
5200 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5201 uint32_t realloc_pipes = pipes_modified(state);
5202 int ret, i;
5203
5204
5205
5206
5207
5208 if (dev_priv->wm.distrust_bios_wm)
5209 (*changed) = true;
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219 for_each_new_crtc_in_state(state, crtc, cstate, i)
5220 (*changed) = true;
5221
5222 if (!*changed)
5223 return 0;
5224
5225
5226
5227
5228
5229
5230
5231 if (dev_priv->wm.distrust_bios_wm) {
5232 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
5233 state->acquire_ctx);
5234 if (ret)
5235 return ret;
5236
5237 intel_state->active_pipe_changes = ~0;
5238
5239
5240
5241
5242
5243
5244
5245 if (!intel_state->modeset)
5246 intel_state->active_crtcs = dev_priv->active_crtcs;
5247 }
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262 if (intel_state->active_pipe_changes) {
5263 realloc_pipes = ~0;
5264 intel_state->wm_results.dirty_pipes = ~0;
5265 }
5266
5267
5268
5269
5270
5271 for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
5272 struct intel_crtc_state *cstate;
5273
5274 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
5275 if (IS_ERR(cstate))
5276 return PTR_ERR(cstate);
5277 }
5278
5279 return 0;
5280}
5281
5282static int
5283skl_compute_wm(struct drm_atomic_state *state)
5284{
5285 struct drm_crtc *crtc;
5286 struct drm_crtc_state *cstate;
5287 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5288 struct skl_ddb_values *results = &intel_state->wm_results;
5289 struct skl_pipe_wm *pipe_wm;
5290 bool changed = false;
5291 int ret, i;
5292
5293
5294 results->dirty_pipes = 0;
5295
5296 ret = skl_ddb_add_affected_pipes(state, &changed);
5297 if (ret || !changed)
5298 return ret;
5299
5300 ret = skl_compute_ddb(state);
5301 if (ret)
5302 return ret;
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314 for_each_new_crtc_in_state(state, crtc, cstate, i) {
5315 struct intel_crtc_state *intel_cstate =
5316 to_intel_crtc_state(cstate);
5317 const struct skl_pipe_wm *old_pipe_wm =
5318 &to_intel_crtc_state(crtc->state)->wm.skl.optimal;
5319
5320 pipe_wm = &intel_cstate->wm.skl.optimal;
5321 ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
5322 &results->ddb, &changed);
5323 if (ret)
5324 return ret;
5325
5326 if (changed)
5327 results->dirty_pipes |= drm_crtc_mask(crtc);
5328
5329 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
5330
5331 continue;
5332
5333 intel_cstate->update_wm_pre = true;
5334 }
5335
5336 skl_print_wm_changes(state);
5337
5338 return 0;
5339}
5340
5341static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5342 struct intel_crtc_state *cstate)
5343{
5344 struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
5345 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5346 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5347 const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5348 enum pipe pipe = crtc->pipe;
5349 enum plane_id plane_id;
5350
5351 if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5352 return;
5353
5354 I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5355
5356 for_each_plane_id_on_crtc(crtc, plane_id) {
5357 if (plane_id != PLANE_CURSOR)
5358 skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id],
5359 ddb, plane_id);
5360 else
5361 skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id],
5362 ddb);
5363 }
5364}
5365
5366static void skl_initial_wm(struct intel_atomic_state *state,
5367 struct intel_crtc_state *cstate)
5368{
5369 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5370 struct drm_device *dev = intel_crtc->base.dev;
5371 struct drm_i915_private *dev_priv = to_i915(dev);
5372 struct skl_ddb_values *results = &state->wm_results;
5373 struct skl_ddb_values *hw_vals = &dev_priv->wm.skl_hw;
5374 enum pipe pipe = intel_crtc->pipe;
5375
5376 if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5377 return;
5378
5379 mutex_lock(&dev_priv->wm.wm_mutex);
5380
5381 if (cstate->base.active_changed)
5382 skl_atomic_update_crtc_wm(state, cstate);
5383
5384 skl_copy_ddb_for_pipe(hw_vals, results, pipe);
5385
5386 mutex_unlock(&dev_priv->wm.wm_mutex);
5387}
5388
5389static void ilk_compute_wm_config(struct drm_device *dev,
5390 struct intel_wm_config *config)
5391{
5392 struct intel_crtc *crtc;
5393
5394
5395 for_each_intel_crtc(dev, crtc) {
5396 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5397
5398 if (!wm->pipe_enabled)
5399 continue;
5400
5401 config->sprites_enabled |= wm->sprites_enabled;
5402 config->sprites_scaled |= wm->sprites_scaled;
5403 config->num_pipes_active++;
5404 }
5405}
5406
5407static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5408{
5409 struct drm_device *dev = &dev_priv->drm;
5410 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5411 struct ilk_wm_maximums max;
5412 struct intel_wm_config config = {};
5413 struct ilk_wm_values results = {};
5414 enum intel_ddb_partitioning partitioning;
5415
5416 ilk_compute_wm_config(dev, &config);
5417
5418 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
5419 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
5420
5421
5422 if (INTEL_GEN(dev_priv) >= 7 &&
5423 config.num_pipes_active == 1 && config.sprites_enabled) {
5424 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
5425 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
5426
5427 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
5428 } else {
5429 best_lp_wm = &lp_wm_1_2;
5430 }
5431
5432 partitioning = (best_lp_wm == &lp_wm_1_2) ?
5433 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5434
5435 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
5436
5437 ilk_write_wm_values(dev_priv, &results);
5438}
5439
5440static void ilk_initial_watermarks(struct intel_atomic_state *state,
5441 struct intel_crtc_state *cstate)
5442{
5443 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5444 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5445
5446 mutex_lock(&dev_priv->wm.wm_mutex);
5447 intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
5448 ilk_program_watermarks(dev_priv);
5449 mutex_unlock(&dev_priv->wm.wm_mutex);
5450}
5451
5452static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5453 struct intel_crtc_state *cstate)
5454{
5455 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5456 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5457
5458 mutex_lock(&dev_priv->wm.wm_mutex);
5459 if (cstate->wm.need_postvbl_update) {
5460 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
5461 ilk_program_watermarks(dev_priv);
5462 }
5463 mutex_unlock(&dev_priv->wm.wm_mutex);
5464}
5465
5466static inline void skl_wm_level_from_reg_val(uint32_t val,
5467 struct skl_wm_level *level)
5468{
5469 level->plane_en = val & PLANE_WM_EN;
5470 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5471 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5472 PLANE_WM_LINES_MASK;
5473}
5474
5475void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
5476 struct skl_pipe_wm *out)
5477{
5478 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5479 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5480 enum pipe pipe = intel_crtc->pipe;
5481 int level, max_level;
5482 enum plane_id plane_id;
5483 uint32_t val;
5484
5485 max_level = ilk_wm_max_level(dev_priv);
5486
5487 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
5488 struct skl_plane_wm *wm = &out->planes[plane_id];
5489
5490 for (level = 0; level <= max_level; level++) {
5491 if (plane_id != PLANE_CURSOR)
5492 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5493 else
5494 val = I915_READ(CUR_WM(pipe, level));
5495
5496 skl_wm_level_from_reg_val(val, &wm->wm[level]);
5497 }
5498
5499 if (plane_id != PLANE_CURSOR)
5500 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5501 else
5502 val = I915_READ(CUR_WM_TRANS(pipe));
5503
5504 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5505 }
5506
5507 if (!intel_crtc->active)
5508 return;
5509
5510 out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5511}
5512
5513void skl_wm_get_hw_state(struct drm_device *dev)
5514{
5515 struct drm_i915_private *dev_priv = to_i915(dev);
5516 struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5517 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5518 struct drm_crtc *crtc;
5519 struct intel_crtc *intel_crtc;
5520 struct intel_crtc_state *cstate;
5521
5522 skl_ddb_get_hw_state(dev_priv, ddb);
5523 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
5524 intel_crtc = to_intel_crtc(crtc);
5525 cstate = to_intel_crtc_state(crtc->state);
5526
5527 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
5528
5529 if (intel_crtc->active)
5530 hw->dirty_pipes |= drm_crtc_mask(crtc);
5531 }
5532
5533 if (dev_priv->active_crtcs) {
5534
5535 dev_priv->wm.distrust_bios_wm = true;
5536 } else {
5537
5538
5539
5540
5541 memset(ddb->plane, 0, sizeof(ddb->plane));
5542 memset(ddb->uv_plane, 0, sizeof(ddb->uv_plane));
5543 }
5544}
5545
5546static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
5547{
5548 struct drm_device *dev = crtc->dev;
5549 struct drm_i915_private *dev_priv = to_i915(dev);
5550 struct ilk_wm_values *hw = &dev_priv->wm.hw;
5551 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5552 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5553 struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
5554 enum pipe pipe = intel_crtc->pipe;
5555 static const i915_reg_t wm0_pipe_reg[] = {
5556 [PIPE_A] = WM0_PIPEA_ILK,
5557 [PIPE_B] = WM0_PIPEB_ILK,
5558 [PIPE_C] = WM0_PIPEC_IVB,
5559 };
5560
5561 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5562 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5563 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5564
5565 memset(active, 0, sizeof(*active));
5566
5567 active->pipe_enabled = intel_crtc->active;
5568
5569 if (active->pipe_enabled) {
5570 u32 tmp = hw->wm_pipe[pipe];
5571
5572
5573
5574
5575
5576
5577
5578 active->wm[0].enable = true;
5579 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5580 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5581 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5582 active->linetime = hw->wm_linetime[pipe];
5583 } else {
5584 int level, max_level = ilk_wm_max_level(dev_priv);
5585
5586
5587
5588
5589
5590
5591 for (level = 0; level <= max_level; level++)
5592 active->wm[level].enable = true;
5593 }
5594
5595 intel_crtc->wm.active.ilk = *active;
5596}
5597
5598#define _FW_WM(value, plane) \
5599 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5600#define _FW_WM_VLV(value, plane) \
5601 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5602
5603static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5604 struct g4x_wm_values *wm)
5605{
5606 uint32_t tmp;
5607
5608 tmp = I915_READ(DSPFW1);
5609 wm->sr.plane = _FW_WM(tmp, SR);
5610 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5611 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5612 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5613
5614 tmp = I915_READ(DSPFW2);
5615 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5616 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5617 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5618 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5619 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5620 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5621
5622 tmp = I915_READ(DSPFW3);
5623 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5624 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5625 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5626 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5627}
5628
5629static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5630 struct vlv_wm_values *wm)
5631{
5632 enum pipe pipe;
5633 uint32_t tmp;
5634
5635 for_each_pipe(dev_priv, pipe) {
5636 tmp = I915_READ(VLV_DDL(pipe));
5637
5638 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5639 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5640 wm->ddl[pipe].plane[PLANE_CURSOR] =
5641 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5642 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5643 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5644 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5645 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5646 }
5647
5648 tmp = I915_READ(DSPFW1);
5649 wm->sr.plane = _FW_WM(tmp, SR);
5650 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5651 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5652 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5653
5654 tmp = I915_READ(DSPFW2);
5655 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5656 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5657 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5658
5659 tmp = I915_READ(DSPFW3);
5660 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5661
5662 if (IS_CHERRYVIEW(dev_priv)) {
5663 tmp = I915_READ(DSPFW7_CHV);
5664 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5665 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5666
5667 tmp = I915_READ(DSPFW8_CHV);
5668 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5669 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5670
5671 tmp = I915_READ(DSPFW9_CHV);
5672 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5673 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5674
5675 tmp = I915_READ(DSPHOWM);
5676 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5677 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5678 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5679 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5680 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5681 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5682 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5683 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5684 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5685 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5686 } else {
5687 tmp = I915_READ(DSPFW7);
5688 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5689 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5690
5691 tmp = I915_READ(DSPHOWM);
5692 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5693 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5694 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5695 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5696 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5697 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5698 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5699 }
5700}
5701
5702#undef _FW_WM
5703#undef _FW_WM_VLV
5704
5705void g4x_wm_get_hw_state(struct drm_device *dev)
5706{
5707 struct drm_i915_private *dev_priv = to_i915(dev);
5708 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5709 struct intel_crtc *crtc;
5710
5711 g4x_read_wm_values(dev_priv, wm);
5712
5713 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5714
5715 for_each_intel_crtc(dev, crtc) {
5716 struct intel_crtc_state *crtc_state =
5717 to_intel_crtc_state(crtc->base.state);
5718 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5719 struct g4x_pipe_wm *raw;
5720 enum pipe pipe = crtc->pipe;
5721 enum plane_id plane_id;
5722 int level, max_level;
5723
5724 active->cxsr = wm->cxsr;
5725 active->hpll_en = wm->hpll_en;
5726 active->fbc_en = wm->fbc_en;
5727
5728 active->sr = wm->sr;
5729 active->hpll = wm->hpll;
5730
5731 for_each_plane_id_on_crtc(crtc, plane_id) {
5732 active->wm.plane[plane_id] =
5733 wm->pipe[pipe].plane[plane_id];
5734 }
5735
5736 if (wm->cxsr && wm->hpll_en)
5737 max_level = G4X_WM_LEVEL_HPLL;
5738 else if (wm->cxsr)
5739 max_level = G4X_WM_LEVEL_SR;
5740 else
5741 max_level = G4X_WM_LEVEL_NORMAL;
5742
5743 level = G4X_WM_LEVEL_NORMAL;
5744 raw = &crtc_state->wm.g4x.raw[level];
5745 for_each_plane_id_on_crtc(crtc, plane_id)
5746 raw->plane[plane_id] = active->wm.plane[plane_id];
5747
5748 if (++level > max_level)
5749 goto out;
5750
5751 raw = &crtc_state->wm.g4x.raw[level];
5752 raw->plane[PLANE_PRIMARY] = active->sr.plane;
5753 raw->plane[PLANE_CURSOR] = active->sr.cursor;
5754 raw->plane[PLANE_SPRITE0] = 0;
5755 raw->fbc = active->sr.fbc;
5756
5757 if (++level > max_level)
5758 goto out;
5759
5760 raw = &crtc_state->wm.g4x.raw[level];
5761 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
5762 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
5763 raw->plane[PLANE_SPRITE0] = 0;
5764 raw->fbc = active->hpll.fbc;
5765
5766 out:
5767 for_each_plane_id_on_crtc(crtc, plane_id)
5768 g4x_raw_plane_wm_set(crtc_state, level,
5769 plane_id, USHRT_MAX);
5770 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
5771
5772 crtc_state->wm.g4x.optimal = *active;
5773 crtc_state->wm.g4x.intermediate = *active;
5774
5775 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
5776 pipe_name(pipe),
5777 wm->pipe[pipe].plane[PLANE_PRIMARY],
5778 wm->pipe[pipe].plane[PLANE_CURSOR],
5779 wm->pipe[pipe].plane[PLANE_SPRITE0]);
5780 }
5781
5782 DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
5783 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
5784 DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
5785 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
5786 DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
5787 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
5788}
5789
5790void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
5791{
5792 struct intel_plane *plane;
5793 struct intel_crtc *crtc;
5794
5795 mutex_lock(&dev_priv->wm.wm_mutex);
5796
5797 for_each_intel_plane(&dev_priv->drm, plane) {
5798 struct intel_crtc *crtc =
5799 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5800 struct intel_crtc_state *crtc_state =
5801 to_intel_crtc_state(crtc->base.state);
5802 struct intel_plane_state *plane_state =
5803 to_intel_plane_state(plane->base.state);
5804 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
5805 enum plane_id plane_id = plane->id;
5806 int level;
5807
5808 if (plane_state->base.visible)
5809 continue;
5810
5811 for (level = 0; level < 3; level++) {
5812 struct g4x_pipe_wm *raw =
5813 &crtc_state->wm.g4x.raw[level];
5814
5815 raw->plane[plane_id] = 0;
5816 wm_state->wm.plane[plane_id] = 0;
5817 }
5818
5819 if (plane_id == PLANE_PRIMARY) {
5820 for (level = 0; level < 3; level++) {
5821 struct g4x_pipe_wm *raw =
5822 &crtc_state->wm.g4x.raw[level];
5823 raw->fbc = 0;
5824 }
5825
5826 wm_state->sr.fbc = 0;
5827 wm_state->hpll.fbc = 0;
5828 wm_state->fbc_en = false;
5829 }
5830 }
5831
5832 for_each_intel_crtc(&dev_priv->drm, crtc) {
5833 struct intel_crtc_state *crtc_state =
5834 to_intel_crtc_state(crtc->base.state);
5835
5836 crtc_state->wm.g4x.intermediate =
5837 crtc_state->wm.g4x.optimal;
5838 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
5839 }
5840
5841 g4x_program_watermarks(dev_priv);
5842
5843 mutex_unlock(&dev_priv->wm.wm_mutex);
5844}
5845
5846void vlv_wm_get_hw_state(struct drm_device *dev)
5847{
5848 struct drm_i915_private *dev_priv = to_i915(dev);
5849 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
5850 struct intel_crtc *crtc;
5851 u32 val;
5852
5853 vlv_read_wm_values(dev_priv, wm);
5854
5855 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
5856 wm->level = VLV_WM_LEVEL_PM2;
5857
5858 if (IS_CHERRYVIEW(dev_priv)) {
5859 mutex_lock(&dev_priv->pcu_lock);
5860
5861 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5862 if (val & DSP_MAXFIFO_PM5_ENABLE)
5863 wm->level = VLV_WM_LEVEL_PM5;
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5875 val |= FORCE_DDR_FREQ_REQ_ACK;
5876 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
5877
5878 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
5879 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
5880 DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
5881 "assuming DDR DVFS is disabled\n");
5882 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
5883 } else {
5884 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5885 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
5886 wm->level = VLV_WM_LEVEL_DDR_DVFS;
5887 }
5888
5889 mutex_unlock(&dev_priv->pcu_lock);
5890 }
5891
5892 for_each_intel_crtc(dev, crtc) {
5893 struct intel_crtc_state *crtc_state =
5894 to_intel_crtc_state(crtc->base.state);
5895 struct vlv_wm_state *active = &crtc->wm.active.vlv;
5896 const struct vlv_fifo_state *fifo_state =
5897 &crtc_state->wm.vlv.fifo_state;
5898 enum pipe pipe = crtc->pipe;
5899 enum plane_id plane_id;
5900 int level;
5901
5902 vlv_get_fifo_size(crtc_state);
5903
5904 active->num_levels = wm->level + 1;
5905 active->cxsr = wm->cxsr;
5906
5907 for (level = 0; level < active->num_levels; level++) {
5908 struct g4x_pipe_wm *raw =
5909 &crtc_state->wm.vlv.raw[level];
5910
5911 active->sr[level].plane = wm->sr.plane;
5912 active->sr[level].cursor = wm->sr.cursor;
5913
5914 for_each_plane_id_on_crtc(crtc, plane_id) {
5915 active->wm[level].plane[plane_id] =
5916 wm->pipe[pipe].plane[plane_id];
5917
5918 raw->plane[plane_id] =
5919 vlv_invert_wm_value(active->wm[level].plane[plane_id],
5920 fifo_state->plane[plane_id]);
5921 }
5922 }
5923
5924 for_each_plane_id_on_crtc(crtc, plane_id)
5925 vlv_raw_plane_wm_set(crtc_state, level,
5926 plane_id, USHRT_MAX);
5927 vlv_invalidate_wms(crtc, active, level);
5928
5929 crtc_state->wm.vlv.optimal = *active;
5930 crtc_state->wm.vlv.intermediate = *active;
5931
5932 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
5933 pipe_name(pipe),
5934 wm->pipe[pipe].plane[PLANE_PRIMARY],
5935 wm->pipe[pipe].plane[PLANE_CURSOR],
5936 wm->pipe[pipe].plane[PLANE_SPRITE0],
5937 wm->pipe[pipe].plane[PLANE_SPRITE1]);
5938 }
5939
5940 DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
5941 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
5942}
5943
5944void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
5945{
5946 struct intel_plane *plane;
5947 struct intel_crtc *crtc;
5948
5949 mutex_lock(&dev_priv->wm.wm_mutex);
5950
5951 for_each_intel_plane(&dev_priv->drm, plane) {
5952 struct intel_crtc *crtc =
5953 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5954 struct intel_crtc_state *crtc_state =
5955 to_intel_crtc_state(crtc->base.state);
5956 struct intel_plane_state *plane_state =
5957 to_intel_plane_state(plane->base.state);
5958 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
5959 const struct vlv_fifo_state *fifo_state =
5960 &crtc_state->wm.vlv.fifo_state;
5961 enum plane_id plane_id = plane->id;
5962 int level;
5963
5964 if (plane_state->base.visible)
5965 continue;
5966
5967 for (level = 0; level < wm_state->num_levels; level++) {
5968 struct g4x_pipe_wm *raw =
5969 &crtc_state->wm.vlv.raw[level];
5970
5971 raw->plane[plane_id] = 0;
5972
5973 wm_state->wm[level].plane[plane_id] =
5974 vlv_invert_wm_value(raw->plane[plane_id],
5975 fifo_state->plane[plane_id]);
5976 }
5977 }
5978
5979 for_each_intel_crtc(&dev_priv->drm, crtc) {
5980 struct intel_crtc_state *crtc_state =
5981 to_intel_crtc_state(crtc->base.state);
5982
5983 crtc_state->wm.vlv.intermediate =
5984 crtc_state->wm.vlv.optimal;
5985 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
5986 }
5987
5988 vlv_program_watermarks(dev_priv);
5989
5990 mutex_unlock(&dev_priv->wm.wm_mutex);
5991}
5992
5993
5994
5995
5996
5997static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
5998{
5999 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6000 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6001 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6002
6003
6004
6005
6006
6007}
6008
6009void ilk_wm_get_hw_state(struct drm_device *dev)
6010{
6011 struct drm_i915_private *dev_priv = to_i915(dev);
6012 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6013 struct drm_crtc *crtc;
6014
6015 ilk_init_lp_watermarks(dev_priv);
6016
6017 for_each_crtc(dev, crtc)
6018 ilk_pipe_wm_get_hw_state(crtc);
6019
6020 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6021 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6022 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6023
6024 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6025 if (INTEL_GEN(dev_priv) >= 7) {
6026 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6027 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6028 }
6029
6030 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6031 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6032 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6033 else if (IS_IVYBRIDGE(dev_priv))
6034 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6035 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6036
6037 hw->enable_fbc_wm =
6038 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6039}
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074void intel_update_watermarks(struct intel_crtc *crtc)
6075{
6076 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6077
6078 if (dev_priv->display.update_wm)
6079 dev_priv->display.update_wm(crtc);
6080}
6081
6082void intel_enable_ipc(struct drm_i915_private *dev_priv)
6083{
6084 u32 val;
6085
6086
6087 if (IS_SKYLAKE(dev_priv)) {
6088 dev_priv->ipc_enabled = false;
6089 return;
6090 }
6091
6092 val = I915_READ(DISP_ARB_CTL2);
6093
6094 if (dev_priv->ipc_enabled)
6095 val |= DISP_IPC_ENABLE;
6096 else
6097 val &= ~DISP_IPC_ENABLE;
6098
6099 I915_WRITE(DISP_ARB_CTL2, val);
6100}
6101
6102void intel_init_ipc(struct drm_i915_private *dev_priv)
6103{
6104 dev_priv->ipc_enabled = false;
6105 if (!HAS_IPC(dev_priv))
6106 return;
6107
6108 dev_priv->ipc_enabled = true;
6109 intel_enable_ipc(dev_priv);
6110}
6111
6112
6113
6114
6115DEFINE_SPINLOCK(mchdev_lock);
6116
6117
6118
6119static struct drm_i915_private *i915_mch_dev;
6120
6121bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
6122{
6123 u16 rgvswctl;
6124
6125 lockdep_assert_held(&mchdev_lock);
6126
6127 rgvswctl = I915_READ16(MEMSWCTL);
6128 if (rgvswctl & MEMCTL_CMD_STS) {
6129 DRM_DEBUG("gpu busy, RCS change rejected\n");
6130 return false;
6131 }
6132
6133 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
6134 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
6135 I915_WRITE16(MEMSWCTL, rgvswctl);
6136 POSTING_READ16(MEMSWCTL);
6137
6138 rgvswctl |= MEMCTL_CMD_STS;
6139 I915_WRITE16(MEMSWCTL, rgvswctl);
6140
6141 return true;
6142}
6143
6144static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
6145{
6146 u32 rgvmodectl;
6147 u8 fmax, fmin, fstart, vstart;
6148
6149 spin_lock_irq(&mchdev_lock);
6150
6151 rgvmodectl = I915_READ(MEMMODECTL);
6152
6153
6154 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
6155 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
6156
6157
6158 I915_WRITE(RCUPEI, 100000);
6159 I915_WRITE(RCDNEI, 100000);
6160
6161
6162 I915_WRITE(RCBMAXAVG, 90000);
6163 I915_WRITE(RCBMINAVG, 80000);
6164
6165 I915_WRITE(MEMIHYST, 1);
6166
6167
6168 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
6169 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
6170 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
6171 MEMMODE_FSTART_SHIFT;
6172
6173 vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
6174 PXVFREQ_PX_SHIFT;
6175
6176 dev_priv->ips.fmax = fmax;
6177 dev_priv->ips.fstart = fstart;
6178
6179 dev_priv->ips.max_delay = fstart;
6180 dev_priv->ips.min_delay = fmin;
6181 dev_priv->ips.cur_delay = fstart;
6182
6183 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
6184 fmax, fmin, fstart);
6185
6186 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
6187
6188
6189
6190
6191
6192 I915_WRITE(VIDSTART, vstart);
6193 POSTING_READ(VIDSTART);
6194
6195 rgvmodectl |= MEMMODE_SWMODE_EN;
6196 I915_WRITE(MEMMODECTL, rgvmodectl);
6197
6198 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
6199 DRM_ERROR("stuck trying to change perf mode\n");
6200 mdelay(1);
6201
6202 ironlake_set_drps(dev_priv, fstart);
6203
6204 dev_priv->ips.last_count1 = I915_READ(DMIEC) +
6205 I915_READ(DDREC) + I915_READ(CSIEC);
6206 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6207 dev_priv->ips.last_count2 = I915_READ(GFXEC);
6208 dev_priv->ips.last_time2 = ktime_get_raw_ns();
6209
6210 spin_unlock_irq(&mchdev_lock);
6211}
6212
6213static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
6214{
6215 u16 rgvswctl;
6216
6217 spin_lock_irq(&mchdev_lock);
6218
6219 rgvswctl = I915_READ16(MEMSWCTL);
6220
6221
6222 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
6223 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
6224 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
6225 I915_WRITE(DEIIR, DE_PCU_EVENT);
6226 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
6227
6228
6229 ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
6230 mdelay(1);
6231 rgvswctl |= MEMCTL_CMD_STS;
6232 I915_WRITE(MEMSWCTL, rgvswctl);
6233 mdelay(1);
6234
6235 spin_unlock_irq(&mchdev_lock);
6236}
6237
6238
6239
6240
6241
6242
6243static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6244{
6245 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6246 u32 limits;
6247
6248
6249
6250
6251
6252
6253
6254 if (INTEL_GEN(dev_priv) >= 9) {
6255 limits = (rps->max_freq_softlimit) << 23;
6256 if (val <= rps->min_freq_softlimit)
6257 limits |= (rps->min_freq_softlimit) << 14;
6258 } else {
6259 limits = rps->max_freq_softlimit << 24;
6260 if (val <= rps->min_freq_softlimit)
6261 limits |= rps->min_freq_softlimit << 16;
6262 }
6263
6264 return limits;
6265}
6266
6267static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
6268{
6269 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6270 u32 threshold_up = 0, threshold_down = 0;
6271 u32 ei_up = 0, ei_down = 0;
6272
6273 lockdep_assert_held(&rps->power.mutex);
6274
6275 if (new_power == rps->power.mode)
6276 return;
6277
6278
6279 switch (new_power) {
6280 case LOW_POWER:
6281
6282 ei_up = 16000;
6283 threshold_up = 95;
6284
6285
6286 ei_down = 32000;
6287 threshold_down = 85;
6288 break;
6289
6290 case BETWEEN:
6291
6292 ei_up = 13000;
6293 threshold_up = 90;
6294
6295
6296 ei_down = 32000;
6297 threshold_down = 75;
6298 break;
6299
6300 case HIGH_POWER:
6301
6302 ei_up = 10000;
6303 threshold_up = 85;
6304
6305
6306 ei_down = 32000;
6307 threshold_down = 60;
6308 break;
6309 }
6310
6311
6312
6313
6314 if (IS_VALLEYVIEW(dev_priv))
6315 goto skip_hw_write;
6316
6317 I915_WRITE(GEN6_RP_UP_EI,
6318 GT_INTERVAL_FROM_US(dev_priv, ei_up));
6319 I915_WRITE(GEN6_RP_UP_THRESHOLD,
6320 GT_INTERVAL_FROM_US(dev_priv,
6321 ei_up * threshold_up / 100));
6322
6323 I915_WRITE(GEN6_RP_DOWN_EI,
6324 GT_INTERVAL_FROM_US(dev_priv, ei_down));
6325 I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6326 GT_INTERVAL_FROM_US(dev_priv,
6327 ei_down * threshold_down / 100));
6328
6329 I915_WRITE(GEN6_RP_CONTROL,
6330 GEN6_RP_MEDIA_TURBO |
6331 GEN6_RP_MEDIA_HW_NORMAL_MODE |
6332 GEN6_RP_MEDIA_IS_GFX |
6333 GEN6_RP_ENABLE |
6334 GEN6_RP_UP_BUSY_AVG |
6335 GEN6_RP_DOWN_IDLE_AVG);
6336
6337skip_hw_write:
6338 rps->power.mode = new_power;
6339 rps->power.up_threshold = threshold_up;
6340 rps->power.down_threshold = threshold_down;
6341}
6342
6343static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6344{
6345 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6346 int new_power;
6347
6348 new_power = rps->power.mode;
6349 switch (rps->power.mode) {
6350 case LOW_POWER:
6351 if (val > rps->efficient_freq + 1 &&
6352 val > rps->cur_freq)
6353 new_power = BETWEEN;
6354 break;
6355
6356 case BETWEEN:
6357 if (val <= rps->efficient_freq &&
6358 val < rps->cur_freq)
6359 new_power = LOW_POWER;
6360 else if (val >= rps->rp0_freq &&
6361 val > rps->cur_freq)
6362 new_power = HIGH_POWER;
6363 break;
6364
6365 case HIGH_POWER:
6366 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6367 val < rps->cur_freq)
6368 new_power = BETWEEN;
6369 break;
6370 }
6371
6372 if (val <= rps->min_freq_softlimit)
6373 new_power = LOW_POWER;
6374 if (val >= rps->max_freq_softlimit)
6375 new_power = HIGH_POWER;
6376
6377 mutex_lock(&rps->power.mutex);
6378 if (rps->power.interactive)
6379 new_power = HIGH_POWER;
6380 rps_set_power(dev_priv, new_power);
6381 mutex_unlock(&rps->power.mutex);
6382 rps->last_adj = 0;
6383}
6384
6385void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
6386{
6387 struct intel_rps *rps = &i915->gt_pm.rps;
6388
6389 if (INTEL_GEN(i915) < 6)
6390 return;
6391
6392 mutex_lock(&rps->power.mutex);
6393 if (interactive) {
6394 if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
6395 rps_set_power(i915, HIGH_POWER);
6396 } else {
6397 GEM_BUG_ON(!rps->power.interactive);
6398 rps->power.interactive--;
6399 }
6400 mutex_unlock(&rps->power.mutex);
6401}
6402
6403static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6404{
6405 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6406 u32 mask = 0;
6407
6408
6409 if (val > rps->min_freq_softlimit)
6410 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6411 if (val < rps->max_freq_softlimit)
6412 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6413
6414 mask &= dev_priv->pm_rps_events;
6415
6416 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6417}
6418
6419
6420
6421
6422static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6423{
6424 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6425
6426
6427
6428
6429 if (val != rps->cur_freq) {
6430 gen6_set_rps_thresholds(dev_priv, val);
6431
6432 if (INTEL_GEN(dev_priv) >= 9)
6433 I915_WRITE(GEN6_RPNSWREQ,
6434 GEN9_FREQUENCY(val));
6435 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6436 I915_WRITE(GEN6_RPNSWREQ,
6437 HSW_FREQUENCY(val));
6438 else
6439 I915_WRITE(GEN6_RPNSWREQ,
6440 GEN6_FREQUENCY(val) |
6441 GEN6_OFFSET(0) |
6442 GEN6_AGGRESSIVE_TURBO);
6443 }
6444
6445
6446
6447
6448 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6449 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6450
6451 rps->cur_freq = val;
6452 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6453
6454 return 0;
6455}
6456
6457static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6458{
6459 int err;
6460
6461 if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6462 "Odd GPU freq value\n"))
6463 val &= ~1;
6464
6465 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6466
6467 if (val != dev_priv->gt_pm.rps.cur_freq) {
6468 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6469 if (err)
6470 return err;
6471
6472 gen6_set_rps_thresholds(dev_priv, val);
6473 }
6474
6475 dev_priv->gt_pm.rps.cur_freq = val;
6476 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6477
6478 return 0;
6479}
6480
6481
6482
6483
6484
6485
6486
6487
6488static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6489{
6490 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6491 u32 val = rps->idle_freq;
6492 int err;
6493
6494 if (rps->cur_freq <= val)
6495 return;
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
6510 err = valleyview_set_rps(dev_priv, val);
6511 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
6512
6513 if (err)
6514 DRM_ERROR("Failed to set RPS for idle\n");
6515}
6516
6517void gen6_rps_busy(struct drm_i915_private *dev_priv)
6518{
6519 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6520
6521 mutex_lock(&dev_priv->pcu_lock);
6522 if (rps->enabled) {
6523 u8 freq;
6524
6525 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6526 gen6_rps_reset_ei(dev_priv);
6527 I915_WRITE(GEN6_PMINTRMSK,
6528 gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6529
6530 gen6_enable_rps_interrupts(dev_priv);
6531
6532
6533
6534
6535 freq = max(rps->cur_freq,
6536 rps->efficient_freq);
6537
6538 if (intel_set_rps(dev_priv,
6539 clamp(freq,
6540 rps->min_freq_softlimit,
6541 rps->max_freq_softlimit)))
6542 DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6543 }
6544 mutex_unlock(&dev_priv->pcu_lock);
6545}
6546
6547void gen6_rps_idle(struct drm_i915_private *dev_priv)
6548{
6549 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6550
6551
6552
6553
6554
6555
6556 gen6_disable_rps_interrupts(dev_priv);
6557
6558 mutex_lock(&dev_priv->pcu_lock);
6559 if (rps->enabled) {
6560 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6561 vlv_set_rps_idle(dev_priv);
6562 else
6563 gen6_set_rps(dev_priv, rps->idle_freq);
6564 rps->last_adj = 0;
6565 I915_WRITE(GEN6_PMINTRMSK,
6566 gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6567 }
6568 mutex_unlock(&dev_priv->pcu_lock);
6569}
6570
6571void gen6_rps_boost(struct i915_request *rq,
6572 struct intel_rps_client *rps_client)
6573{
6574 struct intel_rps *rps = &rq->i915->gt_pm.rps;
6575 unsigned long flags;
6576 bool boost;
6577
6578
6579
6580
6581 if (!rps->enabled)
6582 return;
6583
6584 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
6585 return;
6586
6587
6588 boost = false;
6589 spin_lock_irqsave(&rq->lock, flags);
6590 if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) {
6591 boost = !atomic_fetch_inc(&rps->num_waiters);
6592 rq->waitboost = true;
6593 }
6594 spin_unlock_irqrestore(&rq->lock, flags);
6595 if (!boost)
6596 return;
6597
6598 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6599 schedule_work(&rps->work);
6600
6601 atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);
6602}
6603
6604int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6605{
6606 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6607 int err;
6608
6609 lockdep_assert_held(&dev_priv->pcu_lock);
6610 GEM_BUG_ON(val > rps->max_freq);
6611 GEM_BUG_ON(val < rps->min_freq);
6612
6613 if (!rps->enabled) {
6614 rps->cur_freq = val;
6615 return 0;
6616 }
6617
6618 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6619 err = valleyview_set_rps(dev_priv, val);
6620 else
6621 err = gen6_set_rps(dev_priv, val);
6622
6623 return err;
6624}
6625
6626static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6627{
6628 I915_WRITE(GEN6_RC_CONTROL, 0);
6629 I915_WRITE(GEN9_PG_ENABLE, 0);
6630}
6631
6632static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6633{
6634 I915_WRITE(GEN6_RP_CONTROL, 0);
6635}
6636
6637static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6638{
6639 I915_WRITE(GEN6_RC_CONTROL, 0);
6640}
6641
6642static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6643{
6644 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6645 I915_WRITE(GEN6_RP_CONTROL, 0);
6646}
6647
6648static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6649{
6650 I915_WRITE(GEN6_RC_CONTROL, 0);
6651}
6652
6653static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6654{
6655 I915_WRITE(GEN6_RP_CONTROL, 0);
6656}
6657
6658static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6659{
6660
6661
6662 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6663
6664 I915_WRITE(GEN6_RC_CONTROL, 0);
6665
6666 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6667}
6668
6669static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6670{
6671 I915_WRITE(GEN6_RP_CONTROL, 0);
6672}
6673
6674static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6675{
6676 bool enable_rc6 = true;
6677 unsigned long rc6_ctx_base;
6678 u32 rc_ctl;
6679 int rc_sw_target;
6680
6681 rc_ctl = I915_READ(GEN6_RC_CONTROL);
6682 rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6683 RC_SW_TARGET_STATE_SHIFT;
6684 DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6685 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6686 onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6687 onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6688 rc_sw_target);
6689
6690 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6691 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6692 enable_rc6 = false;
6693 }
6694
6695
6696
6697
6698
6699 rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
6700 if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
6701 (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
6702 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
6703 enable_rc6 = false;
6704 }
6705
6706 if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
6707 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
6708 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
6709 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
6710 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
6711 enable_rc6 = false;
6712 }
6713
6714 if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
6715 !I915_READ(GEN8_PUSHBUS_ENABLE) ||
6716 !I915_READ(GEN8_PUSHBUS_SHIFT)) {
6717 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
6718 enable_rc6 = false;
6719 }
6720
6721 if (!I915_READ(GEN6_GFXPAUSE)) {
6722 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
6723 enable_rc6 = false;
6724 }
6725
6726 if (!I915_READ(GEN8_MISC_CTRL0)) {
6727 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
6728 enable_rc6 = false;
6729 }
6730
6731 return enable_rc6;
6732}
6733
6734static bool sanitize_rc6(struct drm_i915_private *i915)
6735{
6736 struct intel_device_info *info = mkwrite_device_info(i915);
6737
6738
6739 if (intel_vgpu_active(i915))
6740 info->has_rc6 = 0;
6741
6742 if (info->has_rc6 &&
6743 IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
6744 DRM_INFO("RC6 disabled by BIOS\n");
6745 info->has_rc6 = 0;
6746 }
6747
6748
6749
6750
6751
6752
6753
6754 if (!info->has_rc6 && info->has_rc6p)
6755 info->has_rc6p = 0;
6756
6757 return info->has_rc6;
6758}
6759
6760static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
6761{
6762 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6763
6764
6765
6766
6767 if (IS_GEN9_LP(dev_priv)) {
6768 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
6769 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
6770 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
6771 rps->min_freq = (rp_state_cap >> 0) & 0xff;
6772 } else {
6773 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
6774 rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
6775 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
6776 rps->min_freq = (rp_state_cap >> 16) & 0xff;
6777 }
6778
6779 rps->max_freq = rps->rp0_freq;
6780
6781 rps->efficient_freq = rps->rp1_freq;
6782 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
6783 IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6784 u32 ddcc_status = 0;
6785
6786 if (sandybridge_pcode_read(dev_priv,
6787 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
6788 &ddcc_status) == 0)
6789 rps->efficient_freq =
6790 clamp_t(u8,
6791 ((ddcc_status >> 8) & 0xff),
6792 rps->min_freq,
6793 rps->max_freq);
6794 }
6795
6796 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6797
6798
6799
6800 rps->rp0_freq *= GEN9_FREQ_SCALER;
6801 rps->rp1_freq *= GEN9_FREQ_SCALER;
6802 rps->min_freq *= GEN9_FREQ_SCALER;
6803 rps->max_freq *= GEN9_FREQ_SCALER;
6804 rps->efficient_freq *= GEN9_FREQ_SCALER;
6805 }
6806}
6807
6808static void reset_rps(struct drm_i915_private *dev_priv,
6809 int (*set)(struct drm_i915_private *, u8))
6810{
6811 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6812 u8 freq = rps->cur_freq;
6813
6814
6815 rps->power.mode = -1;
6816 rps->cur_freq = -1;
6817
6818 if (set(dev_priv, freq))
6819 DRM_ERROR("Failed to reset RPS to initial values\n");
6820}
6821
6822
6823static void gen9_enable_rps(struct drm_i915_private *dev_priv)
6824{
6825 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6826
6827
6828 if (IS_GEN9(dev_priv))
6829 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6830 GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
6831
6832
6833 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
6834 GT_INTERVAL_FROM_US(dev_priv, 1000000));
6835
6836 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
6837
6838
6839
6840
6841 reset_rps(dev_priv, gen6_set_rps);
6842
6843 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6844}
6845
6846static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
6847{
6848 struct intel_engine_cs *engine;
6849 enum intel_engine_id id;
6850 u32 rc6_mode;
6851
6852
6853 I915_WRITE(GEN6_RC_STATE, 0);
6854
6855
6856
6857 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6858
6859
6860 I915_WRITE(GEN6_RC_CONTROL, 0);
6861
6862
6863 if (INTEL_GEN(dev_priv) >= 10) {
6864 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
6865 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
6866 } else if (IS_SKYLAKE(dev_priv)) {
6867
6868
6869
6870
6871 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
6872 } else {
6873 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
6874 }
6875
6876 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
6877 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
6878 for_each_engine(engine, dev_priv, id)
6879 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6880
6881 if (HAS_GUC(dev_priv))
6882 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
6883
6884 I915_WRITE(GEN6_RC_SLEEP, 0);
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
6908 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
6909
6910
6911 I915_WRITE(GEN6_RC6_THRESHOLD, 37500);
6912
6913
6914 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
6915 rc6_mode = GEN7_RC_CTL_TO_MODE;
6916 else
6917 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
6918
6919 I915_WRITE(GEN6_RC_CONTROL,
6920 GEN6_RC_CTL_HW_ENABLE |
6921 GEN6_RC_CTL_RC6_ENABLE |
6922 rc6_mode);
6923
6924
6925
6926
6927
6928 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
6929 I915_WRITE(GEN9_PG_ENABLE, 0);
6930 else
6931 I915_WRITE(GEN9_PG_ENABLE,
6932 GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
6933
6934 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6935}
6936
6937static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
6938{
6939 struct intel_engine_cs *engine;
6940 enum intel_engine_id id;
6941
6942
6943 I915_WRITE(GEN6_RC_STATE, 0);
6944
6945
6946
6947 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6948
6949
6950 I915_WRITE(GEN6_RC_CONTROL, 0);
6951
6952
6953 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
6954 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
6955 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
6956 for_each_engine(engine, dev_priv, id)
6957 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6958 I915_WRITE(GEN6_RC_SLEEP, 0);
6959 I915_WRITE(GEN6_RC6_THRESHOLD, 625);
6960
6961
6962
6963 I915_WRITE(GEN6_RC_CONTROL,
6964 GEN6_RC_CTL_HW_ENABLE |
6965 GEN7_RC_CTL_TO_MODE |
6966 GEN6_RC_CTL_RC6_ENABLE);
6967
6968 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6969}
6970
6971static void gen8_enable_rps(struct drm_i915_private *dev_priv)
6972{
6973 struct intel_rps *rps = &dev_priv->gt_pm.rps;
6974
6975 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6976
6977
6978 I915_WRITE(GEN6_RPNSWREQ,
6979 HSW_FREQUENCY(rps->rp1_freq));
6980 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6981 HSW_FREQUENCY(rps->rp1_freq));
6982
6983 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128);
6984
6985
6986 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
6987 rps->max_freq_softlimit << 24 |
6988 rps->min_freq_softlimit << 16);
6989
6990 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128);
6991 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128);
6992 I915_WRITE(GEN6_RP_UP_EI, 66000);
6993 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
6994
6995 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6996
6997
6998 I915_WRITE(GEN6_RP_CONTROL,
6999 GEN6_RP_MEDIA_TURBO |
7000 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7001 GEN6_RP_MEDIA_IS_GFX |
7002 GEN6_RP_ENABLE |
7003 GEN6_RP_UP_BUSY_AVG |
7004 GEN6_RP_DOWN_IDLE_AVG);
7005
7006 reset_rps(dev_priv, gen6_set_rps);
7007
7008 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7009}
7010
7011static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
7012{
7013 struct intel_engine_cs *engine;
7014 enum intel_engine_id id;
7015 u32 rc6vids, rc6_mask;
7016 u32 gtfifodbg;
7017 int ret;
7018
7019 I915_WRITE(GEN6_RC_STATE, 0);
7020
7021
7022 gtfifodbg = I915_READ(GTFIFODBG);
7023 if (gtfifodbg) {
7024 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
7025 I915_WRITE(GTFIFODBG, gtfifodbg);
7026 }
7027
7028 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7029
7030
7031 I915_WRITE(GEN6_RC_CONTROL, 0);
7032
7033 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7034 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7035 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7036 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7037 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7038
7039 for_each_engine(engine, dev_priv, id)
7040 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7041
7042 I915_WRITE(GEN6_RC_SLEEP, 0);
7043 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7044 if (IS_IVYBRIDGE(dev_priv))
7045 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
7046 else
7047 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7048 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
7049 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000);
7050
7051
7052 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7053 if (HAS_RC6p(dev_priv))
7054 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
7055 if (HAS_RC6pp(dev_priv))
7056 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
7057 I915_WRITE(GEN6_RC_CONTROL,
7058 rc6_mask |
7059 GEN6_RC_CTL_EI_MODE(1) |
7060 GEN6_RC_CTL_HW_ENABLE);
7061
7062 rc6vids = 0;
7063 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
7064 if (IS_GEN6(dev_priv) && ret) {
7065 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
7066 } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
7067 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
7068 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
7069 rc6vids &= 0xffff00;
7070 rc6vids |= GEN6_ENCODE_RC6_VID(450);
7071 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
7072 if (ret)
7073 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
7074 }
7075
7076 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7077}
7078
7079static void gen6_enable_rps(struct drm_i915_private *dev_priv)
7080{
7081
7082
7083
7084
7085
7086
7087 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7088
7089
7090 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
7091 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7092
7093 reset_rps(dev_priv, gen6_set_rps);
7094
7095 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7096}
7097
7098static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7099{
7100 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7101 const int min_freq = 15;
7102 const int scaling_factor = 180;
7103 unsigned int gpu_freq;
7104 unsigned int max_ia_freq, min_ring_freq;
7105 unsigned int max_gpu_freq, min_gpu_freq;
7106 struct cpufreq_policy *policy;
7107
7108 WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
7109
7110 if (rps->max_freq <= rps->min_freq)
7111 return;
7112
7113 policy = cpufreq_cpu_get(0);
7114 if (policy) {
7115 max_ia_freq = policy->cpuinfo.max_freq;
7116 cpufreq_cpu_put(policy);
7117 } else {
7118
7119
7120
7121
7122 max_ia_freq = tsc_khz;
7123 }
7124
7125
7126 max_ia_freq /= 1000;
7127
7128 min_ring_freq = I915_READ(DCLK) & 0xf;
7129
7130 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
7131
7132 min_gpu_freq = rps->min_freq;
7133 max_gpu_freq = rps->max_freq;
7134 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7135
7136 min_gpu_freq /= GEN9_FREQ_SCALER;
7137 max_gpu_freq /= GEN9_FREQ_SCALER;
7138 }
7139
7140
7141
7142
7143
7144
7145 for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
7146 const int diff = max_gpu_freq - gpu_freq;
7147 unsigned int ia_freq = 0, ring_freq = 0;
7148
7149 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7150
7151
7152
7153
7154 ring_freq = gpu_freq;
7155 } else if (INTEL_GEN(dev_priv) >= 8) {
7156
7157 ring_freq = max(min_ring_freq, gpu_freq);
7158 } else if (IS_HASWELL(dev_priv)) {
7159 ring_freq = mult_frac(gpu_freq, 5, 4);
7160 ring_freq = max(min_ring_freq, ring_freq);
7161
7162 } else {
7163
7164
7165
7166
7167
7168
7169
7170 if (gpu_freq < min_freq)
7171 ia_freq = 800;
7172 else
7173 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7174 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7175 }
7176
7177 sandybridge_pcode_write(dev_priv,
7178 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
7179 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
7180 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
7181 gpu_freq);
7182 }
7183}
7184
7185static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
7186{
7187 u32 val, rp0;
7188
7189 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7190
7191 switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
7192 case 8:
7193
7194 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
7195 break;
7196 case 12:
7197
7198 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
7199 break;
7200 case 16:
7201
7202 default:
7203
7204 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
7205 break;
7206 }
7207
7208 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
7209
7210 return rp0;
7211}
7212
7213static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7214{
7215 u32 val, rpe;
7216
7217 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
7218 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
7219
7220 return rpe;
7221}
7222
7223static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
7224{
7225 u32 val, rp1;
7226
7227 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7228 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
7229
7230 return rp1;
7231}
7232
7233static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
7234{
7235 u32 val, rpn;
7236
7237 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
7238 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
7239 FB_GFX_FREQ_FUSE_MASK);
7240
7241 return rpn;
7242}
7243
7244static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
7245{
7246 u32 val, rp1;
7247
7248 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7249
7250 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
7251
7252 return rp1;
7253}
7254
7255static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
7256{
7257 u32 val, rp0;
7258
7259 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7260
7261 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
7262
7263 rp0 = min_t(u32, rp0, 0xea);
7264
7265 return rp0;
7266}
7267
7268static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7269{
7270 u32 val, rpe;
7271
7272 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
7273 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
7274 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
7275 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
7276
7277 return rpe;
7278}
7279
7280static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7281{
7282 u32 val;
7283
7284 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7285
7286
7287
7288
7289
7290
7291
7292 return max_t(u32, val, 0xc0);
7293}
7294
7295
7296static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7297{
7298 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7299
7300 WARN_ON(pctx_addr != dev_priv->dsm.start +
7301 dev_priv->vlv_pctx->stolen->start);
7302}
7303
7304
7305
7306static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7307{
7308 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7309
7310 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7311}
7312
7313static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7314{
7315 resource_size_t pctx_paddr, paddr;
7316 resource_size_t pctx_size = 32*1024;
7317 u32 pcbr;
7318
7319 pcbr = I915_READ(VLV_PCBR);
7320 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7321 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7322 paddr = dev_priv->dsm.end + 1 - pctx_size;
7323 GEM_BUG_ON(paddr > U32_MAX);
7324
7325 pctx_paddr = (paddr & (~4095));
7326 I915_WRITE(VLV_PCBR, pctx_paddr);
7327 }
7328
7329 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7330}
7331
7332static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7333{
7334 struct drm_i915_gem_object *pctx;
7335 resource_size_t pctx_paddr;
7336 resource_size_t pctx_size = 24*1024;
7337 u32 pcbr;
7338
7339 pcbr = I915_READ(VLV_PCBR);
7340 if (pcbr) {
7341
7342 resource_size_t pcbr_offset;
7343
7344 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7345 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7346 pcbr_offset,
7347 I915_GTT_OFFSET_NONE,
7348 pctx_size);
7349 goto out;
7350 }
7351
7352 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362 pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7363 if (!pctx) {
7364 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7365 goto out;
7366 }
7367
7368 GEM_BUG_ON(range_overflows_t(u64,
7369 dev_priv->dsm.start,
7370 pctx->stolen->start,
7371 U32_MAX));
7372 pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7373 I915_WRITE(VLV_PCBR, pctx_paddr);
7374
7375out:
7376 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7377 dev_priv->vlv_pctx = pctx;
7378}
7379
7380static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7381{
7382 struct drm_i915_gem_object *pctx;
7383
7384 pctx = fetch_and_zero(&dev_priv->vlv_pctx);
7385 if (pctx)
7386 i915_gem_object_put(pctx);
7387}
7388
7389static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7390{
7391 dev_priv->gt_pm.rps.gpll_ref_freq =
7392 vlv_get_cck_clock(dev_priv, "GPLL ref",
7393 CCK_GPLL_CLOCK_CONTROL,
7394 dev_priv->czclk_freq);
7395
7396 DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7397 dev_priv->gt_pm.rps.gpll_ref_freq);
7398}
7399
7400static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7401{
7402 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7403 u32 val;
7404
7405 valleyview_setup_pctx(dev_priv);
7406
7407 vlv_init_gpll_ref_freq(dev_priv);
7408
7409 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7410 switch ((val >> 6) & 3) {
7411 case 0:
7412 case 1:
7413 dev_priv->mem_freq = 800;
7414 break;
7415 case 2:
7416 dev_priv->mem_freq = 1066;
7417 break;
7418 case 3:
7419 dev_priv->mem_freq = 1333;
7420 break;
7421 }
7422 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7423
7424 rps->max_freq = valleyview_rps_max_freq(dev_priv);
7425 rps->rp0_freq = rps->max_freq;
7426 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7427 intel_gpu_freq(dev_priv, rps->max_freq),
7428 rps->max_freq);
7429
7430 rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7431 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7432 intel_gpu_freq(dev_priv, rps->efficient_freq),
7433 rps->efficient_freq);
7434
7435 rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7436 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7437 intel_gpu_freq(dev_priv, rps->rp1_freq),
7438 rps->rp1_freq);
7439
7440 rps->min_freq = valleyview_rps_min_freq(dev_priv);
7441 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7442 intel_gpu_freq(dev_priv, rps->min_freq),
7443 rps->min_freq);
7444}
7445
7446static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7447{
7448 struct intel_rps *rps = &dev_priv->gt_pm.rps;
7449 u32 val;
7450
7451 cherryview_setup_pctx(dev_priv);
7452
7453 vlv_init_gpll_ref_freq(dev_priv);
7454
7455 mutex_lock(&dev_priv->sb_lock);
7456 val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7457 mutex_unlock(&dev_priv->sb_lock);
7458
7459 switch ((val >> 2) & 0x7) {
7460 case 3:
7461 dev_priv->mem_freq = 2000;
7462 break;
7463 default:
7464 dev_priv->mem_freq = 1600;
7465 break;
7466 }
7467 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7468
7469 rps->max_freq = cherryview_rps_max_freq(dev_priv);
7470 rps->rp0_freq = rps->max_freq;
7471 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7472 intel_gpu_freq(dev_priv, rps->max_freq),
7473 rps->max_freq);
7474
7475 rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7476 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7477 intel_gpu_freq(dev_priv, rps->efficient_freq),
7478 rps->efficient_freq);
7479
7480 rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7481 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7482 intel_gpu_freq(dev_priv, rps->rp1_freq),
7483 rps->rp1_freq);
7484
7485 rps->min_freq = cherryview_rps_min_freq(dev_priv);
7486 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7487 intel_gpu_freq(dev_priv, rps->min_freq),
7488 rps->min_freq);
7489
7490 WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7491 rps->min_freq) & 1,
7492 "Odd GPU freq values\n");
7493}
7494
7495static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7496{
7497 valleyview_cleanup_pctx(dev_priv);
7498}
7499
7500static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7501{
7502 struct intel_engine_cs *engine;
7503 enum intel_engine_id id;
7504 u32 gtfifodbg, rc6_mode, pcbr;
7505
7506 gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7507 GT_FIFO_FREE_ENTRIES_CHV);
7508 if (gtfifodbg) {
7509 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7510 gtfifodbg);
7511 I915_WRITE(GTFIFODBG, gtfifodbg);
7512 }
7513
7514 cherryview_check_pctx(dev_priv);
7515
7516
7517
7518 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7519
7520
7521 I915_WRITE(GEN6_RC_CONTROL, 0);
7522
7523
7524 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7525 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7526 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7527
7528 for_each_engine(engine, dev_priv, id)
7529 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7530 I915_WRITE(GEN6_RC_SLEEP, 0);
7531
7532
7533 I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7534
7535
7536 I915_WRITE(VLV_COUNTER_CONTROL,
7537 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7538 VLV_MEDIA_RC6_COUNT_EN |
7539 VLV_RENDER_RC6_COUNT_EN));
7540
7541
7542 pcbr = I915_READ(VLV_PCBR);
7543
7544
7545 rc6_mode = 0;
7546 if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7547 rc6_mode = GEN7_RC_CTL_TO_MODE;
7548 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7549
7550 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7551}
7552
7553static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7554{
7555 u32 val;
7556
7557 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7558
7559
7560 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7561 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7562 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7563 I915_WRITE(GEN6_RP_UP_EI, 66000);
7564 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7565
7566 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7567
7568
7569 I915_WRITE(GEN6_RP_CONTROL,
7570 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7571 GEN6_RP_MEDIA_IS_GFX |
7572 GEN6_RP_ENABLE |
7573 GEN6_RP_UP_BUSY_AVG |
7574 GEN6_RP_DOWN_IDLE_AVG);
7575
7576
7577 val = VLV_OVERRIDE_EN |
7578 VLV_SOC_TDP_EN |
7579 CHV_BIAS_CPU_50_SOC_50;
7580 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7581
7582 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7583
7584
7585 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7586
7587 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7588 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7589
7590 reset_rps(dev_priv, valleyview_set_rps);
7591
7592 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7593}
7594
7595static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7596{
7597 struct intel_engine_cs *engine;
7598 enum intel_engine_id id;
7599 u32 gtfifodbg;
7600
7601 valleyview_check_pctx(dev_priv);
7602
7603 gtfifodbg = I915_READ(GTFIFODBG);
7604 if (gtfifodbg) {
7605 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7606 gtfifodbg);
7607 I915_WRITE(GTFIFODBG, gtfifodbg);
7608 }
7609
7610 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7611
7612
7613 I915_WRITE(GEN6_RC_CONTROL, 0);
7614
7615 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
7616 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7617 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7618
7619 for_each_engine(engine, dev_priv, id)
7620 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7621
7622 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
7623
7624
7625 I915_WRITE(VLV_COUNTER_CONTROL,
7626 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7627 VLV_MEDIA_RC0_COUNT_EN |
7628 VLV_RENDER_RC0_COUNT_EN |
7629 VLV_MEDIA_RC6_COUNT_EN |
7630 VLV_RENDER_RC6_COUNT_EN));
7631
7632 I915_WRITE(GEN6_RC_CONTROL,
7633 GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
7634
7635 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7636}
7637
7638static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
7639{
7640 u32 val;
7641
7642 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7643
7644 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7645 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7646 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7647 I915_WRITE(GEN6_RP_UP_EI, 66000);
7648 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7649
7650 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7651
7652 I915_WRITE(GEN6_RP_CONTROL,
7653 GEN6_RP_MEDIA_TURBO |
7654 GEN6_RP_MEDIA_HW_NORMAL_MODE |
7655 GEN6_RP_MEDIA_IS_GFX |
7656 GEN6_RP_ENABLE |
7657 GEN6_RP_UP_BUSY_AVG |
7658 GEN6_RP_DOWN_IDLE_CONT);
7659
7660
7661 val = VLV_OVERRIDE_EN |
7662 VLV_SOC_TDP_EN |
7663 VLV_BIAS_CPU_125_SOC_875;
7664 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7665
7666 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7667
7668
7669 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7670
7671 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7672 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7673
7674 reset_rps(dev_priv, valleyview_set_rps);
7675
7676 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7677}
7678
7679static unsigned long intel_pxfreq(u32 vidfreq)
7680{
7681 unsigned long freq;
7682 int div = (vidfreq & 0x3f0000) >> 16;
7683 int post = (vidfreq & 0x3000) >> 12;
7684 int pre = (vidfreq & 0x7);
7685
7686 if (!pre)
7687 return 0;
7688
7689 freq = ((div * 133333) / ((1<<post) * pre));
7690
7691 return freq;
7692}
7693
7694static const struct cparams {
7695 u16 i;
7696 u16 t;
7697 u16 m;
7698 u16 c;
7699} cparams[] = {
7700 { 1, 1333, 301, 28664 },
7701 { 1, 1066, 294, 24460 },
7702 { 1, 800, 294, 25192 },
7703 { 0, 1333, 276, 27605 },
7704 { 0, 1066, 276, 27605 },
7705 { 0, 800, 231, 23784 },
7706};
7707
7708static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
7709{
7710 u64 total_count, diff, ret;
7711 u32 count1, count2, count3, m = 0, c = 0;
7712 unsigned long now = jiffies_to_msecs(jiffies), diff1;
7713 int i;
7714
7715 lockdep_assert_held(&mchdev_lock);
7716
7717 diff1 = now - dev_priv->ips.last_time1;
7718
7719
7720
7721
7722
7723
7724 if (diff1 <= 10)
7725 return dev_priv->ips.chipset_power;
7726
7727 count1 = I915_READ(DMIEC);
7728 count2 = I915_READ(DDREC);
7729 count3 = I915_READ(CSIEC);
7730
7731 total_count = count1 + count2 + count3;
7732
7733
7734 if (total_count < dev_priv->ips.last_count1) {
7735 diff = ~0UL - dev_priv->ips.last_count1;
7736 diff += total_count;
7737 } else {
7738 diff = total_count - dev_priv->ips.last_count1;
7739 }
7740
7741 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
7742 if (cparams[i].i == dev_priv->ips.c_m &&
7743 cparams[i].t == dev_priv->ips.r_t) {
7744 m = cparams[i].m;
7745 c = cparams[i].c;
7746 break;
7747 }
7748 }
7749
7750 diff = div_u64(diff, diff1);
7751 ret = ((m * diff) + c);
7752 ret = div_u64(ret, 10);
7753
7754 dev_priv->ips.last_count1 = total_count;
7755 dev_priv->ips.last_time1 = now;
7756
7757 dev_priv->ips.chipset_power = ret;
7758
7759 return ret;
7760}
7761
7762unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
7763{
7764 unsigned long val;
7765
7766 if (!IS_GEN5(dev_priv))
7767 return 0;
7768
7769 spin_lock_irq(&mchdev_lock);
7770
7771 val = __i915_chipset_val(dev_priv);
7772
7773 spin_unlock_irq(&mchdev_lock);
7774
7775 return val;
7776}
7777
7778unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
7779{
7780 unsigned long m, x, b;
7781 u32 tsfs;
7782
7783 tsfs = I915_READ(TSFS);
7784
7785 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
7786 x = I915_READ8(TR1);
7787
7788 b = tsfs & TSFS_INTR_MASK;
7789
7790 return ((m * x) / 127) - b;
7791}
7792
7793static int _pxvid_to_vd(u8 pxvid)
7794{
7795 if (pxvid == 0)
7796 return 0;
7797
7798 if (pxvid >= 8 && pxvid < 31)
7799 pxvid = 31;
7800
7801 return (pxvid + 2) * 125;
7802}
7803
7804static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
7805{
7806 const int vd = _pxvid_to_vd(pxvid);
7807 const int vm = vd - 1125;
7808
7809 if (INTEL_INFO(dev_priv)->is_mobile)
7810 return vm > 0 ? vm : 0;
7811
7812 return vd;
7813}
7814
7815static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
7816{
7817 u64 now, diff, diffms;
7818 u32 count;
7819
7820 lockdep_assert_held(&mchdev_lock);
7821
7822 now = ktime_get_raw_ns();
7823 diffms = now - dev_priv->ips.last_time2;
7824 do_div(diffms, NSEC_PER_MSEC);
7825
7826
7827 if (!diffms)
7828 return;
7829
7830 count = I915_READ(GFXEC);
7831
7832 if (count < dev_priv->ips.last_count2) {
7833 diff = ~0UL - dev_priv->ips.last_count2;
7834 diff += count;
7835 } else {
7836 diff = count - dev_priv->ips.last_count2;
7837 }
7838
7839 dev_priv->ips.last_count2 = count;
7840 dev_priv->ips.last_time2 = now;
7841
7842
7843 diff = diff * 1181;
7844 diff = div_u64(diff, diffms * 10);
7845 dev_priv->ips.gfx_power = diff;
7846}
7847
7848void i915_update_gfx_val(struct drm_i915_private *dev_priv)
7849{
7850 if (!IS_GEN5(dev_priv))
7851 return;
7852
7853 spin_lock_irq(&mchdev_lock);
7854
7855 __i915_update_gfx_val(dev_priv);
7856
7857 spin_unlock_irq(&mchdev_lock);
7858}
7859
7860static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
7861{
7862 unsigned long t, corr, state1, corr2, state2;
7863 u32 pxvid, ext_v;
7864
7865 lockdep_assert_held(&mchdev_lock);
7866
7867 pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
7868 pxvid = (pxvid >> 24) & 0x7f;
7869 ext_v = pvid_to_extvid(dev_priv, pxvid);
7870
7871 state1 = ext_v;
7872
7873 t = i915_mch_val(dev_priv);
7874
7875
7876
7877
7878 if (t > 80)
7879 corr = ((t * 2349) + 135940);
7880 else if (t >= 50)
7881 corr = ((t * 964) + 29317);
7882 else
7883 corr = ((t * 301) + 1004);
7884
7885 corr = corr * ((150142 * state1) / 10000 - 78642);
7886 corr /= 100000;
7887 corr2 = (corr * dev_priv->ips.corr);
7888
7889 state2 = (corr2 * state1) / 10000;
7890 state2 /= 100;
7891
7892 __i915_update_gfx_val(dev_priv);
7893
7894 return dev_priv->ips.gfx_power + state2;
7895}
7896
7897unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
7898{
7899 unsigned long val;
7900
7901 if (!IS_GEN5(dev_priv))
7902 return 0;
7903
7904 spin_lock_irq(&mchdev_lock);
7905
7906 val = __i915_gfx_val(dev_priv);
7907
7908 spin_unlock_irq(&mchdev_lock);
7909
7910 return val;
7911}
7912
7913
7914
7915
7916
7917
7918
7919unsigned long i915_read_mch_val(void)
7920{
7921 struct drm_i915_private *dev_priv;
7922 unsigned long chipset_val, graphics_val, ret = 0;
7923
7924 spin_lock_irq(&mchdev_lock);
7925 if (!i915_mch_dev)
7926 goto out_unlock;
7927 dev_priv = i915_mch_dev;
7928
7929 chipset_val = __i915_chipset_val(dev_priv);
7930 graphics_val = __i915_gfx_val(dev_priv);
7931
7932 ret = chipset_val + graphics_val;
7933
7934out_unlock:
7935 spin_unlock_irq(&mchdev_lock);
7936
7937 return ret;
7938}
7939EXPORT_SYMBOL_GPL(i915_read_mch_val);
7940
7941
7942
7943
7944
7945
7946bool i915_gpu_raise(void)
7947{
7948 struct drm_i915_private *dev_priv;
7949 bool ret = true;
7950
7951 spin_lock_irq(&mchdev_lock);
7952 if (!i915_mch_dev) {
7953 ret = false;
7954 goto out_unlock;
7955 }
7956 dev_priv = i915_mch_dev;
7957
7958 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
7959 dev_priv->ips.max_delay--;
7960
7961out_unlock:
7962 spin_unlock_irq(&mchdev_lock);
7963
7964 return ret;
7965}
7966EXPORT_SYMBOL_GPL(i915_gpu_raise);
7967
7968
7969
7970
7971
7972
7973
7974bool i915_gpu_lower(void)
7975{
7976 struct drm_i915_private *dev_priv;
7977 bool ret = true;
7978
7979 spin_lock_irq(&mchdev_lock);
7980 if (!i915_mch_dev) {
7981 ret = false;
7982 goto out_unlock;
7983 }
7984 dev_priv = i915_mch_dev;
7985
7986 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
7987 dev_priv->ips.max_delay++;
7988
7989out_unlock:
7990 spin_unlock_irq(&mchdev_lock);
7991
7992 return ret;
7993}
7994EXPORT_SYMBOL_GPL(i915_gpu_lower);
7995
7996
7997
7998
7999
8000
8001bool i915_gpu_busy(void)
8002{
8003 bool ret = false;
8004
8005 spin_lock_irq(&mchdev_lock);
8006 if (i915_mch_dev)
8007 ret = i915_mch_dev->gt.awake;
8008 spin_unlock_irq(&mchdev_lock);
8009
8010 return ret;
8011}
8012EXPORT_SYMBOL_GPL(i915_gpu_busy);
8013
8014
8015
8016
8017
8018
8019
8020bool i915_gpu_turbo_disable(void)
8021{
8022 struct drm_i915_private *dev_priv;
8023 bool ret = true;
8024
8025 spin_lock_irq(&mchdev_lock);
8026 if (!i915_mch_dev) {
8027 ret = false;
8028 goto out_unlock;
8029 }
8030 dev_priv = i915_mch_dev;
8031
8032 dev_priv->ips.max_delay = dev_priv->ips.fstart;
8033
8034 if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
8035 ret = false;
8036
8037out_unlock:
8038 spin_unlock_irq(&mchdev_lock);
8039
8040 return ret;
8041}
8042EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052static void
8053ips_ping_for_i915_load(void)
8054{
8055 void (*link)(void);
8056
8057 link = symbol_get(ips_link_to_i915_driver);
8058 if (link) {
8059 link();
8060 symbol_put(ips_link_to_i915_driver);
8061 }
8062}
8063
8064void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
8065{
8066
8067
8068 spin_lock_irq(&mchdev_lock);
8069 i915_mch_dev = dev_priv;
8070 spin_unlock_irq(&mchdev_lock);
8071
8072 ips_ping_for_i915_load();
8073}
8074
8075void intel_gpu_ips_teardown(void)
8076{
8077 spin_lock_irq(&mchdev_lock);
8078 i915_mch_dev = NULL;
8079 spin_unlock_irq(&mchdev_lock);
8080}
8081
8082static void intel_init_emon(struct drm_i915_private *dev_priv)
8083{
8084 u32 lcfuse;
8085 u8 pxw[16];
8086 int i;
8087
8088
8089 I915_WRITE(ECR, 0);
8090 POSTING_READ(ECR);
8091
8092
8093 I915_WRITE(SDEW, 0x15040d00);
8094 I915_WRITE(CSIEW0, 0x007f0000);
8095 I915_WRITE(CSIEW1, 0x1e220004);
8096 I915_WRITE(CSIEW2, 0x04000004);
8097
8098 for (i = 0; i < 5; i++)
8099 I915_WRITE(PEW(i), 0);
8100 for (i = 0; i < 3; i++)
8101 I915_WRITE(DEW(i), 0);
8102
8103
8104 for (i = 0; i < 16; i++) {
8105 u32 pxvidfreq = I915_READ(PXVFREQ(i));
8106 unsigned long freq = intel_pxfreq(pxvidfreq);
8107 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8108 PXVFREQ_PX_SHIFT;
8109 unsigned long val;
8110
8111 val = vid * vid;
8112 val *= (freq / 1000);
8113 val *= 255;
8114 val /= (127*127*900);
8115 if (val > 0xff)
8116 DRM_ERROR("bad pxval: %ld\n", val);
8117 pxw[i] = val;
8118 }
8119
8120 pxw[14] = 0;
8121 pxw[15] = 0;
8122
8123 for (i = 0; i < 4; i++) {
8124 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8125 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8126 I915_WRITE(PXW(i), val);
8127 }
8128
8129
8130 I915_WRITE(OGW0, 0);
8131 I915_WRITE(OGW1, 0);
8132 I915_WRITE(EG0, 0x00007f00);
8133 I915_WRITE(EG1, 0x0000000e);
8134 I915_WRITE(EG2, 0x000e0000);
8135 I915_WRITE(EG3, 0x68000300);
8136 I915_WRITE(EG4, 0x42000000);
8137 I915_WRITE(EG5, 0x00140031);
8138 I915_WRITE(EG6, 0);
8139 I915_WRITE(EG7, 0);
8140
8141 for (i = 0; i < 8; i++)
8142 I915_WRITE(PXWL(i), 0);
8143
8144
8145 I915_WRITE(ECR, 0x80000019);
8146
8147 lcfuse = I915_READ(LCFUSE02);
8148
8149 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8150}
8151
8152void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8153{
8154 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8155
8156
8157
8158
8159
8160 if (!sanitize_rc6(dev_priv)) {
8161 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
8162 intel_runtime_pm_get(dev_priv);
8163 }
8164
8165 mutex_lock(&dev_priv->pcu_lock);
8166
8167
8168 if (IS_CHERRYVIEW(dev_priv))
8169 cherryview_init_gt_powersave(dev_priv);
8170 else if (IS_VALLEYVIEW(dev_priv))
8171 valleyview_init_gt_powersave(dev_priv);
8172 else if (INTEL_GEN(dev_priv) >= 6)
8173 gen6_init_rps_frequencies(dev_priv);
8174
8175
8176 rps->idle_freq = rps->min_freq;
8177 rps->cur_freq = rps->idle_freq;
8178
8179 rps->max_freq_softlimit = rps->max_freq;
8180 rps->min_freq_softlimit = rps->min_freq;
8181
8182 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
8183 rps->min_freq_softlimit =
8184 max_t(int,
8185 rps->efficient_freq,
8186 intel_freq_opcode(dev_priv, 450));
8187
8188
8189 if (IS_GEN6(dev_priv) ||
8190 IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
8191 u32 params = 0;
8192
8193 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, ¶ms);
8194 if (params & BIT(31)) {
8195 DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
8196 (rps->max_freq & 0xff) * 50,
8197 (params & 0xff) * 50);
8198 rps->max_freq = params & 0xff;
8199 }
8200 }
8201
8202
8203 rps->boost_freq = rps->max_freq;
8204
8205 mutex_unlock(&dev_priv->pcu_lock);
8206}
8207
8208void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8209{
8210 if (IS_VALLEYVIEW(dev_priv))
8211 valleyview_cleanup_gt_powersave(dev_priv);
8212
8213 if (!HAS_RC6(dev_priv))
8214 intel_runtime_pm_put(dev_priv);
8215}
8216
8217
8218
8219
8220
8221
8222
8223
8224
8225void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
8226{
8227 if (INTEL_GEN(dev_priv) < 6)
8228 return;
8229
8230
8231}
8232
8233void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8234{
8235 dev_priv->gt_pm.rps.enabled = true;
8236 dev_priv->gt_pm.rc6.enabled = true;
8237 intel_disable_gt_powersave(dev_priv);
8238
8239 if (INTEL_GEN(dev_priv) >= 11)
8240 gen11_reset_rps_interrupts(dev_priv);
8241 else
8242 gen6_reset_rps_interrupts(dev_priv);
8243}
8244
8245static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8246{
8247 lockdep_assert_held(&i915->pcu_lock);
8248
8249 if (!i915->gt_pm.llc_pstate.enabled)
8250 return;
8251
8252
8253
8254 i915->gt_pm.llc_pstate.enabled = false;
8255}
8256
8257static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8258{
8259 lockdep_assert_held(&dev_priv->pcu_lock);
8260
8261 if (!dev_priv->gt_pm.rc6.enabled)
8262 return;
8263
8264 if (INTEL_GEN(dev_priv) >= 9)
8265 gen9_disable_rc6(dev_priv);
8266 else if (IS_CHERRYVIEW(dev_priv))
8267 cherryview_disable_rc6(dev_priv);
8268 else if (IS_VALLEYVIEW(dev_priv))
8269 valleyview_disable_rc6(dev_priv);
8270 else if (INTEL_GEN(dev_priv) >= 6)
8271 gen6_disable_rc6(dev_priv);
8272
8273 dev_priv->gt_pm.rc6.enabled = false;
8274}
8275
8276static void intel_disable_rps(struct drm_i915_private *dev_priv)
8277{
8278 lockdep_assert_held(&dev_priv->pcu_lock);
8279
8280 if (!dev_priv->gt_pm.rps.enabled)
8281 return;
8282
8283 if (INTEL_GEN(dev_priv) >= 9)
8284 gen9_disable_rps(dev_priv);
8285 else if (IS_CHERRYVIEW(dev_priv))
8286 cherryview_disable_rps(dev_priv);
8287 else if (IS_VALLEYVIEW(dev_priv))
8288 valleyview_disable_rps(dev_priv);
8289 else if (INTEL_GEN(dev_priv) >= 6)
8290 gen6_disable_rps(dev_priv);
8291 else if (IS_IRONLAKE_M(dev_priv))
8292 ironlake_disable_drps(dev_priv);
8293
8294 dev_priv->gt_pm.rps.enabled = false;
8295}
8296
8297void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8298{
8299 mutex_lock(&dev_priv->pcu_lock);
8300
8301 intel_disable_rc6(dev_priv);
8302 intel_disable_rps(dev_priv);
8303 if (HAS_LLC(dev_priv))
8304 intel_disable_llc_pstate(dev_priv);
8305
8306 mutex_unlock(&dev_priv->pcu_lock);
8307}
8308
8309static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8310{
8311 lockdep_assert_held(&i915->pcu_lock);
8312
8313 if (i915->gt_pm.llc_pstate.enabled)
8314 return;
8315
8316 gen6_update_ring_freq(i915);
8317
8318 i915->gt_pm.llc_pstate.enabled = true;
8319}
8320
8321static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8322{
8323 lockdep_assert_held(&dev_priv->pcu_lock);
8324
8325 if (dev_priv->gt_pm.rc6.enabled)
8326 return;
8327
8328 if (IS_CHERRYVIEW(dev_priv))
8329 cherryview_enable_rc6(dev_priv);
8330 else if (IS_VALLEYVIEW(dev_priv))
8331 valleyview_enable_rc6(dev_priv);
8332 else if (INTEL_GEN(dev_priv) >= 9)
8333 gen9_enable_rc6(dev_priv);
8334 else if (IS_BROADWELL(dev_priv))
8335 gen8_enable_rc6(dev_priv);
8336 else if (INTEL_GEN(dev_priv) >= 6)
8337 gen6_enable_rc6(dev_priv);
8338
8339 dev_priv->gt_pm.rc6.enabled = true;
8340}
8341
8342static void intel_enable_rps(struct drm_i915_private *dev_priv)
8343{
8344 struct intel_rps *rps = &dev_priv->gt_pm.rps;
8345
8346 lockdep_assert_held(&dev_priv->pcu_lock);
8347
8348 if (rps->enabled)
8349 return;
8350
8351 if (IS_CHERRYVIEW(dev_priv)) {
8352 cherryview_enable_rps(dev_priv);
8353 } else if (IS_VALLEYVIEW(dev_priv)) {
8354 valleyview_enable_rps(dev_priv);
8355 } else if (INTEL_GEN(dev_priv) >= 9) {
8356 gen9_enable_rps(dev_priv);
8357 } else if (IS_BROADWELL(dev_priv)) {
8358 gen8_enable_rps(dev_priv);
8359 } else if (INTEL_GEN(dev_priv) >= 6) {
8360 gen6_enable_rps(dev_priv);
8361 } else if (IS_IRONLAKE_M(dev_priv)) {
8362 ironlake_enable_drps(dev_priv);
8363 intel_init_emon(dev_priv);
8364 }
8365
8366 WARN_ON(rps->max_freq < rps->min_freq);
8367 WARN_ON(rps->idle_freq > rps->max_freq);
8368
8369 WARN_ON(rps->efficient_freq < rps->min_freq);
8370 WARN_ON(rps->efficient_freq > rps->max_freq);
8371
8372 rps->enabled = true;
8373}
8374
8375void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8376{
8377
8378 if (intel_vgpu_active(dev_priv))
8379 return;
8380
8381 mutex_lock(&dev_priv->pcu_lock);
8382
8383 if (HAS_RC6(dev_priv))
8384 intel_enable_rc6(dev_priv);
8385 intel_enable_rps(dev_priv);
8386 if (HAS_LLC(dev_priv))
8387 intel_enable_llc_pstate(dev_priv);
8388
8389 mutex_unlock(&dev_priv->pcu_lock);
8390}
8391
8392static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8393{
8394
8395
8396
8397
8398
8399 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8400}
8401
8402static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8403{
8404 enum pipe pipe;
8405
8406 for_each_pipe(dev_priv, pipe) {
8407 I915_WRITE(DSPCNTR(pipe),
8408 I915_READ(DSPCNTR(pipe)) |
8409 DISPPLANE_TRICKLE_FEED_DISABLE);
8410
8411 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8412 POSTING_READ(DSPSURF(pipe));
8413 }
8414}
8415
8416static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8417{
8418 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8419
8420
8421
8422
8423
8424 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8425 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8426 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8427
8428 I915_WRITE(PCH_3DCGDIS0,
8429 MARIUNIT_CLOCK_GATE_DISABLE |
8430 SVSMUNIT_CLOCK_GATE_DISABLE);
8431 I915_WRITE(PCH_3DCGDIS1,
8432 VFMUNIT_CLOCK_GATE_DISABLE);
8433
8434
8435
8436
8437
8438
8439
8440
8441 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8442 (I915_READ(ILK_DISPLAY_CHICKEN2) |
8443 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8444 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8445 I915_WRITE(DISP_ARB_CTL,
8446 (I915_READ(DISP_ARB_CTL) |
8447 DISP_FBC_WM_DIS));
8448
8449
8450
8451
8452
8453
8454
8455
8456 if (IS_IRONLAKE_M(dev_priv)) {
8457
8458 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8459 I915_READ(ILK_DISPLAY_CHICKEN1) |
8460 ILK_FBCQ_DIS);
8461 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8462 I915_READ(ILK_DISPLAY_CHICKEN2) |
8463 ILK_DPARB_GATE);
8464 }
8465
8466 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8467
8468 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8469 I915_READ(ILK_DISPLAY_CHICKEN2) |
8470 ILK_ELPIN_409_SELECT);
8471 I915_WRITE(_3D_CHICKEN2,
8472 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8473 _3D_CHICKEN2_WM_READ_PIPELINED);
8474
8475
8476 I915_WRITE(CACHE_MODE_0,
8477 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8478
8479
8480 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8481
8482 g4x_disable_trickle_feed(dev_priv);
8483
8484 ibx_init_clock_gating(dev_priv);
8485}
8486
8487static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8488{
8489 int pipe;
8490 uint32_t val;
8491
8492
8493
8494
8495
8496
8497 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8498 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8499 PCH_CPUNIT_CLOCK_GATE_DISABLE);
8500 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8501 DPLS_EDP_PPS_FIX_DIS);
8502
8503
8504
8505 for_each_pipe(dev_priv, pipe) {
8506 val = I915_READ(TRANS_CHICKEN2(pipe));
8507 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8508 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8509 if (dev_priv->vbt.fdi_rx_polarity_inverted)
8510 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8511 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
8512 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
8513 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
8514 I915_WRITE(TRANS_CHICKEN2(pipe), val);
8515 }
8516
8517 for_each_pipe(dev_priv, pipe) {
8518 I915_WRITE(TRANS_CHICKEN1(pipe),
8519 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8520 }
8521}
8522
8523static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
8524{
8525 uint32_t tmp;
8526
8527 tmp = I915_READ(MCH_SSKPD);
8528 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
8529 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
8530 tmp);
8531}
8532
8533static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
8534{
8535 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8536
8537 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8538
8539 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8540 I915_READ(ILK_DISPLAY_CHICKEN2) |
8541 ILK_ELPIN_409_SELECT);
8542
8543
8544 I915_WRITE(_3D_CHICKEN,
8545 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
8546
8547
8548 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558 I915_WRITE(GEN6_GT_MODE,
8559 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8560
8561 I915_WRITE(CACHE_MODE_0,
8562 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
8563
8564 I915_WRITE(GEN6_UCGCTL1,
8565 I915_READ(GEN6_UCGCTL1) |
8566 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
8567 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582 I915_WRITE(GEN6_UCGCTL2,
8583 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8584 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8585
8586
8587 I915_WRITE(_3D_CHICKEN3,
8588 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
8589
8590
8591
8592
8593
8594
8595 I915_WRITE(_3D_CHICKEN3,
8596 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
8597
8598
8599
8600
8601
8602
8603
8604
8605
8606
8607
8608
8609 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8610 I915_READ(ILK_DISPLAY_CHICKEN1) |
8611 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8612 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8613 I915_READ(ILK_DISPLAY_CHICKEN2) |
8614 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8615 I915_WRITE(ILK_DSPCLK_GATE_D,
8616 I915_READ(ILK_DSPCLK_GATE_D) |
8617 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
8618 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
8619
8620 g4x_disable_trickle_feed(dev_priv);
8621
8622 cpt_init_clock_gating(dev_priv);
8623
8624 gen6_check_mch_setup(dev_priv);
8625}
8626
8627static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
8628{
8629 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
8630
8631
8632
8633
8634
8635
8636
8637 reg &= ~GEN7_FF_SCHED_MASK;
8638 reg |= GEN7_FF_TS_SCHED_HW;
8639 reg |= GEN7_FF_VS_SCHED_HW;
8640 reg |= GEN7_FF_DS_SCHED_HW;
8641
8642 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
8643}
8644
8645static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
8646{
8647
8648
8649
8650
8651 if (HAS_PCH_LPT_LP(dev_priv))
8652 I915_WRITE(SOUTH_DSPCLK_GATE_D,
8653 I915_READ(SOUTH_DSPCLK_GATE_D) |
8654 PCH_LP_PARTITION_LEVEL_DISABLE);
8655
8656
8657 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
8658 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
8659 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8660}
8661
8662static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
8663{
8664 if (HAS_PCH_LPT_LP(dev_priv)) {
8665 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
8666
8667 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
8668 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
8669 }
8670}
8671
8672static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
8673 int general_prio_credits,
8674 int high_prio_credits)
8675{
8676 u32 misccpctl;
8677 u32 val;
8678
8679
8680 misccpctl = I915_READ(GEN7_MISCCPCTL);
8681 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
8682
8683 val = I915_READ(GEN8_L3SQCREG1);
8684 val &= ~L3_PRIO_CREDITS_MASK;
8685 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
8686 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
8687 I915_WRITE(GEN8_L3SQCREG1, val);
8688
8689
8690
8691
8692
8693 POSTING_READ(GEN8_L3SQCREG1);
8694 udelay(1);
8695 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
8696}
8697
8698static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
8699{
8700
8701 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
8702 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
8703}
8704
8705static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
8706{
8707 if (!HAS_PCH_CNP(dev_priv))
8708 return;
8709
8710
8711 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
8712 CNP_PWM_CGE_GATING_DISABLE);
8713}
8714
8715static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
8716{
8717 u32 val;
8718 cnp_init_clock_gating(dev_priv);
8719
8720
8721 I915_WRITE(_3D_CHICKEN3,
8722 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
8723
8724
8725 I915_WRITE(GEN8_CHICKEN_DCPR_1,
8726 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
8727
8728
8729 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
8730 DISP_FBC_MEMORY_WAKE);
8731
8732 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
8733
8734 val |= RCCUNIT_CLKGATE_DIS;
8735
8736 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
8737 val |= SARBUNIT_CLKGATE_DIS;
8738 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
8739
8740
8741 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
8742 val |= GWUNIT_CLKGATE_DIS;
8743 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
8744
8745
8746
8747 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
8748 val |= VFUNIT_CLKGATE_DIS;
8749 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
8750}
8751
8752static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
8753{
8754 cnp_init_clock_gating(dev_priv);
8755 gen9_init_clock_gating(dev_priv);
8756
8757
8758 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8759 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8760}
8761
8762static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
8763{
8764 gen9_init_clock_gating(dev_priv);
8765
8766
8767 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8768 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8769 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8770
8771
8772 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8773 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
8774 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
8775
8776
8777 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8778 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8779}
8780
8781static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
8782{
8783 gen9_init_clock_gating(dev_priv);
8784
8785
8786 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
8787 FBC_LLC_FULLY_OPEN);
8788
8789
8790 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8791 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8792}
8793
8794static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
8795{
8796
8797 bool can_use_gtt_cache = !HAS_PAGE_SIZES(dev_priv,
8798 I915_GTT_PAGE_SIZE_2M);
8799 enum pipe pipe;
8800
8801
8802 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8803
8804
8805 I915_WRITE(CHICKEN_PAR1_1,
8806 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
8807
8808
8809 for_each_pipe(dev_priv, pipe) {
8810 I915_WRITE(CHICKEN_PIPESL_1(pipe),
8811 I915_READ(CHICKEN_PIPESL_1(pipe)) |
8812 BDW_DPRS_MASK_VBLANK_SRD);
8813 }
8814
8815
8816
8817 I915_WRITE(GEN7_FF_THREAD_MODE,
8818 I915_READ(GEN7_FF_THREAD_MODE) &
8819 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
8820
8821 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
8822 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
8823
8824
8825 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8826 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8827
8828
8829 gen8_set_l3sqc_credits(dev_priv, 30, 2);
8830
8831
8832 I915_WRITE(HSW_GTT_CACHE_EN, can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
8833
8834
8835 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
8836 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
8837
8838 lpt_init_clock_gating(dev_priv);
8839
8840
8841
8842
8843
8844
8845 I915_WRITE(GEN6_UCGCTL1,
8846 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
8847}
8848
8849static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
8850{
8851
8852 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
8853 I915_WRITE(HSW_ROW_CHICKEN3,
8854 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
8855
8856
8857 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8858 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8859 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8860
8861
8862 I915_WRITE(GEN7_FF_THREAD_MODE,
8863 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
8864
8865
8866 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8867
8868
8869 I915_WRITE(CACHE_MODE_0_GEN7,
8870 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8871
8872
8873 I915_WRITE(CACHE_MODE_1,
8874 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8875
8876
8877
8878
8879
8880
8881
8882
8883
8884 I915_WRITE(GEN7_GT_MODE,
8885 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8886
8887
8888 I915_WRITE(HALF_SLICE_CHICKEN3,
8889 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
8890
8891
8892 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8893
8894 lpt_init_clock_gating(dev_priv);
8895}
8896
8897static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
8898{
8899 uint32_t snpcr;
8900
8901 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
8902
8903
8904 I915_WRITE(_3D_CHICKEN3,
8905 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
8906
8907
8908 I915_WRITE(IVB_CHICKEN3,
8909 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8910 CHICKEN3_DGMG_DONE_FIX_DISABLE);
8911
8912
8913 if (IS_IVB_GT1(dev_priv))
8914 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
8915 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
8916
8917
8918 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8919
8920
8921 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
8922 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
8923
8924
8925 I915_WRITE(GEN7_L3CNTLREG1,
8926 GEN7_WA_FOR_GEN7_L3_CONTROL);
8927 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8928 GEN7_WA_L3_CHICKEN_MODE);
8929 if (IS_IVB_GT1(dev_priv))
8930 I915_WRITE(GEN7_ROW_CHICKEN2,
8931 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8932 else {
8933
8934 I915_WRITE(GEN7_ROW_CHICKEN2,
8935 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8936 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
8937 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8938 }
8939
8940
8941 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
8942 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
8943
8944
8945
8946
8947
8948 I915_WRITE(GEN6_UCGCTL2,
8949 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
8950
8951
8952 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8953 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8954 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8955
8956 g4x_disable_trickle_feed(dev_priv);
8957
8958 gen7_setup_fixed_func_scheduler(dev_priv);
8959
8960 if (0) {
8961
8962 I915_WRITE(CACHE_MODE_0_GEN7,
8963 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8964 }
8965
8966
8967 I915_WRITE(CACHE_MODE_1,
8968 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8969
8970
8971
8972
8973
8974
8975
8976
8977
8978 I915_WRITE(GEN7_GT_MODE,
8979 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8980
8981 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
8982 snpcr &= ~GEN6_MBC_SNPCR_MASK;
8983 snpcr |= GEN6_MBC_SNPCR_MED;
8984 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
8985
8986 if (!HAS_PCH_NOP(dev_priv))
8987 cpt_init_clock_gating(dev_priv);
8988
8989 gen6_check_mch_setup(dev_priv);
8990}
8991
8992static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
8993{
8994
8995 I915_WRITE(_3D_CHICKEN3,
8996 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
8997
8998
8999 I915_WRITE(IVB_CHICKEN3,
9000 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9001 CHICKEN3_DGMG_DONE_FIX_DISABLE);
9002
9003
9004
9005 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9006 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
9007 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9008
9009
9010 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9011
9012
9013 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9014 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9015
9016
9017 I915_WRITE(GEN7_ROW_CHICKEN2,
9018 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9019
9020
9021 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9022 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9023 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9024
9025 gen7_setup_fixed_func_scheduler(dev_priv);
9026
9027
9028
9029
9030
9031 I915_WRITE(GEN6_UCGCTL2,
9032 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9033
9034
9035
9036
9037 I915_WRITE(GEN7_UCGCTL4,
9038 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
9039
9040
9041
9042
9043
9044 I915_WRITE(CACHE_MODE_1,
9045 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9046
9047
9048
9049
9050
9051
9052
9053
9054
9055 I915_WRITE(GEN7_GT_MODE,
9056 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9057
9058
9059
9060
9061
9062 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
9063
9064
9065
9066
9067
9068
9069 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
9070}
9071
9072static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
9073{
9074
9075
9076 I915_WRITE(GEN7_FF_THREAD_MODE,
9077 I915_READ(GEN7_FF_THREAD_MODE) &
9078 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9079
9080
9081 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9082 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9083
9084
9085 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9086 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9087
9088
9089 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9090 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9091
9092
9093
9094
9095
9096
9097 gen8_set_l3sqc_credits(dev_priv, 38, 2);
9098
9099
9100
9101
9102
9103 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
9104}
9105
9106static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
9107{
9108 uint32_t dspclk_gate;
9109
9110 I915_WRITE(RENCLK_GATE_D1, 0);
9111 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
9112 GS_UNIT_CLOCK_GATE_DISABLE |
9113 CL_UNIT_CLOCK_GATE_DISABLE);
9114 I915_WRITE(RAMCLK_GATE_D, 0);
9115 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
9116 OVRUNIT_CLOCK_GATE_DISABLE |
9117 OVCUNIT_CLOCK_GATE_DISABLE;
9118 if (IS_GM45(dev_priv))
9119 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
9120 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
9121
9122
9123 I915_WRITE(CACHE_MODE_0,
9124 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
9125
9126
9127 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9128
9129 g4x_disable_trickle_feed(dev_priv);
9130}
9131
9132static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
9133{
9134 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
9135 I915_WRITE(RENCLK_GATE_D2, 0);
9136 I915_WRITE(DSPCLK_GATE_D, 0);
9137 I915_WRITE(RAMCLK_GATE_D, 0);
9138 I915_WRITE16(DEUC, 0);
9139 I915_WRITE(MI_ARB_STATE,
9140 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9141
9142
9143 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9144}
9145
9146static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
9147{
9148 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
9149 I965_RCC_CLOCK_GATE_DISABLE |
9150 I965_RCPB_CLOCK_GATE_DISABLE |
9151 I965_ISC_CLOCK_GATE_DISABLE |
9152 I965_FBC_CLOCK_GATE_DISABLE);
9153 I915_WRITE(RENCLK_GATE_D2, 0);
9154 I915_WRITE(MI_ARB_STATE,
9155 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9156
9157
9158 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9159}
9160
9161static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
9162{
9163 u32 dstate = I915_READ(D_STATE);
9164
9165 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
9166 DSTATE_DOT_CLOCK_GATING;
9167 I915_WRITE(D_STATE, dstate);
9168
9169 if (IS_PINEVIEW(dev_priv))
9170 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
9171
9172
9173 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
9174
9175
9176 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
9177
9178
9179 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
9180
9181 I915_WRITE(MI_ARB_STATE,
9182 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9183}
9184
9185static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
9186{
9187 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
9188
9189
9190 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
9191 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
9192
9193 I915_WRITE(MEM_MODE,
9194 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
9195}
9196
9197static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
9198{
9199 I915_WRITE(MEM_MODE,
9200 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
9201 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
9202}
9203
9204void intel_init_clock_gating(struct drm_i915_private *dev_priv)
9205{
9206 dev_priv->display.init_clock_gating(dev_priv);
9207}
9208
9209void intel_suspend_hw(struct drm_i915_private *dev_priv)
9210{
9211 if (HAS_PCH_LPT(dev_priv))
9212 lpt_suspend_hw(dev_priv);
9213}
9214
9215static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
9216{
9217 DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
9218}
9219
9220
9221
9222
9223
9224
9225
9226
9227
9228
9229void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
9230{
9231 if (IS_ICELAKE(dev_priv))
9232 dev_priv->display.init_clock_gating = icl_init_clock_gating;
9233 else if (IS_CANNONLAKE(dev_priv))
9234 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
9235 else if (IS_COFFEELAKE(dev_priv))
9236 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
9237 else if (IS_SKYLAKE(dev_priv))
9238 dev_priv->display.init_clock_gating = skl_init_clock_gating;
9239 else if (IS_KABYLAKE(dev_priv))
9240 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
9241 else if (IS_BROXTON(dev_priv))
9242 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
9243 else if (IS_GEMINILAKE(dev_priv))
9244 dev_priv->display.init_clock_gating = glk_init_clock_gating;
9245 else if (IS_BROADWELL(dev_priv))
9246 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
9247 else if (IS_CHERRYVIEW(dev_priv))
9248 dev_priv->display.init_clock_gating = chv_init_clock_gating;
9249 else if (IS_HASWELL(dev_priv))
9250 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
9251 else if (IS_IVYBRIDGE(dev_priv))
9252 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
9253 else if (IS_VALLEYVIEW(dev_priv))
9254 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
9255 else if (IS_GEN6(dev_priv))
9256 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9257 else if (IS_GEN5(dev_priv))
9258 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
9259 else if (IS_G4X(dev_priv))
9260 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9261 else if (IS_I965GM(dev_priv))
9262 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
9263 else if (IS_I965G(dev_priv))
9264 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
9265 else if (IS_GEN3(dev_priv))
9266 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9267 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
9268 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9269 else if (IS_GEN2(dev_priv))
9270 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9271 else {
9272 MISSING_CASE(INTEL_DEVID(dev_priv));
9273 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9274 }
9275}
9276
9277
9278void intel_init_pm(struct drm_i915_private *dev_priv)
9279{
9280 intel_fbc_init(dev_priv);
9281
9282
9283 if (IS_PINEVIEW(dev_priv))
9284 i915_pineview_get_mem_freq(dev_priv);
9285 else if (IS_GEN5(dev_priv))
9286 i915_ironlake_get_mem_freq(dev_priv);
9287
9288
9289 if (INTEL_GEN(dev_priv) >= 9) {
9290 skl_setup_wm_latency(dev_priv);
9291 dev_priv->display.initial_watermarks = skl_initial_wm;
9292 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
9293 dev_priv->display.compute_global_watermarks = skl_compute_wm;
9294 } else if (HAS_PCH_SPLIT(dev_priv)) {
9295 ilk_setup_wm_latency(dev_priv);
9296
9297 if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
9298 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9299 (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
9300 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9301 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9302 dev_priv->display.compute_intermediate_wm =
9303 ilk_compute_intermediate_wm;
9304 dev_priv->display.initial_watermarks =
9305 ilk_initial_watermarks;
9306 dev_priv->display.optimize_watermarks =
9307 ilk_optimize_watermarks;
9308 } else {
9309 DRM_DEBUG_KMS("Failed to read display plane latency. "
9310 "Disable CxSR\n");
9311 }
9312 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9313 vlv_setup_wm_latency(dev_priv);
9314 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9315 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9316 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9317 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9318 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9319 } else if (IS_G4X(dev_priv)) {
9320 g4x_setup_wm_latency(dev_priv);
9321 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9322 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9323 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9324 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9325 } else if (IS_PINEVIEW(dev_priv)) {
9326 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
9327 dev_priv->is_ddr3,
9328 dev_priv->fsb_freq,
9329 dev_priv->mem_freq)) {
9330 DRM_INFO("failed to find known CxSR latency "
9331 "(found ddr%s fsb freq %d, mem freq %d), "
9332 "disabling CxSR\n",
9333 (dev_priv->is_ddr3 == 1) ? "3" : "2",
9334 dev_priv->fsb_freq, dev_priv->mem_freq);
9335
9336 intel_set_memory_cxsr(dev_priv, false);
9337 dev_priv->display.update_wm = NULL;
9338 } else
9339 dev_priv->display.update_wm = pineview_update_wm;
9340 } else if (IS_GEN4(dev_priv)) {
9341 dev_priv->display.update_wm = i965_update_wm;
9342 } else if (IS_GEN3(dev_priv)) {
9343 dev_priv->display.update_wm = i9xx_update_wm;
9344 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9345 } else if (IS_GEN2(dev_priv)) {
9346 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9347 dev_priv->display.update_wm = i845_update_wm;
9348 dev_priv->display.get_fifo_size = i845_get_fifo_size;
9349 } else {
9350 dev_priv->display.update_wm = i9xx_update_wm;
9351 dev_priv->display.get_fifo_size = i830_get_fifo_size;
9352 }
9353 } else {
9354 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9355 }
9356}
9357
9358static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
9359{
9360 uint32_t flags =
9361 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9362
9363 switch (flags) {
9364 case GEN6_PCODE_SUCCESS:
9365 return 0;
9366 case GEN6_PCODE_UNIMPLEMENTED_CMD:
9367 return -ENODEV;
9368 case GEN6_PCODE_ILLEGAL_CMD:
9369 return -ENXIO;
9370 case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9371 case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9372 return -EOVERFLOW;
9373 case GEN6_PCODE_TIMEOUT:
9374 return -ETIMEDOUT;
9375 default:
9376 MISSING_CASE(flags);
9377 return 0;
9378 }
9379}
9380
9381static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
9382{
9383 uint32_t flags =
9384 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9385
9386 switch (flags) {
9387 case GEN6_PCODE_SUCCESS:
9388 return 0;
9389 case GEN6_PCODE_ILLEGAL_CMD:
9390 return -ENXIO;
9391 case GEN7_PCODE_TIMEOUT:
9392 return -ETIMEDOUT;
9393 case GEN7_PCODE_ILLEGAL_DATA:
9394 return -EINVAL;
9395 case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9396 return -EOVERFLOW;
9397 default:
9398 MISSING_CASE(flags);
9399 return 0;
9400 }
9401}
9402
9403int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
9404{
9405 int status;
9406
9407 WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9408
9409
9410
9411
9412
9413
9414 if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9415 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps\n",
9416 mbox, __builtin_return_address(0));
9417 return -EAGAIN;
9418 }
9419
9420 I915_WRITE_FW(GEN6_PCODE_DATA, *val);
9421 I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9422 I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9423
9424 if (__intel_wait_for_register_fw(dev_priv,
9425 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9426 500, 0, NULL)) {
9427 DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
9428 mbox, __builtin_return_address(0));
9429 return -ETIMEDOUT;
9430 }
9431
9432 *val = I915_READ_FW(GEN6_PCODE_DATA);
9433 I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9434
9435 if (INTEL_GEN(dev_priv) > 6)
9436 status = gen7_check_mailbox_status(dev_priv);
9437 else
9438 status = gen6_check_mailbox_status(dev_priv);
9439
9440 if (status) {
9441 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n",
9442 mbox, __builtin_return_address(0), status);
9443 return status;
9444 }
9445
9446 return 0;
9447}
9448
9449int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
9450 u32 mbox, u32 val,
9451 int fast_timeout_us, int slow_timeout_ms)
9452{
9453 int status;
9454
9455 WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9456
9457
9458
9459
9460
9461
9462 if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9463 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps\n",
9464 val, mbox, __builtin_return_address(0));
9465 return -EAGAIN;
9466 }
9467
9468 I915_WRITE_FW(GEN6_PCODE_DATA, val);
9469 I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9470 I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9471
9472 if (__intel_wait_for_register_fw(dev_priv,
9473 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9474 fast_timeout_us, slow_timeout_ms,
9475 NULL)) {
9476 DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
9477 val, mbox, __builtin_return_address(0));
9478 return -ETIMEDOUT;
9479 }
9480
9481 I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9482
9483 if (INTEL_GEN(dev_priv) > 6)
9484 status = gen7_check_mailbox_status(dev_priv);
9485 else
9486 status = gen6_check_mailbox_status(dev_priv);
9487
9488 if (status) {
9489 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n",
9490 val, mbox, __builtin_return_address(0), status);
9491 return status;
9492 }
9493
9494 return 0;
9495}
9496
9497static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox,
9498 u32 request, u32 reply_mask, u32 reply,
9499 u32 *status)
9500{
9501 u32 val = request;
9502
9503 *status = sandybridge_pcode_read(dev_priv, mbox, &val);
9504
9505 return *status || ((val & reply_mask) == reply);
9506}
9507
9508
9509
9510
9511
9512
9513
9514
9515
9516
9517
9518
9519
9520
9521
9522
9523
9524
9525
9526
9527int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
9528 u32 reply_mask, u32 reply, int timeout_base_ms)
9529{
9530 u32 status;
9531 int ret;
9532
9533 WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9534
9535#define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \
9536 &status)
9537
9538
9539
9540
9541
9542
9543
9544 if (COND) {
9545 ret = 0;
9546 goto out;
9547 }
9548 ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10);
9549 if (!ret)
9550 goto out;
9551
9552
9553
9554
9555
9556
9557
9558
9559
9560
9561
9562 DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n");
9563 WARN_ON_ONCE(timeout_base_ms > 3);
9564 preempt_disable();
9565 ret = wait_for_atomic(COND, 50);
9566 preempt_enable();
9567
9568out:
9569 return ret ? ret : status;
9570#undef COND
9571}
9572
9573static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9574{
9575 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9576
9577
9578
9579
9580
9581 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9582}
9583
9584static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9585{
9586 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9587
9588 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9589}
9590
9591static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9592{
9593 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9594
9595
9596
9597
9598
9599 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9600}
9601
9602static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9603{
9604 struct intel_rps *rps = &dev_priv->gt_pm.rps;
9605
9606
9607 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9608}
9609
9610int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9611{
9612 if (INTEL_GEN(dev_priv) >= 9)
9613 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9614 GEN9_FREQ_SCALER);
9615 else if (IS_CHERRYVIEW(dev_priv))
9616 return chv_gpu_freq(dev_priv, val);
9617 else if (IS_VALLEYVIEW(dev_priv))
9618 return byt_gpu_freq(dev_priv, val);
9619 else
9620 return val * GT_FREQUENCY_MULTIPLIER;
9621}
9622
9623int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9624{
9625 if (INTEL_GEN(dev_priv) >= 9)
9626 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9627 GT_FREQUENCY_MULTIPLIER);
9628 else if (IS_CHERRYVIEW(dev_priv))
9629 return chv_freq_opcode(dev_priv, val);
9630 else if (IS_VALLEYVIEW(dev_priv))
9631 return byt_freq_opcode(dev_priv, val);
9632 else
9633 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9634}
9635
9636void intel_pm_setup(struct drm_i915_private *dev_priv)
9637{
9638 mutex_init(&dev_priv->pcu_lock);
9639 mutex_init(&dev_priv->gt_pm.rps.power.mutex);
9640
9641 atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9642
9643 dev_priv->runtime_pm.suspended = false;
9644 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9645}
9646
9647static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9648 const i915_reg_t reg)
9649{
9650 u32 lower, upper, tmp;
9651 int loop = 2;
9652
9653
9654
9655
9656
9657 lockdep_assert_held(&dev_priv->uncore.lock);
9658
9659
9660
9661
9662
9663
9664
9665
9666
9667
9668
9669 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9670 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9671 upper = I915_READ_FW(reg);
9672 do {
9673 tmp = upper;
9674
9675 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9676 _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9677 lower = I915_READ_FW(reg);
9678
9679 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9680 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9681 upper = I915_READ_FW(reg);
9682 } while (upper != tmp && --loop);
9683
9684
9685
9686
9687
9688
9689
9690 return lower | (u64)upper << 8;
9691}
9692
9693u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9694 const i915_reg_t reg)
9695{
9696 u64 time_hw, prev_hw, overflow_hw;
9697 unsigned int fw_domains;
9698 unsigned long flags;
9699 unsigned int i;
9700 u32 mul, div;
9701
9702 if (!HAS_RC6(dev_priv))
9703 return 0;
9704
9705
9706
9707
9708
9709
9710
9711
9712 i = (i915_mmio_reg_offset(reg) -
9713 i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
9714 if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
9715 return 0;
9716
9717 fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
9718
9719 spin_lock_irqsave(&dev_priv->uncore.lock, flags);
9720 intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
9721
9722
9723 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9724 mul = 1000000;
9725 div = dev_priv->czclk_freq;
9726 overflow_hw = BIT_ULL(40);
9727 time_hw = vlv_residency_raw(dev_priv, reg);
9728 } else {
9729
9730 if (IS_GEN9_LP(dev_priv)) {
9731 mul = 10000;
9732 div = 12;
9733 } else {
9734 mul = 1280;
9735 div = 1;
9736 }
9737
9738 overflow_hw = BIT_ULL(32);
9739 time_hw = I915_READ_FW(reg);
9740 }
9741
9742
9743
9744
9745
9746
9747
9748 prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
9749 dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
9750
9751
9752 if (time_hw >= prev_hw)
9753 time_hw -= prev_hw;
9754 else
9755 time_hw += overflow_hw - prev_hw;
9756
9757
9758 time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
9759 dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
9760
9761 intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
9762 spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
9763
9764 return mul_u64_u32_div(time_hw, mul, div);
9765}
9766
9767u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
9768{
9769 u32 cagf;
9770
9771 if (INTEL_GEN(dev_priv) >= 9)
9772 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
9773 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
9774 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
9775 else
9776 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
9777
9778 return cagf;
9779}
9780