1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/cpufreq.h>
29#include <drm/drm_plane_helper.h>
30#include "i915_drv.h"
31#include "intel_drv.h"
32#include "../../../platform/x86/intel_ips.h"
33#include <linux/module.h>
34#include <drm/drm_atomic_helper.h>
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55#define INTEL_RC6_ENABLE (1<<0)
56#define INTEL_RC6p_ENABLE (1<<1)
57#define INTEL_RC6pp_ENABLE (1<<2)
58
59static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
60{
61
62 I915_WRITE(CHICKEN_PAR1_1,
63 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
64
65 I915_WRITE(GEN8_CONFIG0,
66 I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES);
67
68
69 I915_WRITE(GEN8_CHICKEN_DCPR_1,
70 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
71
72
73
74 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
75 DISP_FBC_WM_DIS |
76 DISP_FBC_MEMORY_WAKE);
77
78
79 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
80 ILK_DPFC_DISABLE_DUMMY0);
81}
82
83static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
84{
85 gen9_init_clock_gating(dev_priv);
86
87
88 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
89 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
90
91
92
93
94
95 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
96 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
97
98
99
100
101
102 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
103 PWM1_GATING_DIS | PWM2_GATING_DIS);
104}
105
106static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
107{
108 gen9_init_clock_gating(dev_priv);
109
110
111
112
113
114
115 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
116 PWM1_GATING_DIS | PWM2_GATING_DIS);
117
118
119 if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
120 u32 val = I915_READ(CHICKEN_MISC_2);
121 val &= ~(GLK_CL0_PWR_DOWN |
122 GLK_CL1_PWR_DOWN |
123 GLK_CL2_PWR_DOWN);
124 I915_WRITE(CHICKEN_MISC_2, val);
125 }
126
127}
128
129static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
130{
131 u32 tmp;
132
133 tmp = I915_READ(CLKCFG);
134
135 switch (tmp & CLKCFG_FSB_MASK) {
136 case CLKCFG_FSB_533:
137 dev_priv->fsb_freq = 533;
138 break;
139 case CLKCFG_FSB_800:
140 dev_priv->fsb_freq = 800;
141 break;
142 case CLKCFG_FSB_667:
143 dev_priv->fsb_freq = 667;
144 break;
145 case CLKCFG_FSB_400:
146 dev_priv->fsb_freq = 400;
147 break;
148 }
149
150 switch (tmp & CLKCFG_MEM_MASK) {
151 case CLKCFG_MEM_533:
152 dev_priv->mem_freq = 533;
153 break;
154 case CLKCFG_MEM_667:
155 dev_priv->mem_freq = 667;
156 break;
157 case CLKCFG_MEM_800:
158 dev_priv->mem_freq = 800;
159 break;
160 }
161
162
163 tmp = I915_READ(CSHRDDR3CTL);
164 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
165}
166
167static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
168{
169 u16 ddrpll, csipll;
170
171 ddrpll = I915_READ16(DDRMPLL1);
172 csipll = I915_READ16(CSIPLL0);
173
174 switch (ddrpll & 0xff) {
175 case 0xc:
176 dev_priv->mem_freq = 800;
177 break;
178 case 0x10:
179 dev_priv->mem_freq = 1066;
180 break;
181 case 0x14:
182 dev_priv->mem_freq = 1333;
183 break;
184 case 0x18:
185 dev_priv->mem_freq = 1600;
186 break;
187 default:
188 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
189 ddrpll & 0xff);
190 dev_priv->mem_freq = 0;
191 break;
192 }
193
194 dev_priv->ips.r_t = dev_priv->mem_freq;
195
196 switch (csipll & 0x3ff) {
197 case 0x00c:
198 dev_priv->fsb_freq = 3200;
199 break;
200 case 0x00e:
201 dev_priv->fsb_freq = 3733;
202 break;
203 case 0x010:
204 dev_priv->fsb_freq = 4266;
205 break;
206 case 0x012:
207 dev_priv->fsb_freq = 4800;
208 break;
209 case 0x014:
210 dev_priv->fsb_freq = 5333;
211 break;
212 case 0x016:
213 dev_priv->fsb_freq = 5866;
214 break;
215 case 0x018:
216 dev_priv->fsb_freq = 6400;
217 break;
218 default:
219 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
220 csipll & 0x3ff);
221 dev_priv->fsb_freq = 0;
222 break;
223 }
224
225 if (dev_priv->fsb_freq == 3200) {
226 dev_priv->ips.c_m = 0;
227 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
228 dev_priv->ips.c_m = 1;
229 } else {
230 dev_priv->ips.c_m = 2;
231 }
232}
233
234static const struct cxsr_latency cxsr_latency_table[] = {
235 {1, 0, 800, 400, 3382, 33382, 3983, 33983},
236 {1, 0, 800, 667, 3354, 33354, 3807, 33807},
237 {1, 0, 800, 800, 3347, 33347, 3763, 33763},
238 {1, 1, 800, 667, 6420, 36420, 6873, 36873},
239 {1, 1, 800, 800, 5902, 35902, 6318, 36318},
240
241 {1, 0, 667, 400, 3400, 33400, 4021, 34021},
242 {1, 0, 667, 667, 3372, 33372, 3845, 33845},
243 {1, 0, 667, 800, 3386, 33386, 3822, 33822},
244 {1, 1, 667, 667, 6438, 36438, 6911, 36911},
245 {1, 1, 667, 800, 5941, 35941, 6377, 36377},
246
247 {1, 0, 400, 400, 3472, 33472, 4173, 34173},
248 {1, 0, 400, 667, 3443, 33443, 3996, 33996},
249 {1, 0, 400, 800, 3430, 33430, 3946, 33946},
250 {1, 1, 400, 667, 6509, 36509, 7062, 37062},
251 {1, 1, 400, 800, 5985, 35985, 6501, 36501},
252
253 {0, 0, 800, 400, 3438, 33438, 4065, 34065},
254 {0, 0, 800, 667, 3410, 33410, 3889, 33889},
255 {0, 0, 800, 800, 3403, 33403, 3845, 33845},
256 {0, 1, 800, 667, 6476, 36476, 6955, 36955},
257 {0, 1, 800, 800, 5958, 35958, 6400, 36400},
258
259 {0, 0, 667, 400, 3456, 33456, 4103, 34106},
260 {0, 0, 667, 667, 3428, 33428, 3927, 33927},
261 {0, 0, 667, 800, 3443, 33443, 3905, 33905},
262 {0, 1, 667, 667, 6494, 36494, 6993, 36993},
263 {0, 1, 667, 800, 5998, 35998, 6460, 36460},
264
265 {0, 0, 400, 400, 3528, 33528, 4255, 34255},
266 {0, 0, 400, 667, 3500, 33500, 4079, 34079},
267 {0, 0, 400, 800, 3487, 33487, 4029, 34029},
268 {0, 1, 400, 667, 6566, 36566, 7145, 37145},
269 {0, 1, 400, 800, 6042, 36042, 6584, 36584},
270};
271
272static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
273 bool is_ddr3,
274 int fsb,
275 int mem)
276{
277 const struct cxsr_latency *latency;
278 int i;
279
280 if (fsb == 0 || mem == 0)
281 return NULL;
282
283 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
284 latency = &cxsr_latency_table[i];
285 if (is_desktop == latency->is_desktop &&
286 is_ddr3 == latency->is_ddr3 &&
287 fsb == latency->fsb_freq && mem == latency->mem_freq)
288 return latency;
289 }
290
291 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
292
293 return NULL;
294}
295
296static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
297{
298 u32 val;
299
300 mutex_lock(&dev_priv->rps.hw_lock);
301
302 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
303 if (enable)
304 val &= ~FORCE_DDR_HIGH_FREQ;
305 else
306 val |= FORCE_DDR_HIGH_FREQ;
307 val &= ~FORCE_DDR_LOW_FREQ;
308 val |= FORCE_DDR_FREQ_REQ_ACK;
309 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
310
311 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
312 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
313 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
314
315 mutex_unlock(&dev_priv->rps.hw_lock);
316}
317
318static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
319{
320 u32 val;
321
322 mutex_lock(&dev_priv->rps.hw_lock);
323
324 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
325 if (enable)
326 val |= DSP_MAXFIFO_PM5_ENABLE;
327 else
328 val &= ~DSP_MAXFIFO_PM5_ENABLE;
329 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
330
331 mutex_unlock(&dev_priv->rps.hw_lock);
332}
333
334#define FW_WM(value, plane) \
335 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
336
337static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
338{
339 bool was_enabled;
340 u32 val;
341
342 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
343 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
344 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
345 POSTING_READ(FW_BLC_SELF_VLV);
346 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
347 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
348 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
349 POSTING_READ(FW_BLC_SELF);
350 } else if (IS_PINEVIEW(dev_priv)) {
351 val = I915_READ(DSPFW3);
352 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
353 if (enable)
354 val |= PINEVIEW_SELF_REFRESH_EN;
355 else
356 val &= ~PINEVIEW_SELF_REFRESH_EN;
357 I915_WRITE(DSPFW3, val);
358 POSTING_READ(DSPFW3);
359 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
360 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
361 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
362 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
363 I915_WRITE(FW_BLC_SELF, val);
364 POSTING_READ(FW_BLC_SELF);
365 } else if (IS_I915GM(dev_priv)) {
366
367
368
369
370
371 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
372 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
373 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
374 I915_WRITE(INSTPM, val);
375 POSTING_READ(INSTPM);
376 } else {
377 return false;
378 }
379
380 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
381
382 DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
383 enableddisabled(enable),
384 enableddisabled(was_enabled));
385
386 return was_enabled;
387}
388
389bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
390{
391 bool ret;
392
393 mutex_lock(&dev_priv->wm.wm_mutex);
394 ret = _intel_set_memory_cxsr(dev_priv, enable);
395 dev_priv->wm.vlv.cxsr = enable;
396 mutex_unlock(&dev_priv->wm.wm_mutex);
397
398 return ret;
399}
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415static const int pessimal_latency_ns = 5000;
416
417#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
418 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
419
420static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
421{
422 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
423 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
424 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
425 enum pipe pipe = crtc->pipe;
426 int sprite0_start, sprite1_start;
427
428 switch (pipe) {
429 uint32_t dsparb, dsparb2, dsparb3;
430 case PIPE_A:
431 dsparb = I915_READ(DSPARB);
432 dsparb2 = I915_READ(DSPARB2);
433 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
434 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
435 break;
436 case PIPE_B:
437 dsparb = I915_READ(DSPARB);
438 dsparb2 = I915_READ(DSPARB2);
439 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
440 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
441 break;
442 case PIPE_C:
443 dsparb2 = I915_READ(DSPARB2);
444 dsparb3 = I915_READ(DSPARB3);
445 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
446 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
447 break;
448 default:
449 MISSING_CASE(pipe);
450 return;
451 }
452
453 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
454 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
455 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
456 fifo_state->plane[PLANE_CURSOR] = 63;
457
458 DRM_DEBUG_KMS("Pipe %c FIFO size: %d/%d/%d/%d\n",
459 pipe_name(pipe),
460 fifo_state->plane[PLANE_PRIMARY],
461 fifo_state->plane[PLANE_SPRITE0],
462 fifo_state->plane[PLANE_SPRITE1],
463 fifo_state->plane[PLANE_CURSOR]);
464}
465
466static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
467{
468 uint32_t dsparb = I915_READ(DSPARB);
469 int size;
470
471 size = dsparb & 0x7f;
472 if (plane)
473 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
474
475 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
476 plane ? "B" : "A", size);
477
478 return size;
479}
480
481static int i830_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
482{
483 uint32_t dsparb = I915_READ(DSPARB);
484 int size;
485
486 size = dsparb & 0x1ff;
487 if (plane)
488 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
489 size >>= 1;
490
491 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
492 plane ? "B" : "A", size);
493
494 return size;
495}
496
497static int i845_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
498{
499 uint32_t dsparb = I915_READ(DSPARB);
500 int size;
501
502 size = dsparb & 0x7f;
503 size >>= 2;
504
505 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
506 plane ? "B" : "A",
507 size);
508
509 return size;
510}
511
512
513static const struct intel_watermark_params pineview_display_wm = {
514 .fifo_size = PINEVIEW_DISPLAY_FIFO,
515 .max_wm = PINEVIEW_MAX_WM,
516 .default_wm = PINEVIEW_DFT_WM,
517 .guard_size = PINEVIEW_GUARD_WM,
518 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
519};
520static const struct intel_watermark_params pineview_display_hplloff_wm = {
521 .fifo_size = PINEVIEW_DISPLAY_FIFO,
522 .max_wm = PINEVIEW_MAX_WM,
523 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
524 .guard_size = PINEVIEW_GUARD_WM,
525 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
526};
527static const struct intel_watermark_params pineview_cursor_wm = {
528 .fifo_size = PINEVIEW_CURSOR_FIFO,
529 .max_wm = PINEVIEW_CURSOR_MAX_WM,
530 .default_wm = PINEVIEW_CURSOR_DFT_WM,
531 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
532 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
533};
534static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
535 .fifo_size = PINEVIEW_CURSOR_FIFO,
536 .max_wm = PINEVIEW_CURSOR_MAX_WM,
537 .default_wm = PINEVIEW_CURSOR_DFT_WM,
538 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
539 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
540};
541static const struct intel_watermark_params g4x_wm_info = {
542 .fifo_size = G4X_FIFO_SIZE,
543 .max_wm = G4X_MAX_WM,
544 .default_wm = G4X_MAX_WM,
545 .guard_size = 2,
546 .cacheline_size = G4X_FIFO_LINE_SIZE,
547};
548static const struct intel_watermark_params g4x_cursor_wm_info = {
549 .fifo_size = I965_CURSOR_FIFO,
550 .max_wm = I965_CURSOR_MAX_WM,
551 .default_wm = I965_CURSOR_DFT_WM,
552 .guard_size = 2,
553 .cacheline_size = G4X_FIFO_LINE_SIZE,
554};
555static const struct intel_watermark_params i965_cursor_wm_info = {
556 .fifo_size = I965_CURSOR_FIFO,
557 .max_wm = I965_CURSOR_MAX_WM,
558 .default_wm = I965_CURSOR_DFT_WM,
559 .guard_size = 2,
560 .cacheline_size = I915_FIFO_LINE_SIZE,
561};
562static const struct intel_watermark_params i945_wm_info = {
563 .fifo_size = I945_FIFO_SIZE,
564 .max_wm = I915_MAX_WM,
565 .default_wm = 1,
566 .guard_size = 2,
567 .cacheline_size = I915_FIFO_LINE_SIZE,
568};
569static const struct intel_watermark_params i915_wm_info = {
570 .fifo_size = I915_FIFO_SIZE,
571 .max_wm = I915_MAX_WM,
572 .default_wm = 1,
573 .guard_size = 2,
574 .cacheline_size = I915_FIFO_LINE_SIZE,
575};
576static const struct intel_watermark_params i830_a_wm_info = {
577 .fifo_size = I855GM_FIFO_SIZE,
578 .max_wm = I915_MAX_WM,
579 .default_wm = 1,
580 .guard_size = 2,
581 .cacheline_size = I830_FIFO_LINE_SIZE,
582};
583static const struct intel_watermark_params i830_bc_wm_info = {
584 .fifo_size = I855GM_FIFO_SIZE,
585 .max_wm = I915_MAX_WM/2,
586 .default_wm = 1,
587 .guard_size = 2,
588 .cacheline_size = I830_FIFO_LINE_SIZE,
589};
590static const struct intel_watermark_params i845_wm_info = {
591 .fifo_size = I830_FIFO_SIZE,
592 .max_wm = I915_MAX_WM,
593 .default_wm = 1,
594 .guard_size = 2,
595 .cacheline_size = I830_FIFO_LINE_SIZE,
596};
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
617 const struct intel_watermark_params *wm,
618 int fifo_size, int cpp,
619 unsigned long latency_ns)
620{
621 long entries_required, wm_size;
622
623
624
625
626
627
628
629 entries_required = ((clock_in_khz / 1000) * cpp * latency_ns) /
630 1000;
631 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
632
633 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
634
635 wm_size = fifo_size - (entries_required + wm->guard_size);
636
637 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
638
639
640 if (wm_size > (long)wm->max_wm)
641 wm_size = wm->max_wm;
642 if (wm_size <= 0)
643 wm_size = wm->default_wm;
644
645
646
647
648
649
650
651
652 if (wm_size <= 8)
653 wm_size = 8;
654
655 return wm_size;
656}
657
658static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
659 const struct intel_plane_state *plane_state)
660{
661 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
662
663
664 if (!crtc_state->base.active)
665 return false;
666
667
668
669
670
671
672
673
674
675 if (plane->id == PLANE_CURSOR)
676 return plane_state->base.fb != NULL;
677 else
678 return plane_state->base.visible;
679}
680
681static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
682{
683 struct intel_crtc *crtc, *enabled = NULL;
684
685 for_each_intel_crtc(&dev_priv->drm, crtc) {
686 if (intel_crtc_active(crtc)) {
687 if (enabled)
688 return NULL;
689 enabled = crtc;
690 }
691 }
692
693 return enabled;
694}
695
696static void pineview_update_wm(struct intel_crtc *unused_crtc)
697{
698 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
699 struct intel_crtc *crtc;
700 const struct cxsr_latency *latency;
701 u32 reg;
702 unsigned long wm;
703
704 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
705 dev_priv->is_ddr3,
706 dev_priv->fsb_freq,
707 dev_priv->mem_freq);
708 if (!latency) {
709 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
710 intel_set_memory_cxsr(dev_priv, false);
711 return;
712 }
713
714 crtc = single_enabled_crtc(dev_priv);
715 if (crtc) {
716 const struct drm_display_mode *adjusted_mode =
717 &crtc->config->base.adjusted_mode;
718 const struct drm_framebuffer *fb =
719 crtc->base.primary->state->fb;
720 int cpp = fb->format->cpp[0];
721 int clock = adjusted_mode->crtc_clock;
722
723
724 wm = intel_calculate_wm(clock, &pineview_display_wm,
725 pineview_display_wm.fifo_size,
726 cpp, latency->display_sr);
727 reg = I915_READ(DSPFW1);
728 reg &= ~DSPFW_SR_MASK;
729 reg |= FW_WM(wm, SR);
730 I915_WRITE(DSPFW1, reg);
731 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
732
733
734 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
735 pineview_display_wm.fifo_size,
736 cpp, latency->cursor_sr);
737 reg = I915_READ(DSPFW3);
738 reg &= ~DSPFW_CURSOR_SR_MASK;
739 reg |= FW_WM(wm, CURSOR_SR);
740 I915_WRITE(DSPFW3, reg);
741
742
743 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
744 pineview_display_hplloff_wm.fifo_size,
745 cpp, latency->display_hpll_disable);
746 reg = I915_READ(DSPFW3);
747 reg &= ~DSPFW_HPLL_SR_MASK;
748 reg |= FW_WM(wm, HPLL_SR);
749 I915_WRITE(DSPFW3, reg);
750
751
752 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
753 pineview_display_hplloff_wm.fifo_size,
754 cpp, latency->cursor_hpll_disable);
755 reg = I915_READ(DSPFW3);
756 reg &= ~DSPFW_HPLL_CURSOR_MASK;
757 reg |= FW_WM(wm, HPLL_CURSOR);
758 I915_WRITE(DSPFW3, reg);
759 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
760
761 intel_set_memory_cxsr(dev_priv, true);
762 } else {
763 intel_set_memory_cxsr(dev_priv, false);
764 }
765}
766
767static bool g4x_compute_wm0(struct drm_i915_private *dev_priv,
768 int plane,
769 const struct intel_watermark_params *display,
770 int display_latency_ns,
771 const struct intel_watermark_params *cursor,
772 int cursor_latency_ns,
773 int *plane_wm,
774 int *cursor_wm)
775{
776 struct intel_crtc *crtc;
777 const struct drm_display_mode *adjusted_mode;
778 const struct drm_framebuffer *fb;
779 int htotal, hdisplay, clock, cpp;
780 int line_time_us, line_count;
781 int entries, tlb_miss;
782
783 crtc = intel_get_crtc_for_plane(dev_priv, plane);
784 if (!intel_crtc_active(crtc)) {
785 *cursor_wm = cursor->guard_size;
786 *plane_wm = display->guard_size;
787 return false;
788 }
789
790 adjusted_mode = &crtc->config->base.adjusted_mode;
791 fb = crtc->base.primary->state->fb;
792 clock = adjusted_mode->crtc_clock;
793 htotal = adjusted_mode->crtc_htotal;
794 hdisplay = crtc->config->pipe_src_w;
795 cpp = fb->format->cpp[0];
796
797
798 entries = ((clock * cpp / 1000) * display_latency_ns) / 1000;
799 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
800 if (tlb_miss > 0)
801 entries += tlb_miss;
802 entries = DIV_ROUND_UP(entries, display->cacheline_size);
803 *plane_wm = entries + display->guard_size;
804 if (*plane_wm > (int)display->max_wm)
805 *plane_wm = display->max_wm;
806
807
808 line_time_us = max(htotal * 1000 / clock, 1);
809 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
810 entries = line_count * crtc->base.cursor->state->crtc_w * cpp;
811 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
812 if (tlb_miss > 0)
813 entries += tlb_miss;
814 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
815 *cursor_wm = entries + cursor->guard_size;
816 if (*cursor_wm > (int)cursor->max_wm)
817 *cursor_wm = (int)cursor->max_wm;
818
819 return true;
820}
821
822
823
824
825
826
827
828
829static bool g4x_check_srwm(struct drm_i915_private *dev_priv,
830 int display_wm, int cursor_wm,
831 const struct intel_watermark_params *display,
832 const struct intel_watermark_params *cursor)
833{
834 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
835 display_wm, cursor_wm);
836
837 if (display_wm > display->max_wm) {
838 DRM_DEBUG_KMS("display watermark is too large(%d/%u), disabling\n",
839 display_wm, display->max_wm);
840 return false;
841 }
842
843 if (cursor_wm > cursor->max_wm) {
844 DRM_DEBUG_KMS("cursor watermark is too large(%d/%u), disabling\n",
845 cursor_wm, cursor->max_wm);
846 return false;
847 }
848
849 if (!(display_wm || cursor_wm)) {
850 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
851 return false;
852 }
853
854 return true;
855}
856
857static bool g4x_compute_srwm(struct drm_i915_private *dev_priv,
858 int plane,
859 int latency_ns,
860 const struct intel_watermark_params *display,
861 const struct intel_watermark_params *cursor,
862 int *display_wm, int *cursor_wm)
863{
864 struct intel_crtc *crtc;
865 const struct drm_display_mode *adjusted_mode;
866 const struct drm_framebuffer *fb;
867 int hdisplay, htotal, cpp, clock;
868 unsigned long line_time_us;
869 int line_count, line_size;
870 int small, large;
871 int entries;
872
873 if (!latency_ns) {
874 *display_wm = *cursor_wm = 0;
875 return false;
876 }
877
878 crtc = intel_get_crtc_for_plane(dev_priv, plane);
879 adjusted_mode = &crtc->config->base.adjusted_mode;
880 fb = crtc->base.primary->state->fb;
881 clock = adjusted_mode->crtc_clock;
882 htotal = adjusted_mode->crtc_htotal;
883 hdisplay = crtc->config->pipe_src_w;
884 cpp = fb->format->cpp[0];
885
886 line_time_us = max(htotal * 1000 / clock, 1);
887 line_count = (latency_ns / line_time_us + 1000) / 1000;
888 line_size = hdisplay * cpp;
889
890
891 small = ((clock * cpp / 1000) * latency_ns) / 1000;
892 large = line_count * line_size;
893
894 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
895 *display_wm = entries + display->guard_size;
896
897
898 entries = line_count * cpp * crtc->base.cursor->state->crtc_w;
899 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
900 *cursor_wm = entries + cursor->guard_size;
901
902 return g4x_check_srwm(dev_priv,
903 *display_wm, *cursor_wm,
904 display, cursor);
905}
906
907#define FW_WM_VLV(value, plane) \
908 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
909
910static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
911 const struct vlv_wm_values *wm)
912{
913 enum pipe pipe;
914
915 for_each_pipe(dev_priv, pipe) {
916 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
917
918 I915_WRITE(VLV_DDL(pipe),
919 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
920 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
921 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
922 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
923 }
924
925
926
927
928
929
930 I915_WRITE(DSPHOWM, 0);
931 I915_WRITE(DSPHOWM1, 0);
932 I915_WRITE(DSPFW4, 0);
933 I915_WRITE(DSPFW5, 0);
934 I915_WRITE(DSPFW6, 0);
935
936 I915_WRITE(DSPFW1,
937 FW_WM(wm->sr.plane, SR) |
938 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
939 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
940 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
941 I915_WRITE(DSPFW2,
942 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
943 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
944 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
945 I915_WRITE(DSPFW3,
946 FW_WM(wm->sr.cursor, CURSOR_SR));
947
948 if (IS_CHERRYVIEW(dev_priv)) {
949 I915_WRITE(DSPFW7_CHV,
950 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
951 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
952 I915_WRITE(DSPFW8_CHV,
953 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
954 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
955 I915_WRITE(DSPFW9_CHV,
956 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
957 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
958 I915_WRITE(DSPHOWM,
959 FW_WM(wm->sr.plane >> 9, SR_HI) |
960 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
961 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
962 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
963 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
964 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
965 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
966 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
967 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
968 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
969 } else {
970 I915_WRITE(DSPFW7,
971 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
972 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
973 I915_WRITE(DSPHOWM,
974 FW_WM(wm->sr.plane >> 9, SR_HI) |
975 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
976 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
977 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
978 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
979 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
980 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
981 }
982
983 POSTING_READ(DSPFW1);
984}
985
986#undef FW_WM_VLV
987
988
989static unsigned int vlv_wm_method2(unsigned int pixel_rate,
990 unsigned int pipe_htotal,
991 unsigned int horiz_pixels,
992 unsigned int cpp,
993 unsigned int latency)
994{
995 unsigned int ret;
996
997 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
998 ret = (ret + 1) * horiz_pixels * cpp;
999 ret = DIV_ROUND_UP(ret, 64);
1000
1001 return ret;
1002}
1003
1004static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1005{
1006
1007 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1008
1009 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1010
1011 if (IS_CHERRYVIEW(dev_priv)) {
1012 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1013 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1014
1015 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1016 }
1017}
1018
1019static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1020 const struct intel_plane_state *plane_state,
1021 int level)
1022{
1023 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1024 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1025 const struct drm_display_mode *adjusted_mode =
1026 &crtc_state->base.adjusted_mode;
1027 int clock, htotal, cpp, width, wm;
1028
1029 if (dev_priv->wm.pri_latency[level] == 0)
1030 return USHRT_MAX;
1031
1032 if (!plane_state->base.visible)
1033 return 0;
1034
1035 cpp = plane_state->base.fb->format->cpp[0];
1036 clock = adjusted_mode->crtc_clock;
1037 htotal = adjusted_mode->crtc_htotal;
1038 width = crtc_state->pipe_src_w;
1039 if (WARN_ON(htotal == 0))
1040 htotal = 1;
1041
1042 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
1043
1044
1045
1046
1047
1048
1049 wm = 63;
1050 } else {
1051 wm = vlv_wm_method2(clock, htotal, width, cpp,
1052 dev_priv->wm.pri_latency[level] * 10);
1053 }
1054
1055 return min_t(int, wm, USHRT_MAX);
1056}
1057
1058static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1059{
1060 return (active_planes & (BIT(PLANE_SPRITE0) |
1061 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1062}
1063
1064static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1065{
1066 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1067 const struct vlv_pipe_wm *raw =
1068 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1069 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1070 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1071 int num_active_planes = hweight32(active_planes);
1072 const int fifo_size = 511;
1073 int fifo_extra, fifo_left = fifo_size;
1074 int sprite0_fifo_extra = 0;
1075 unsigned int total_rate;
1076 enum plane_id plane_id;
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086 if (vlv_need_sprite0_fifo_workaround(active_planes))
1087 sprite0_fifo_extra = 1;
1088
1089 total_rate = raw->plane[PLANE_PRIMARY] +
1090 raw->plane[PLANE_SPRITE0] +
1091 raw->plane[PLANE_SPRITE1] +
1092 sprite0_fifo_extra;
1093
1094 if (total_rate > fifo_size)
1095 return -EINVAL;
1096
1097 if (total_rate == 0)
1098 total_rate = 1;
1099
1100 for_each_plane_id_on_crtc(crtc, plane_id) {
1101 unsigned int rate;
1102
1103 if ((active_planes & BIT(plane_id)) == 0) {
1104 fifo_state->plane[plane_id] = 0;
1105 continue;
1106 }
1107
1108 rate = raw->plane[plane_id];
1109 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1110 fifo_left -= fifo_state->plane[plane_id];
1111 }
1112
1113 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1114 fifo_left -= sprite0_fifo_extra;
1115
1116 fifo_state->plane[PLANE_CURSOR] = 63;
1117
1118 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1119
1120
1121 for_each_plane_id_on_crtc(crtc, plane_id) {
1122 int plane_extra;
1123
1124 if (fifo_left == 0)
1125 break;
1126
1127 if ((active_planes & BIT(plane_id)) == 0)
1128 continue;
1129
1130 plane_extra = min(fifo_extra, fifo_left);
1131 fifo_state->plane[plane_id] += plane_extra;
1132 fifo_left -= plane_extra;
1133 }
1134
1135 WARN_ON(active_planes != 0 && fifo_left != 0);
1136
1137
1138 if (active_planes == 0) {
1139 WARN_ON(fifo_left != fifo_size);
1140 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1141 }
1142
1143 return 0;
1144}
1145
1146static int vlv_num_wm_levels(struct drm_i915_private *dev_priv)
1147{
1148 return dev_priv->wm.max_level + 1;
1149}
1150
1151
1152static void vlv_invalidate_wms(struct intel_crtc *crtc,
1153 struct vlv_wm_state *wm_state, int level)
1154{
1155 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1156
1157 for (; level < vlv_num_wm_levels(dev_priv); level++) {
1158 enum plane_id plane_id;
1159
1160 for_each_plane_id_on_crtc(crtc, plane_id)
1161 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1162
1163 wm_state->sr[level].cursor = USHRT_MAX;
1164 wm_state->sr[level].plane = USHRT_MAX;
1165 }
1166}
1167
1168static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1169{
1170 if (wm > fifo_size)
1171 return USHRT_MAX;
1172 else
1173 return fifo_size - wm;
1174}
1175
1176
1177
1178
1179
1180static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1181 int level, enum plane_id plane_id, u16 value)
1182{
1183 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1184 int num_levels = vlv_num_wm_levels(dev_priv);
1185 bool dirty = false;
1186
1187 for (; level < num_levels; level++) {
1188 struct vlv_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1189
1190 dirty |= raw->plane[plane_id] != value;
1191 raw->plane[plane_id] = value;
1192 }
1193
1194 return dirty;
1195}
1196
1197static bool vlv_plane_wm_compute(struct intel_crtc_state *crtc_state,
1198 const struct intel_plane_state *plane_state)
1199{
1200 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1201 enum plane_id plane_id = plane->id;
1202 int num_levels = vlv_num_wm_levels(to_i915(plane->base.dev));
1203 int level;
1204 bool dirty = false;
1205
1206 if (!plane_state->base.visible) {
1207 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1208 goto out;
1209 }
1210
1211 for (level = 0; level < num_levels; level++) {
1212 struct vlv_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1213 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1214 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1215
1216 if (wm > max_wm)
1217 break;
1218
1219 dirty |= raw->plane[plane_id] != wm;
1220 raw->plane[plane_id] = wm;
1221 }
1222
1223
1224 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1225
1226out:
1227 if (dirty)
1228 DRM_DEBUG_KMS("%s wms: [0]=%d,[1]=%d,[2]=%d\n",
1229 plane->base.name,
1230 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1231 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1232 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1233
1234 return dirty;
1235}
1236
1237static bool vlv_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1238 enum plane_id plane_id, int level)
1239{
1240 const struct vlv_pipe_wm *raw =
1241 &crtc_state->wm.vlv.raw[level];
1242 const struct vlv_fifo_state *fifo_state =
1243 &crtc_state->wm.vlv.fifo_state;
1244
1245 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1246}
1247
1248static bool vlv_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1249{
1250 return vlv_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1251 vlv_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1252 vlv_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1253 vlv_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1254}
1255
1256static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1257{
1258 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1259 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1260 struct intel_atomic_state *state =
1261 to_intel_atomic_state(crtc_state->base.state);
1262 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1263 const struct vlv_fifo_state *fifo_state =
1264 &crtc_state->wm.vlv.fifo_state;
1265 int num_active_planes = hweight32(crtc_state->active_planes &
1266 ~BIT(PLANE_CURSOR));
1267 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1268 struct intel_plane_state *plane_state;
1269 struct intel_plane *plane;
1270 enum plane_id plane_id;
1271 int level, ret, i;
1272 unsigned int dirty = 0;
1273
1274 for_each_intel_plane_in_state(state, plane, plane_state, i) {
1275 const struct intel_plane_state *old_plane_state =
1276 to_intel_plane_state(plane->base.state);
1277
1278 if (plane_state->base.crtc != &crtc->base &&
1279 old_plane_state->base.crtc != &crtc->base)
1280 continue;
1281
1282 if (vlv_plane_wm_compute(crtc_state, plane_state))
1283 dirty |= BIT(plane->id);
1284 }
1285
1286
1287
1288
1289
1290
1291
1292 if (needs_modeset)
1293 crtc_state->fifo_changed = true;
1294
1295 if (!dirty)
1296 return 0;
1297
1298
1299 if (dirty & ~BIT(PLANE_CURSOR)) {
1300 const struct intel_crtc_state *old_crtc_state =
1301 to_intel_crtc_state(crtc->base.state);
1302 const struct vlv_fifo_state *old_fifo_state =
1303 &old_crtc_state->wm.vlv.fifo_state;
1304
1305 ret = vlv_compute_fifo(crtc_state);
1306 if (ret)
1307 return ret;
1308
1309 if (needs_modeset ||
1310 memcmp(old_fifo_state, fifo_state,
1311 sizeof(*fifo_state)) != 0)
1312 crtc_state->fifo_changed = true;
1313 }
1314
1315
1316 wm_state->num_levels = vlv_num_wm_levels(dev_priv);
1317
1318
1319
1320
1321
1322 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1323
1324 for (level = 0; level < wm_state->num_levels; level++) {
1325 const struct vlv_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1326 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1327
1328 if (!vlv_crtc_wm_is_valid(crtc_state, level))
1329 break;
1330
1331 for_each_plane_id_on_crtc(crtc, plane_id) {
1332 wm_state->wm[level].plane[plane_id] =
1333 vlv_invert_wm_value(raw->plane[plane_id],
1334 fifo_state->plane[plane_id]);
1335 }
1336
1337 wm_state->sr[level].plane =
1338 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1339 raw->plane[PLANE_SPRITE0],
1340 raw->plane[PLANE_SPRITE1]),
1341 sr_fifo_size);
1342
1343 wm_state->sr[level].cursor =
1344 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1345 63);
1346 }
1347
1348 if (level == 0)
1349 return -EINVAL;
1350
1351
1352 wm_state->num_levels = level;
1353
1354
1355 vlv_invalidate_wms(crtc, wm_state, level);
1356
1357 return 0;
1358}
1359
1360#define VLV_FIFO(plane, value) \
1361 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1362
1363static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1364 struct intel_crtc_state *crtc_state)
1365{
1366 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1367 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1368 const struct vlv_fifo_state *fifo_state =
1369 &crtc_state->wm.vlv.fifo_state;
1370 int sprite0_start, sprite1_start, fifo_size;
1371
1372 if (!crtc_state->fifo_changed)
1373 return;
1374
1375 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1376 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1377 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1378
1379 WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1380 WARN_ON(fifo_size != 511);
1381
1382 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393 spin_lock(&dev_priv->uncore.lock);
1394
1395 switch (crtc->pipe) {
1396 uint32_t dsparb, dsparb2, dsparb3;
1397 case PIPE_A:
1398 dsparb = I915_READ_FW(DSPARB);
1399 dsparb2 = I915_READ_FW(DSPARB2);
1400
1401 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1402 VLV_FIFO(SPRITEB, 0xff));
1403 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1404 VLV_FIFO(SPRITEB, sprite1_start));
1405
1406 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1407 VLV_FIFO(SPRITEB_HI, 0x1));
1408 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1409 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1410
1411 I915_WRITE_FW(DSPARB, dsparb);
1412 I915_WRITE_FW(DSPARB2, dsparb2);
1413 break;
1414 case PIPE_B:
1415 dsparb = I915_READ_FW(DSPARB);
1416 dsparb2 = I915_READ_FW(DSPARB2);
1417
1418 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1419 VLV_FIFO(SPRITED, 0xff));
1420 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1421 VLV_FIFO(SPRITED, sprite1_start));
1422
1423 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1424 VLV_FIFO(SPRITED_HI, 0xff));
1425 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1426 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1427
1428 I915_WRITE_FW(DSPARB, dsparb);
1429 I915_WRITE_FW(DSPARB2, dsparb2);
1430 break;
1431 case PIPE_C:
1432 dsparb3 = I915_READ_FW(DSPARB3);
1433 dsparb2 = I915_READ_FW(DSPARB2);
1434
1435 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1436 VLV_FIFO(SPRITEF, 0xff));
1437 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1438 VLV_FIFO(SPRITEF, sprite1_start));
1439
1440 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1441 VLV_FIFO(SPRITEF_HI, 0xff));
1442 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1443 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1444
1445 I915_WRITE_FW(DSPARB3, dsparb3);
1446 I915_WRITE_FW(DSPARB2, dsparb2);
1447 break;
1448 default:
1449 break;
1450 }
1451
1452 POSTING_READ_FW(DSPARB);
1453
1454 spin_unlock(&dev_priv->uncore.lock);
1455}
1456
1457#undef VLV_FIFO
1458
1459static int vlv_compute_intermediate_wm(struct drm_device *dev,
1460 struct intel_crtc *crtc,
1461 struct intel_crtc_state *crtc_state)
1462{
1463 struct vlv_wm_state *intermediate = &crtc_state->wm.vlv.intermediate;
1464 const struct vlv_wm_state *optimal = &crtc_state->wm.vlv.optimal;
1465 const struct vlv_wm_state *active = &crtc->wm.active.vlv;
1466 int level;
1467
1468 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1469 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1470 !crtc_state->disable_cxsr;
1471
1472 for (level = 0; level < intermediate->num_levels; level++) {
1473 enum plane_id plane_id;
1474
1475 for_each_plane_id_on_crtc(crtc, plane_id) {
1476 intermediate->wm[level].plane[plane_id] =
1477 min(optimal->wm[level].plane[plane_id],
1478 active->wm[level].plane[plane_id]);
1479 }
1480
1481 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1482 active->sr[level].plane);
1483 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1484 active->sr[level].cursor);
1485 }
1486
1487 vlv_invalidate_wms(crtc, intermediate, level);
1488
1489
1490
1491
1492
1493 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1494 crtc_state->wm.need_postvbl_update = true;
1495
1496 return 0;
1497}
1498
1499static void vlv_merge_wm(struct drm_i915_private *dev_priv,
1500 struct vlv_wm_values *wm)
1501{
1502 struct intel_crtc *crtc;
1503 int num_active_crtcs = 0;
1504
1505 wm->level = dev_priv->wm.max_level;
1506 wm->cxsr = true;
1507
1508 for_each_intel_crtc(&dev_priv->drm, crtc) {
1509 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1510
1511 if (!crtc->active)
1512 continue;
1513
1514 if (!wm_state->cxsr)
1515 wm->cxsr = false;
1516
1517 num_active_crtcs++;
1518 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1519 }
1520
1521 if (num_active_crtcs != 1)
1522 wm->cxsr = false;
1523
1524 if (num_active_crtcs > 1)
1525 wm->level = VLV_WM_LEVEL_PM2;
1526
1527 for_each_intel_crtc(&dev_priv->drm, crtc) {
1528 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1529 enum pipe pipe = crtc->pipe;
1530
1531 wm->pipe[pipe] = wm_state->wm[wm->level];
1532 if (crtc->active && wm->cxsr)
1533 wm->sr = wm_state->sr[wm->level];
1534
1535 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
1536 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
1537 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
1538 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
1539 }
1540}
1541
1542static bool is_disabling(int old, int new, int threshold)
1543{
1544 return old >= threshold && new < threshold;
1545}
1546
1547static bool is_enabling(int old, int new, int threshold)
1548{
1549 return old < threshold && new >= threshold;
1550}
1551
1552static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
1553{
1554 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
1555 struct vlv_wm_values new_wm = {};
1556
1557 vlv_merge_wm(dev_priv, &new_wm);
1558
1559 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1560 return;
1561
1562 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1563 chv_set_memory_dvfs(dev_priv, false);
1564
1565 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1566 chv_set_memory_pm5(dev_priv, false);
1567
1568 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1569 _intel_set_memory_cxsr(dev_priv, false);
1570
1571 vlv_write_wm_values(dev_priv, &new_wm);
1572
1573 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1574 _intel_set_memory_cxsr(dev_priv, true);
1575
1576 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1577 chv_set_memory_pm5(dev_priv, true);
1578
1579 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1580 chv_set_memory_dvfs(dev_priv, true);
1581
1582 *old_wm = new_wm;
1583}
1584
1585static void vlv_initial_watermarks(struct intel_atomic_state *state,
1586 struct intel_crtc_state *crtc_state)
1587{
1588 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1589 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1590
1591 mutex_lock(&dev_priv->wm.wm_mutex);
1592 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
1593 vlv_program_watermarks(dev_priv);
1594 mutex_unlock(&dev_priv->wm.wm_mutex);
1595}
1596
1597static void vlv_optimize_watermarks(struct intel_atomic_state *state,
1598 struct intel_crtc_state *crtc_state)
1599{
1600 struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1601 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1602
1603 if (!crtc_state->wm.need_postvbl_update)
1604 return;
1605
1606 mutex_lock(&dev_priv->wm.wm_mutex);
1607 intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
1608 vlv_program_watermarks(dev_priv);
1609 mutex_unlock(&dev_priv->wm.wm_mutex);
1610}
1611
1612#define single_plane_enabled(mask) is_power_of_2(mask)
1613
1614static void g4x_update_wm(struct intel_crtc *crtc)
1615{
1616 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1617 static const int sr_latency_ns = 12000;
1618 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1619 int plane_sr, cursor_sr;
1620 unsigned int enabled = 0;
1621 bool cxsr_enabled;
1622
1623 if (g4x_compute_wm0(dev_priv, PIPE_A,
1624 &g4x_wm_info, pessimal_latency_ns,
1625 &g4x_cursor_wm_info, pessimal_latency_ns,
1626 &planea_wm, &cursora_wm))
1627 enabled |= 1 << PIPE_A;
1628
1629 if (g4x_compute_wm0(dev_priv, PIPE_B,
1630 &g4x_wm_info, pessimal_latency_ns,
1631 &g4x_cursor_wm_info, pessimal_latency_ns,
1632 &planeb_wm, &cursorb_wm))
1633 enabled |= 1 << PIPE_B;
1634
1635 if (single_plane_enabled(enabled) &&
1636 g4x_compute_srwm(dev_priv, ffs(enabled) - 1,
1637 sr_latency_ns,
1638 &g4x_wm_info,
1639 &g4x_cursor_wm_info,
1640 &plane_sr, &cursor_sr)) {
1641 cxsr_enabled = true;
1642 } else {
1643 cxsr_enabled = false;
1644 intel_set_memory_cxsr(dev_priv, false);
1645 plane_sr = cursor_sr = 0;
1646 }
1647
1648 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
1649 "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1650 planea_wm, cursora_wm,
1651 planeb_wm, cursorb_wm,
1652 plane_sr, cursor_sr);
1653
1654 I915_WRITE(DSPFW1,
1655 FW_WM(plane_sr, SR) |
1656 FW_WM(cursorb_wm, CURSORB) |
1657 FW_WM(planeb_wm, PLANEB) |
1658 FW_WM(planea_wm, PLANEA));
1659 I915_WRITE(DSPFW2,
1660 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1661 FW_WM(cursora_wm, CURSORA));
1662
1663 I915_WRITE(DSPFW3,
1664 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1665 FW_WM(cursor_sr, CURSOR_SR));
1666
1667 if (cxsr_enabled)
1668 intel_set_memory_cxsr(dev_priv, true);
1669}
1670
1671static void i965_update_wm(struct intel_crtc *unused_crtc)
1672{
1673 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
1674 struct intel_crtc *crtc;
1675 int srwm = 1;
1676 int cursor_sr = 16;
1677 bool cxsr_enabled;
1678
1679
1680 crtc = single_enabled_crtc(dev_priv);
1681 if (crtc) {
1682
1683 static const int sr_latency_ns = 12000;
1684 const struct drm_display_mode *adjusted_mode =
1685 &crtc->config->base.adjusted_mode;
1686 const struct drm_framebuffer *fb =
1687 crtc->base.primary->state->fb;
1688 int clock = adjusted_mode->crtc_clock;
1689 int htotal = adjusted_mode->crtc_htotal;
1690 int hdisplay = crtc->config->pipe_src_w;
1691 int cpp = fb->format->cpp[0];
1692 unsigned long line_time_us;
1693 int entries;
1694
1695 line_time_us = max(htotal * 1000 / clock, 1);
1696
1697
1698 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1699 cpp * hdisplay;
1700 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1701 srwm = I965_FIFO_SIZE - entries;
1702 if (srwm < 0)
1703 srwm = 1;
1704 srwm &= 0x1ff;
1705 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1706 entries, srwm);
1707
1708 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1709 cpp * crtc->base.cursor->state->crtc_w;
1710 entries = DIV_ROUND_UP(entries,
1711 i965_cursor_wm_info.cacheline_size);
1712 cursor_sr = i965_cursor_wm_info.fifo_size -
1713 (entries + i965_cursor_wm_info.guard_size);
1714
1715 if (cursor_sr > i965_cursor_wm_info.max_wm)
1716 cursor_sr = i965_cursor_wm_info.max_wm;
1717
1718 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1719 "cursor %d\n", srwm, cursor_sr);
1720
1721 cxsr_enabled = true;
1722 } else {
1723 cxsr_enabled = false;
1724
1725 intel_set_memory_cxsr(dev_priv, false);
1726 }
1727
1728 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1729 srwm);
1730
1731
1732 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
1733 FW_WM(8, CURSORB) |
1734 FW_WM(8, PLANEB) |
1735 FW_WM(8, PLANEA));
1736 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
1737 FW_WM(8, PLANEC_OLD));
1738
1739 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
1740
1741 if (cxsr_enabled)
1742 intel_set_memory_cxsr(dev_priv, true);
1743}
1744
1745#undef FW_WM
1746
1747static void i9xx_update_wm(struct intel_crtc *unused_crtc)
1748{
1749 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
1750 const struct intel_watermark_params *wm_info;
1751 uint32_t fwater_lo;
1752 uint32_t fwater_hi;
1753 int cwm, srwm = 1;
1754 int fifo_size;
1755 int planea_wm, planeb_wm;
1756 struct intel_crtc *crtc, *enabled = NULL;
1757
1758 if (IS_I945GM(dev_priv))
1759 wm_info = &i945_wm_info;
1760 else if (!IS_GEN2(dev_priv))
1761 wm_info = &i915_wm_info;
1762 else
1763 wm_info = &i830_a_wm_info;
1764
1765 fifo_size = dev_priv->display.get_fifo_size(dev_priv, 0);
1766 crtc = intel_get_crtc_for_plane(dev_priv, 0);
1767 if (intel_crtc_active(crtc)) {
1768 const struct drm_display_mode *adjusted_mode =
1769 &crtc->config->base.adjusted_mode;
1770 const struct drm_framebuffer *fb =
1771 crtc->base.primary->state->fb;
1772 int cpp;
1773
1774 if (IS_GEN2(dev_priv))
1775 cpp = 4;
1776 else
1777 cpp = fb->format->cpp[0];
1778
1779 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1780 wm_info, fifo_size, cpp,
1781 pessimal_latency_ns);
1782 enabled = crtc;
1783 } else {
1784 planea_wm = fifo_size - wm_info->guard_size;
1785 if (planea_wm > (long)wm_info->max_wm)
1786 planea_wm = wm_info->max_wm;
1787 }
1788
1789 if (IS_GEN2(dev_priv))
1790 wm_info = &i830_bc_wm_info;
1791
1792 fifo_size = dev_priv->display.get_fifo_size(dev_priv, 1);
1793 crtc = intel_get_crtc_for_plane(dev_priv, 1);
1794 if (intel_crtc_active(crtc)) {
1795 const struct drm_display_mode *adjusted_mode =
1796 &crtc->config->base.adjusted_mode;
1797 const struct drm_framebuffer *fb =
1798 crtc->base.primary->state->fb;
1799 int cpp;
1800
1801 if (IS_GEN2(dev_priv))
1802 cpp = 4;
1803 else
1804 cpp = fb->format->cpp[0];
1805
1806 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1807 wm_info, fifo_size, cpp,
1808 pessimal_latency_ns);
1809 if (enabled == NULL)
1810 enabled = crtc;
1811 else
1812 enabled = NULL;
1813 } else {
1814 planeb_wm = fifo_size - wm_info->guard_size;
1815 if (planeb_wm > (long)wm_info->max_wm)
1816 planeb_wm = wm_info->max_wm;
1817 }
1818
1819 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1820
1821 if (IS_I915GM(dev_priv) && enabled) {
1822 struct drm_i915_gem_object *obj;
1823
1824 obj = intel_fb_obj(enabled->base.primary->state->fb);
1825
1826
1827 if (!i915_gem_object_is_tiled(obj))
1828 enabled = NULL;
1829 }
1830
1831
1832
1833
1834 cwm = 2;
1835
1836
1837 intel_set_memory_cxsr(dev_priv, false);
1838
1839
1840 if (HAS_FW_BLC(dev_priv) && enabled) {
1841
1842 static const int sr_latency_ns = 6000;
1843 const struct drm_display_mode *adjusted_mode =
1844 &enabled->config->base.adjusted_mode;
1845 const struct drm_framebuffer *fb =
1846 enabled->base.primary->state->fb;
1847 int clock = adjusted_mode->crtc_clock;
1848 int htotal = adjusted_mode->crtc_htotal;
1849 int hdisplay = enabled->config->pipe_src_w;
1850 int cpp;
1851 unsigned long line_time_us;
1852 int entries;
1853
1854 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
1855 cpp = 4;
1856 else
1857 cpp = fb->format->cpp[0];
1858
1859 line_time_us = max(htotal * 1000 / clock, 1);
1860
1861
1862 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1863 cpp * hdisplay;
1864 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1865 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1866 srwm = wm_info->fifo_size - entries;
1867 if (srwm < 0)
1868 srwm = 1;
1869
1870 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
1871 I915_WRITE(FW_BLC_SELF,
1872 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1873 else
1874 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1875 }
1876
1877 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1878 planea_wm, planeb_wm, cwm, srwm);
1879
1880 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1881 fwater_hi = (cwm & 0x1f);
1882
1883
1884 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1885 fwater_hi = fwater_hi | (1 << 8);
1886
1887 I915_WRITE(FW_BLC, fwater_lo);
1888 I915_WRITE(FW_BLC2, fwater_hi);
1889
1890 if (enabled)
1891 intel_set_memory_cxsr(dev_priv, true);
1892}
1893
1894static void i845_update_wm(struct intel_crtc *unused_crtc)
1895{
1896 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
1897 struct intel_crtc *crtc;
1898 const struct drm_display_mode *adjusted_mode;
1899 uint32_t fwater_lo;
1900 int planea_wm;
1901
1902 crtc = single_enabled_crtc(dev_priv);
1903 if (crtc == NULL)
1904 return;
1905
1906 adjusted_mode = &crtc->config->base.adjusted_mode;
1907 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1908 &i845_wm_info,
1909 dev_priv->display.get_fifo_size(dev_priv, 0),
1910 4, pessimal_latency_ns);
1911 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1912 fwater_lo |= (3<<8) | planea_wm;
1913
1914 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1915
1916 I915_WRITE(FW_BLC, fwater_lo);
1917}
1918
1919
1920static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latency)
1921{
1922 uint64_t ret;
1923
1924 if (WARN(latency == 0, "Latency value missing\n"))
1925 return UINT_MAX;
1926
1927 ret = (uint64_t) pixel_rate * cpp * latency;
1928 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1929
1930 return ret;
1931}
1932
1933
1934static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
1935 uint32_t horiz_pixels, uint8_t cpp,
1936 uint32_t latency)
1937{
1938 uint32_t ret;
1939
1940 if (WARN(latency == 0, "Latency value missing\n"))
1941 return UINT_MAX;
1942 if (WARN_ON(!pipe_htotal))
1943 return UINT_MAX;
1944
1945 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1946 ret = (ret + 1) * horiz_pixels * cpp;
1947 ret = DIV_ROUND_UP(ret, 64) + 2;
1948 return ret;
1949}
1950
1951static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
1952 uint8_t cpp)
1953{
1954
1955
1956
1957
1958
1959
1960 if (WARN_ON(!cpp))
1961 return 0;
1962 if (WARN_ON(!horiz_pixels))
1963 return 0;
1964
1965 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
1966}
1967
1968struct ilk_wm_maximums {
1969 uint16_t pri;
1970 uint16_t spr;
1971 uint16_t cur;
1972 uint16_t fbc;
1973};
1974
1975
1976
1977
1978
1979static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
1980 const struct intel_plane_state *pstate,
1981 uint32_t mem_value,
1982 bool is_lp)
1983{
1984 uint32_t method1, method2;
1985 int cpp;
1986
1987 if (!intel_wm_plane_visible(cstate, pstate))
1988 return 0;
1989
1990 cpp = pstate->base.fb->format->cpp[0];
1991
1992 method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
1993
1994 if (!is_lp)
1995 return method1;
1996
1997 method2 = ilk_wm_method2(cstate->pixel_rate,
1998 cstate->base.adjusted_mode.crtc_htotal,
1999 drm_rect_width(&pstate->base.dst),
2000 cpp, mem_value);
2001
2002 return min(method1, method2);
2003}
2004
2005
2006
2007
2008
2009static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2010 const struct intel_plane_state *pstate,
2011 uint32_t mem_value)
2012{
2013 uint32_t method1, method2;
2014 int cpp;
2015
2016 if (!intel_wm_plane_visible(cstate, pstate))
2017 return 0;
2018
2019 cpp = pstate->base.fb->format->cpp[0];
2020
2021 method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2022 method2 = ilk_wm_method2(cstate->pixel_rate,
2023 cstate->base.adjusted_mode.crtc_htotal,
2024 drm_rect_width(&pstate->base.dst),
2025 cpp, mem_value);
2026 return min(method1, method2);
2027}
2028
2029
2030
2031
2032
2033static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2034 const struct intel_plane_state *pstate,
2035 uint32_t mem_value)
2036{
2037 int cpp;
2038
2039 if (!intel_wm_plane_visible(cstate, pstate))
2040 return 0;
2041
2042 cpp = pstate->base.fb->format->cpp[0];
2043
2044 return ilk_wm_method2(cstate->pixel_rate,
2045 cstate->base.adjusted_mode.crtc_htotal,
2046 pstate->base.crtc_w, cpp, mem_value);
2047}
2048
2049
2050static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2051 const struct intel_plane_state *pstate,
2052 uint32_t pri_val)
2053{
2054 int cpp;
2055
2056 if (!intel_wm_plane_visible(cstate, pstate))
2057 return 0;
2058
2059 cpp = pstate->base.fb->format->cpp[0];
2060
2061 return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2062}
2063
2064static unsigned int
2065ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2066{
2067 if (INTEL_GEN(dev_priv) >= 8)
2068 return 3072;
2069 else if (INTEL_GEN(dev_priv) >= 7)
2070 return 768;
2071 else
2072 return 512;
2073}
2074
2075static unsigned int
2076ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2077 int level, bool is_sprite)
2078{
2079 if (INTEL_GEN(dev_priv) >= 8)
2080
2081 return level == 0 ? 255 : 2047;
2082 else if (INTEL_GEN(dev_priv) >= 7)
2083
2084 return level == 0 ? 127 : 1023;
2085 else if (!is_sprite)
2086
2087 return level == 0 ? 127 : 511;
2088 else
2089
2090 return level == 0 ? 63 : 255;
2091}
2092
2093static unsigned int
2094ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2095{
2096 if (INTEL_GEN(dev_priv) >= 7)
2097 return level == 0 ? 63 : 255;
2098 else
2099 return level == 0 ? 31 : 63;
2100}
2101
2102static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2103{
2104 if (INTEL_GEN(dev_priv) >= 8)
2105 return 31;
2106 else
2107 return 15;
2108}
2109
2110
2111static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2112 int level,
2113 const struct intel_wm_config *config,
2114 enum intel_ddb_partitioning ddb_partitioning,
2115 bool is_sprite)
2116{
2117 struct drm_i915_private *dev_priv = to_i915(dev);
2118 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2119
2120
2121 if (is_sprite && !config->sprites_enabled)
2122 return 0;
2123
2124
2125 if (level == 0 || config->num_pipes_active > 1) {
2126 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2127
2128
2129
2130
2131
2132
2133 if (INTEL_GEN(dev_priv) <= 6)
2134 fifo_size /= 2;
2135 }
2136
2137 if (config->sprites_enabled) {
2138
2139 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2140 if (is_sprite)
2141 fifo_size *= 5;
2142 fifo_size /= 6;
2143 } else {
2144 fifo_size /= 2;
2145 }
2146 }
2147
2148
2149 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2150}
2151
2152
2153static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2154 int level,
2155 const struct intel_wm_config *config)
2156{
2157
2158 if (level > 0 && config->num_pipes_active > 1)
2159 return 64;
2160
2161
2162 return ilk_cursor_wm_reg_max(to_i915(dev), level);
2163}
2164
2165static void ilk_compute_wm_maximums(const struct drm_device *dev,
2166 int level,
2167 const struct intel_wm_config *config,
2168 enum intel_ddb_partitioning ddb_partitioning,
2169 struct ilk_wm_maximums *max)
2170{
2171 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2172 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2173 max->cur = ilk_cursor_wm_max(dev, level, config);
2174 max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
2175}
2176
2177static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2178 int level,
2179 struct ilk_wm_maximums *max)
2180{
2181 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2182 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2183 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2184 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2185}
2186
2187static bool ilk_validate_wm_level(int level,
2188 const struct ilk_wm_maximums *max,
2189 struct intel_wm_level *result)
2190{
2191 bool ret;
2192
2193
2194 if (!result->enable)
2195 return false;
2196
2197 result->enable = result->pri_val <= max->pri &&
2198 result->spr_val <= max->spr &&
2199 result->cur_val <= max->cur;
2200
2201 ret = result->enable;
2202
2203
2204
2205
2206
2207
2208 if (level == 0 && !result->enable) {
2209 if (result->pri_val > max->pri)
2210 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2211 level, result->pri_val, max->pri);
2212 if (result->spr_val > max->spr)
2213 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2214 level, result->spr_val, max->spr);
2215 if (result->cur_val > max->cur)
2216 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2217 level, result->cur_val, max->cur);
2218
2219 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2220 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2221 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2222 result->enable = true;
2223 }
2224
2225 return ret;
2226}
2227
2228static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2229 const struct intel_crtc *intel_crtc,
2230 int level,
2231 struct intel_crtc_state *cstate,
2232 struct intel_plane_state *pristate,
2233 struct intel_plane_state *sprstate,
2234 struct intel_plane_state *curstate,
2235 struct intel_wm_level *result)
2236{
2237 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2238 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2239 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2240
2241
2242 if (level > 0) {
2243 pri_latency *= 5;
2244 spr_latency *= 5;
2245 cur_latency *= 5;
2246 }
2247
2248 if (pristate) {
2249 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2250 pri_latency, level);
2251 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2252 }
2253
2254 if (sprstate)
2255 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2256
2257 if (curstate)
2258 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2259
2260 result->enable = true;
2261}
2262
2263static uint32_t
2264hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2265{
2266 const struct intel_atomic_state *intel_state =
2267 to_intel_atomic_state(cstate->base.state);
2268 const struct drm_display_mode *adjusted_mode =
2269 &cstate->base.adjusted_mode;
2270 u32 linetime, ips_linetime;
2271
2272 if (!cstate->base.active)
2273 return 0;
2274 if (WARN_ON(adjusted_mode->crtc_clock == 0))
2275 return 0;
2276 if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2277 return 0;
2278
2279
2280
2281
2282 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2283 adjusted_mode->crtc_clock);
2284 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2285 intel_state->cdclk.logical.cdclk);
2286
2287 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2288 PIPE_WM_LINETIME_TIME(linetime);
2289}
2290
2291static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2292 uint16_t wm[8])
2293{
2294 if (IS_GEN9(dev_priv)) {
2295 uint32_t val;
2296 int ret, i;
2297 int level, max_level = ilk_wm_max_level(dev_priv);
2298
2299
2300 val = 0;
2301 mutex_lock(&dev_priv->rps.hw_lock);
2302 ret = sandybridge_pcode_read(dev_priv,
2303 GEN9_PCODE_READ_MEM_LATENCY,
2304 &val);
2305 mutex_unlock(&dev_priv->rps.hw_lock);
2306
2307 if (ret) {
2308 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2309 return;
2310 }
2311
2312 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2313 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2314 GEN9_MEM_LATENCY_LEVEL_MASK;
2315 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2316 GEN9_MEM_LATENCY_LEVEL_MASK;
2317 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2318 GEN9_MEM_LATENCY_LEVEL_MASK;
2319
2320
2321 val = 1;
2322 mutex_lock(&dev_priv->rps.hw_lock);
2323 ret = sandybridge_pcode_read(dev_priv,
2324 GEN9_PCODE_READ_MEM_LATENCY,
2325 &val);
2326 mutex_unlock(&dev_priv->rps.hw_lock);
2327 if (ret) {
2328 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2329 return;
2330 }
2331
2332 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2333 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2334 GEN9_MEM_LATENCY_LEVEL_MASK;
2335 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2336 GEN9_MEM_LATENCY_LEVEL_MASK;
2337 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2338 GEN9_MEM_LATENCY_LEVEL_MASK;
2339
2340
2341
2342
2343
2344
2345 for (level = 1; level <= max_level; level++) {
2346 if (wm[level] == 0) {
2347 for (i = level + 1; i <= max_level; i++)
2348 wm[i] = 0;
2349 break;
2350 }
2351 }
2352
2353
2354
2355
2356
2357
2358
2359
2360 if (wm[0] == 0) {
2361 wm[0] += 2;
2362 for (level = 1; level <= max_level; level++) {
2363 if (wm[level] == 0)
2364 break;
2365 wm[level] += 2;
2366 }
2367 }
2368
2369 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2370 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2371
2372 wm[0] = (sskpd >> 56) & 0xFF;
2373 if (wm[0] == 0)
2374 wm[0] = sskpd & 0xF;
2375 wm[1] = (sskpd >> 4) & 0xFF;
2376 wm[2] = (sskpd >> 12) & 0xFF;
2377 wm[3] = (sskpd >> 20) & 0x1FF;
2378 wm[4] = (sskpd >> 32) & 0x1FF;
2379 } else if (INTEL_GEN(dev_priv) >= 6) {
2380 uint32_t sskpd = I915_READ(MCH_SSKPD);
2381
2382 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2383 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2384 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2385 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2386 } else if (INTEL_GEN(dev_priv) >= 5) {
2387 uint32_t mltr = I915_READ(MLTR_ILK);
2388
2389
2390 wm[0] = 7;
2391 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2392 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2393 }
2394}
2395
2396static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2397 uint16_t wm[5])
2398{
2399
2400 if (IS_GEN5(dev_priv))
2401 wm[0] = 13;
2402}
2403
2404static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2405 uint16_t wm[5])
2406{
2407
2408 if (IS_GEN5(dev_priv))
2409 wm[0] = 13;
2410
2411
2412 if (IS_IVYBRIDGE(dev_priv))
2413 wm[3] *= 2;
2414}
2415
2416int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2417{
2418
2419 if (INTEL_GEN(dev_priv) >= 9)
2420 return 7;
2421 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2422 return 4;
2423 else if (INTEL_GEN(dev_priv) >= 6)
2424 return 3;
2425 else
2426 return 2;
2427}
2428
2429static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2430 const char *name,
2431 const uint16_t wm[8])
2432{
2433 int level, max_level = ilk_wm_max_level(dev_priv);
2434
2435 for (level = 0; level <= max_level; level++) {
2436 unsigned int latency = wm[level];
2437
2438 if (latency == 0) {
2439 DRM_ERROR("%s WM%d latency not provided\n",
2440 name, level);
2441 continue;
2442 }
2443
2444
2445
2446
2447
2448 if (IS_GEN9(dev_priv))
2449 latency *= 10;
2450 else if (level > 0)
2451 latency *= 5;
2452
2453 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2454 name, level, wm[level],
2455 latency / 10, latency % 10);
2456 }
2457}
2458
2459static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2460 uint16_t wm[5], uint16_t min)
2461{
2462 int level, max_level = ilk_wm_max_level(dev_priv);
2463
2464 if (wm[0] >= min)
2465 return false;
2466
2467 wm[0] = max(wm[0], min);
2468 for (level = 1; level <= max_level; level++)
2469 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2470
2471 return true;
2472}
2473
2474static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2475{
2476 bool changed;
2477
2478
2479
2480
2481
2482 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2483 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2484 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2485
2486 if (!changed)
2487 return;
2488
2489 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2490 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
2491 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
2492 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
2493}
2494
2495static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2496{
2497 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
2498
2499 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2500 sizeof(dev_priv->wm.pri_latency));
2501 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2502 sizeof(dev_priv->wm.pri_latency));
2503
2504 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
2505 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
2506
2507 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
2508 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
2509 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
2510
2511 if (IS_GEN6(dev_priv))
2512 snb_wm_latency_quirk(dev_priv);
2513}
2514
2515static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
2516{
2517 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
2518 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
2519}
2520
2521static bool ilk_validate_pipe_wm(struct drm_device *dev,
2522 struct intel_pipe_wm *pipe_wm)
2523{
2524
2525 const struct intel_wm_config config = {
2526 .num_pipes_active = 1,
2527 .sprites_enabled = pipe_wm->sprites_enabled,
2528 .sprites_scaled = pipe_wm->sprites_scaled,
2529 };
2530 struct ilk_wm_maximums max;
2531
2532
2533 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2534
2535
2536 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
2537 DRM_DEBUG_KMS("LP0 watermark invalid\n");
2538 return false;
2539 }
2540
2541 return true;
2542}
2543
2544
2545static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
2546{
2547 struct drm_atomic_state *state = cstate->base.state;
2548 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
2549 struct intel_pipe_wm *pipe_wm;
2550 struct drm_device *dev = state->dev;
2551 const struct drm_i915_private *dev_priv = to_i915(dev);
2552 struct intel_plane *intel_plane;
2553 struct intel_plane_state *pristate = NULL;
2554 struct intel_plane_state *sprstate = NULL;
2555 struct intel_plane_state *curstate = NULL;
2556 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
2557 struct ilk_wm_maximums max;
2558
2559 pipe_wm = &cstate->wm.ilk.optimal;
2560
2561 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
2562 struct intel_plane_state *ps;
2563
2564 ps = intel_atomic_get_existing_plane_state(state,
2565 intel_plane);
2566 if (!ps)
2567 continue;
2568
2569 if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2570 pristate = ps;
2571 else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2572 sprstate = ps;
2573 else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
2574 curstate = ps;
2575 }
2576
2577 pipe_wm->pipe_enabled = cstate->base.active;
2578 if (sprstate) {
2579 pipe_wm->sprites_enabled = sprstate->base.visible;
2580 pipe_wm->sprites_scaled = sprstate->base.visible &&
2581 (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
2582 drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
2583 }
2584
2585 usable_level = max_level;
2586
2587
2588 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
2589 usable_level = 1;
2590
2591
2592 if (pipe_wm->sprites_scaled)
2593 usable_level = 0;
2594
2595 ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
2596 pristate, sprstate, curstate, &pipe_wm->raw_wm[0]);
2597
2598 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2599 pipe_wm->wm[0] = pipe_wm->raw_wm[0];
2600
2601 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2602 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
2603
2604 if (!ilk_validate_pipe_wm(dev, pipe_wm))
2605 return -EINVAL;
2606
2607 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2608
2609 for (level = 1; level <= max_level; level++) {
2610 struct intel_wm_level *wm = &pipe_wm->raw_wm[level];
2611
2612 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
2613 pristate, sprstate, curstate, wm);
2614
2615
2616
2617
2618
2619
2620 if (level > usable_level)
2621 continue;
2622
2623 if (ilk_validate_wm_level(level, &max, wm))
2624 pipe_wm->wm[level] = *wm;
2625 else
2626 usable_level = level;
2627 }
2628
2629 return 0;
2630}
2631
2632
2633
2634
2635
2636
2637static int ilk_compute_intermediate_wm(struct drm_device *dev,
2638 struct intel_crtc *intel_crtc,
2639 struct intel_crtc_state *newstate)
2640{
2641 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
2642 struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk;
2643 int level, max_level = ilk_wm_max_level(to_i915(dev));
2644
2645
2646
2647
2648
2649
2650 *a = newstate->wm.ilk.optimal;
2651 a->pipe_enabled |= b->pipe_enabled;
2652 a->sprites_enabled |= b->sprites_enabled;
2653 a->sprites_scaled |= b->sprites_scaled;
2654
2655 for (level = 0; level <= max_level; level++) {
2656 struct intel_wm_level *a_wm = &a->wm[level];
2657 const struct intel_wm_level *b_wm = &b->wm[level];
2658
2659 a_wm->enable &= b_wm->enable;
2660 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
2661 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
2662 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
2663 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
2664 }
2665
2666
2667
2668
2669
2670
2671
2672 if (!ilk_validate_pipe_wm(dev, a))
2673 return -EINVAL;
2674
2675
2676
2677
2678
2679 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
2680 newstate->wm.need_postvbl_update = true;
2681
2682 return 0;
2683}
2684
2685
2686
2687
2688static void ilk_merge_wm_level(struct drm_device *dev,
2689 int level,
2690 struct intel_wm_level *ret_wm)
2691{
2692 const struct intel_crtc *intel_crtc;
2693
2694 ret_wm->enable = true;
2695
2696 for_each_intel_crtc(dev, intel_crtc) {
2697 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
2698 const struct intel_wm_level *wm = &active->wm[level];
2699
2700 if (!active->pipe_enabled)
2701 continue;
2702
2703
2704
2705
2706
2707
2708 if (!wm->enable)
2709 ret_wm->enable = false;
2710
2711 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2712 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2713 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2714 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2715 }
2716}
2717
2718
2719
2720
2721static void ilk_wm_merge(struct drm_device *dev,
2722 const struct intel_wm_config *config,
2723 const struct ilk_wm_maximums *max,
2724 struct intel_pipe_wm *merged)
2725{
2726 struct drm_i915_private *dev_priv = to_i915(dev);
2727 int level, max_level = ilk_wm_max_level(dev_priv);
2728 int last_enabled_level = max_level;
2729
2730
2731 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
2732 config->num_pipes_active > 1)
2733 last_enabled_level = 0;
2734
2735
2736 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
2737
2738
2739 for (level = 1; level <= max_level; level++) {
2740 struct intel_wm_level *wm = &merged->wm[level];
2741
2742 ilk_merge_wm_level(dev, level, wm);
2743
2744 if (level > last_enabled_level)
2745 wm->enable = false;
2746 else if (!ilk_validate_wm_level(level, max, wm))
2747
2748 last_enabled_level = level - 1;
2749
2750
2751
2752
2753
2754 if (wm->fbc_val > max->fbc) {
2755 if (wm->enable)
2756 merged->fbc_wm_enabled = false;
2757 wm->fbc_val = 0;
2758 }
2759 }
2760
2761
2762
2763
2764
2765
2766
2767 if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
2768 intel_fbc_is_active(dev_priv)) {
2769 for (level = 2; level <= max_level; level++) {
2770 struct intel_wm_level *wm = &merged->wm[level];
2771
2772 wm->enable = false;
2773 }
2774 }
2775}
2776
2777static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2778{
2779
2780 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2781}
2782
2783
2784static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2785{
2786 struct drm_i915_private *dev_priv = to_i915(dev);
2787
2788 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2789 return 2 * level;
2790 else
2791 return dev_priv->wm.pri_latency[level];
2792}
2793
2794static void ilk_compute_wm_results(struct drm_device *dev,
2795 const struct intel_pipe_wm *merged,
2796 enum intel_ddb_partitioning partitioning,
2797 struct ilk_wm_values *results)
2798{
2799 struct drm_i915_private *dev_priv = to_i915(dev);
2800 struct intel_crtc *intel_crtc;
2801 int level, wm_lp;
2802
2803 results->enable_fbc_wm = merged->fbc_wm_enabled;
2804 results->partitioning = partitioning;
2805
2806
2807 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2808 const struct intel_wm_level *r;
2809
2810 level = ilk_wm_lp_to_level(wm_lp, merged);
2811
2812 r = &merged->wm[level];
2813
2814
2815
2816
2817
2818 results->wm_lp[wm_lp - 1] =
2819 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2820 (r->pri_val << WM1_LP_SR_SHIFT) |
2821 r->cur_val;
2822
2823 if (r->enable)
2824 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2825
2826 if (INTEL_GEN(dev_priv) >= 8)
2827 results->wm_lp[wm_lp - 1] |=
2828 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2829 else
2830 results->wm_lp[wm_lp - 1] |=
2831 r->fbc_val << WM1_LP_FBC_SHIFT;
2832
2833
2834
2835
2836
2837 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
2838 WARN_ON(wm_lp != 1);
2839 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2840 } else
2841 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2842 }
2843
2844
2845 for_each_intel_crtc(dev, intel_crtc) {
2846 enum pipe pipe = intel_crtc->pipe;
2847 const struct intel_wm_level *r =
2848 &intel_crtc->wm.active.ilk.wm[0];
2849
2850 if (WARN_ON(!r->enable))
2851 continue;
2852
2853 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
2854
2855 results->wm_pipe[pipe] =
2856 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2857 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2858 r->cur_val;
2859 }
2860}
2861
2862
2863
2864static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2865 struct intel_pipe_wm *r1,
2866 struct intel_pipe_wm *r2)
2867{
2868 int level, max_level = ilk_wm_max_level(to_i915(dev));
2869 int level1 = 0, level2 = 0;
2870
2871 for (level = 1; level <= max_level; level++) {
2872 if (r1->wm[level].enable)
2873 level1 = level;
2874 if (r2->wm[level].enable)
2875 level2 = level;
2876 }
2877
2878 if (level1 == level2) {
2879 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2880 return r2;
2881 else
2882 return r1;
2883 } else if (level1 > level2) {
2884 return r1;
2885 } else {
2886 return r2;
2887 }
2888}
2889
2890
2891#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2892#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2893#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2894#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2895#define WM_DIRTY_FBC (1 << 24)
2896#define WM_DIRTY_DDB (1 << 25)
2897
2898static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
2899 const struct ilk_wm_values *old,
2900 const struct ilk_wm_values *new)
2901{
2902 unsigned int dirty = 0;
2903 enum pipe pipe;
2904 int wm_lp;
2905
2906 for_each_pipe(dev_priv, pipe) {
2907 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2908 dirty |= WM_DIRTY_LINETIME(pipe);
2909
2910 dirty |= WM_DIRTY_LP_ALL;
2911 }
2912
2913 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2914 dirty |= WM_DIRTY_PIPE(pipe);
2915
2916 dirty |= WM_DIRTY_LP_ALL;
2917 }
2918 }
2919
2920 if (old->enable_fbc_wm != new->enable_fbc_wm) {
2921 dirty |= WM_DIRTY_FBC;
2922
2923 dirty |= WM_DIRTY_LP_ALL;
2924 }
2925
2926 if (old->partitioning != new->partitioning) {
2927 dirty |= WM_DIRTY_DDB;
2928
2929 dirty |= WM_DIRTY_LP_ALL;
2930 }
2931
2932
2933 if (dirty & WM_DIRTY_LP_ALL)
2934 return dirty;
2935
2936
2937 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2938 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2939 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2940 break;
2941 }
2942
2943
2944 for (; wm_lp <= 3; wm_lp++)
2945 dirty |= WM_DIRTY_LP(wm_lp);
2946
2947 return dirty;
2948}
2949
2950static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2951 unsigned int dirty)
2952{
2953 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2954 bool changed = false;
2955
2956 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2957 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2958 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2959 changed = true;
2960 }
2961 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2962 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2963 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2964 changed = true;
2965 }
2966 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2967 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2968 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2969 changed = true;
2970 }
2971
2972
2973
2974
2975
2976
2977 return changed;
2978}
2979
2980
2981
2982
2983
2984static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2985 struct ilk_wm_values *results)
2986{
2987 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2988 unsigned int dirty;
2989 uint32_t val;
2990
2991 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
2992 if (!dirty)
2993 return;
2994
2995 _ilk_disable_lp_wm(dev_priv, dirty);
2996
2997 if (dirty & WM_DIRTY_PIPE(PIPE_A))
2998 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2999 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3000 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3001 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3002 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3003
3004 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3005 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3006 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3007 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3008 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3009 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3010
3011 if (dirty & WM_DIRTY_DDB) {
3012 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3013 val = I915_READ(WM_MISC);
3014 if (results->partitioning == INTEL_DDB_PART_1_2)
3015 val &= ~WM_MISC_DATA_PARTITION_5_6;
3016 else
3017 val |= WM_MISC_DATA_PARTITION_5_6;
3018 I915_WRITE(WM_MISC, val);
3019 } else {
3020 val = I915_READ(DISP_ARB_CTL2);
3021 if (results->partitioning == INTEL_DDB_PART_1_2)
3022 val &= ~DISP_DATA_PARTITION_5_6;
3023 else
3024 val |= DISP_DATA_PARTITION_5_6;
3025 I915_WRITE(DISP_ARB_CTL2, val);
3026 }
3027 }
3028
3029 if (dirty & WM_DIRTY_FBC) {
3030 val = I915_READ(DISP_ARB_CTL);
3031 if (results->enable_fbc_wm)
3032 val &= ~DISP_FBC_WM_DIS;
3033 else
3034 val |= DISP_FBC_WM_DIS;
3035 I915_WRITE(DISP_ARB_CTL, val);
3036 }
3037
3038 if (dirty & WM_DIRTY_LP(1) &&
3039 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3040 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3041
3042 if (INTEL_GEN(dev_priv) >= 7) {
3043 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3044 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3045 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3046 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3047 }
3048
3049 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3050 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3051 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3052 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3053 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3054 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3055
3056 dev_priv->wm.hw = *results;
3057}
3058
3059bool ilk_disable_lp_wm(struct drm_device *dev)
3060{
3061 struct drm_i915_private *dev_priv = to_i915(dev);
3062
3063 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3064}
3065
3066#define SKL_SAGV_BLOCK_TIME 30
3067
3068
3069
3070
3071
3072static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
3073{
3074 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3075
3076 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
3077 return true;
3078
3079 return false;
3080}
3081
3082static bool
3083intel_has_sagv(struct drm_i915_private *dev_priv)
3084{
3085 if (IS_KABYLAKE(dev_priv))
3086 return true;
3087
3088 if (IS_SKYLAKE(dev_priv) &&
3089 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
3090 return true;
3091
3092 return false;
3093}
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106int
3107intel_enable_sagv(struct drm_i915_private *dev_priv)
3108{
3109 int ret;
3110
3111 if (!intel_has_sagv(dev_priv))
3112 return 0;
3113
3114 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3115 return 0;
3116
3117 DRM_DEBUG_KMS("Enabling the SAGV\n");
3118 mutex_lock(&dev_priv->rps.hw_lock);
3119
3120 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3121 GEN9_SAGV_ENABLE);
3122
3123
3124 mutex_unlock(&dev_priv->rps.hw_lock);
3125
3126
3127
3128
3129
3130 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3131 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3132 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3133 return 0;
3134 } else if (ret < 0) {
3135 DRM_ERROR("Failed to enable the SAGV\n");
3136 return ret;
3137 }
3138
3139 dev_priv->sagv_status = I915_SAGV_ENABLED;
3140 return 0;
3141}
3142
3143int
3144intel_disable_sagv(struct drm_i915_private *dev_priv)
3145{
3146 int ret;
3147
3148 if (!intel_has_sagv(dev_priv))
3149 return 0;
3150
3151 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3152 return 0;
3153
3154 DRM_DEBUG_KMS("Disabling the SAGV\n");
3155 mutex_lock(&dev_priv->rps.hw_lock);
3156
3157
3158 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3159 GEN9_SAGV_DISABLE,
3160 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3161 1);
3162 mutex_unlock(&dev_priv->rps.hw_lock);
3163
3164
3165
3166
3167
3168 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3169 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3170 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3171 return 0;
3172 } else if (ret < 0) {
3173 DRM_ERROR("Failed to disable the SAGV (%d)\n", ret);
3174 return ret;
3175 }
3176
3177 dev_priv->sagv_status = I915_SAGV_DISABLED;
3178 return 0;
3179}
3180
3181bool intel_can_enable_sagv(struct drm_atomic_state *state)
3182{
3183 struct drm_device *dev = state->dev;
3184 struct drm_i915_private *dev_priv = to_i915(dev);
3185 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3186 struct intel_crtc *crtc;
3187 struct intel_plane *plane;
3188 struct intel_crtc_state *cstate;
3189 enum pipe pipe;
3190 int level, latency;
3191
3192 if (!intel_has_sagv(dev_priv))
3193 return false;
3194
3195
3196
3197
3198
3199
3200
3201 if (hweight32(intel_state->active_crtcs) == 0)
3202 return true;
3203 else if (hweight32(intel_state->active_crtcs) > 1)
3204 return false;
3205
3206
3207 pipe = ffs(intel_state->active_crtcs) - 1;
3208 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3209 cstate = to_intel_crtc_state(crtc->base.state);
3210
3211 if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3212 return false;
3213
3214 for_each_intel_plane_on_crtc(dev, crtc, plane) {
3215 struct skl_plane_wm *wm =
3216 &cstate->wm.skl.optimal.planes[plane->id];
3217
3218
3219 if (!wm->wm[0].plane_en)
3220 continue;
3221
3222
3223 for (level = ilk_wm_max_level(dev_priv);
3224 !wm->wm[level].plane_en; --level)
3225 { }
3226
3227 latency = dev_priv->wm.skl_latency[level];
3228
3229 if (skl_needs_memory_bw_wa(intel_state) &&
3230 plane->base.state->fb->modifier ==
3231 I915_FORMAT_MOD_X_TILED)
3232 latency += 15;
3233
3234
3235
3236
3237
3238
3239 if (latency < SKL_SAGV_BLOCK_TIME)
3240 return false;
3241 }
3242
3243 return true;
3244}
3245
3246static void
3247skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3248 const struct intel_crtc_state *cstate,
3249 struct skl_ddb_entry *alloc,
3250 int *num_active )
3251{
3252 struct drm_atomic_state *state = cstate->base.state;
3253 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3254 struct drm_i915_private *dev_priv = to_i915(dev);
3255 struct drm_crtc *for_crtc = cstate->base.crtc;
3256 unsigned int pipe_size, ddb_size;
3257 int nth_active_pipe;
3258
3259 if (WARN_ON(!state) || !cstate->base.active) {
3260 alloc->start = 0;
3261 alloc->end = 0;
3262 *num_active = hweight32(dev_priv->active_crtcs);
3263 return;
3264 }
3265
3266 if (intel_state->active_pipe_changes)
3267 *num_active = hweight32(intel_state->active_crtcs);
3268 else
3269 *num_active = hweight32(dev_priv->active_crtcs);
3270
3271 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3272 WARN_ON(ddb_size == 0);
3273
3274 ddb_size -= 4;
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284 if (!intel_state->active_pipe_changes) {
3285
3286
3287
3288
3289 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3290 return;
3291 }
3292
3293 nth_active_pipe = hweight32(intel_state->active_crtcs &
3294 (drm_crtc_mask(for_crtc) - 1));
3295 pipe_size = ddb_size / hweight32(intel_state->active_crtcs);
3296 alloc->start = nth_active_pipe * ddb_size / *num_active;
3297 alloc->end = alloc->start + pipe_size;
3298}
3299
3300static unsigned int skl_cursor_allocation(int num_active)
3301{
3302 if (num_active == 1)
3303 return 32;
3304
3305 return 8;
3306}
3307
3308static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
3309{
3310 entry->start = reg & 0x3ff;
3311 entry->end = (reg >> 16) & 0x3ff;
3312 if (entry->end)
3313 entry->end += 1;
3314}
3315
3316void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
3317 struct skl_ddb_allocation *ddb )
3318{
3319 struct intel_crtc *crtc;
3320
3321 memset(ddb, 0, sizeof(*ddb));
3322
3323 for_each_intel_crtc(&dev_priv->drm, crtc) {
3324 enum intel_display_power_domain power_domain;
3325 enum plane_id plane_id;
3326 enum pipe pipe = crtc->pipe;
3327
3328 power_domain = POWER_DOMAIN_PIPE(pipe);
3329 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3330 continue;
3331
3332 for_each_plane_id_on_crtc(crtc, plane_id) {
3333 u32 val;
3334
3335 if (plane_id != PLANE_CURSOR)
3336 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
3337 else
3338 val = I915_READ(CUR_BUF_CFG(pipe));
3339
3340 skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane_id], val);
3341 }
3342
3343 intel_display_power_put(dev_priv, power_domain);
3344 }
3345}
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363static uint32_t
3364skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
3365 const struct intel_plane_state *pstate)
3366{
3367 struct intel_plane *plane = to_intel_plane(pstate->base.plane);
3368 uint32_t downscale_h, downscale_w;
3369 uint32_t src_w, src_h, dst_w, dst_h;
3370
3371 if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
3372 return DRM_PLANE_HELPER_NO_SCALING;
3373
3374
3375 if (plane->id == PLANE_CURSOR) {
3376
3377
3378
3379
3380 src_w = pstate->base.src_w;
3381 src_h = pstate->base.src_h;
3382 dst_w = pstate->base.crtc_w;
3383 dst_h = pstate->base.crtc_h;
3384 } else {
3385
3386
3387
3388
3389
3390 src_w = drm_rect_width(&pstate->base.src);
3391 src_h = drm_rect_height(&pstate->base.src);
3392 dst_w = drm_rect_width(&pstate->base.dst);
3393 dst_h = drm_rect_height(&pstate->base.dst);
3394 }
3395
3396 downscale_h = max(src_h / dst_h, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
3397 downscale_w = max(src_w / dst_w, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
3398
3399
3400 return (uint64_t)downscale_w * downscale_h >> 16;
3401}
3402
3403static unsigned int
3404skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
3405 const struct drm_plane_state *pstate,
3406 int y)
3407{
3408 struct intel_plane *plane = to_intel_plane(pstate->plane);
3409 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
3410 uint32_t down_scale_amount, data_rate;
3411 uint32_t width = 0, height = 0;
3412 struct drm_framebuffer *fb;
3413 u32 format;
3414
3415 if (!intel_pstate->base.visible)
3416 return 0;
3417
3418 fb = pstate->fb;
3419 format = fb->format->format;
3420
3421 if (plane->id == PLANE_CURSOR)
3422 return 0;
3423 if (y && format != DRM_FORMAT_NV12)
3424 return 0;
3425
3426
3427
3428
3429
3430
3431 width = drm_rect_width(&intel_pstate->base.src) >> 16;
3432 height = drm_rect_height(&intel_pstate->base.src) >> 16;
3433
3434
3435 if (format == DRM_FORMAT_NV12) {
3436 if (y)
3437 data_rate = width * height *
3438 fb->format->cpp[0];
3439 else
3440 data_rate = (width / 2) * (height / 2) *
3441 fb->format->cpp[1];
3442 } else {
3443
3444 data_rate = width * height * fb->format->cpp[0];
3445 }
3446
3447 down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
3448
3449 return (uint64_t)data_rate * down_scale_amount >> 16;
3450}
3451
3452
3453
3454
3455
3456
3457static unsigned int
3458skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
3459 unsigned *plane_data_rate,
3460 unsigned *plane_y_data_rate)
3461{
3462 struct drm_crtc_state *cstate = &intel_cstate->base;
3463 struct drm_atomic_state *state = cstate->state;
3464 struct drm_plane *plane;
3465 const struct drm_plane_state *pstate;
3466 unsigned int total_data_rate = 0;
3467
3468 if (WARN_ON(!state))
3469 return 0;
3470
3471
3472 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
3473 enum plane_id plane_id = to_intel_plane(plane)->id;
3474 unsigned int rate;
3475
3476
3477 rate = skl_plane_relative_data_rate(intel_cstate,
3478 pstate, 0);
3479 plane_data_rate[plane_id] = rate;
3480
3481 total_data_rate += rate;
3482
3483
3484 rate = skl_plane_relative_data_rate(intel_cstate,
3485 pstate, 1);
3486 plane_y_data_rate[plane_id] = rate;
3487
3488 total_data_rate += rate;
3489 }
3490
3491 return total_data_rate;
3492}
3493
3494static uint16_t
3495skl_ddb_min_alloc(const struct drm_plane_state *pstate,
3496 const int y)
3497{
3498 struct drm_framebuffer *fb = pstate->fb;
3499 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
3500 uint32_t src_w, src_h;
3501 uint32_t min_scanlines = 8;
3502 uint8_t plane_bpp;
3503
3504 if (WARN_ON(!fb))
3505 return 0;
3506
3507
3508 if (y && fb->format->format != DRM_FORMAT_NV12)
3509 return 0;
3510
3511
3512 if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
3513 fb->modifier != I915_FORMAT_MOD_Yf_TILED)
3514 return 8;
3515
3516
3517
3518
3519
3520
3521 src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
3522 src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
3523
3524
3525 if (fb->format->format == DRM_FORMAT_NV12 && !y) {
3526 src_w /= 2;
3527 src_h /= 2;
3528 }
3529
3530 if (fb->format->format == DRM_FORMAT_NV12 && !y)
3531 plane_bpp = fb->format->cpp[1];
3532 else
3533 plane_bpp = fb->format->cpp[0];
3534
3535 if (drm_rotation_90_or_270(pstate->rotation)) {
3536 switch (plane_bpp) {
3537 case 1:
3538 min_scanlines = 32;
3539 break;
3540 case 2:
3541 min_scanlines = 16;
3542 break;
3543 case 4:
3544 min_scanlines = 8;
3545 break;
3546 case 8:
3547 min_scanlines = 4;
3548 break;
3549 default:
3550 WARN(1, "Unsupported pixel depth %u for rotation",
3551 plane_bpp);
3552 min_scanlines = 32;
3553 }
3554 }
3555
3556 return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
3557}
3558
3559static void
3560skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
3561 uint16_t *minimum, uint16_t *y_minimum)
3562{
3563 const struct drm_plane_state *pstate;
3564 struct drm_plane *plane;
3565
3566 drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
3567 enum plane_id plane_id = to_intel_plane(plane)->id;
3568
3569 if (plane_id == PLANE_CURSOR)
3570 continue;
3571
3572 if (!pstate->visible)
3573 continue;
3574
3575 minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
3576 y_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
3577 }
3578
3579 minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
3580}
3581
3582static int
3583skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
3584 struct skl_ddb_allocation *ddb )
3585{
3586 struct drm_atomic_state *state = cstate->base.state;
3587 struct drm_crtc *crtc = cstate->base.crtc;
3588 struct drm_device *dev = crtc->dev;
3589 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3590 enum pipe pipe = intel_crtc->pipe;
3591 struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
3592 uint16_t alloc_size, start;
3593 uint16_t minimum[I915_MAX_PLANES] = {};
3594 uint16_t y_minimum[I915_MAX_PLANES] = {};
3595 unsigned int total_data_rate;
3596 enum plane_id plane_id;
3597 int num_active;
3598 unsigned plane_data_rate[I915_MAX_PLANES] = {};
3599 unsigned plane_y_data_rate[I915_MAX_PLANES] = {};
3600
3601
3602 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
3603 memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
3604
3605 if (WARN_ON(!state))
3606 return 0;
3607
3608 if (!cstate->base.active) {
3609 alloc->start = alloc->end = 0;
3610 return 0;
3611 }
3612
3613 skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active);
3614 alloc_size = skl_ddb_entry_size(alloc);
3615 if (alloc_size == 0) {
3616 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
3617 return 0;
3618 }
3619
3620 skl_ddb_calc_min(cstate, num_active, minimum, y_minimum);
3621
3622
3623
3624
3625
3626
3627
3628 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
3629 alloc_size -= minimum[plane_id];
3630 alloc_size -= y_minimum[plane_id];
3631 }
3632
3633 ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
3634 ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
3635
3636
3637
3638
3639
3640
3641
3642 total_data_rate = skl_get_total_relative_data_rate(cstate,
3643 plane_data_rate,
3644 plane_y_data_rate);
3645 if (total_data_rate == 0)
3646 return 0;
3647
3648 start = alloc->start;
3649 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
3650 unsigned int data_rate, y_data_rate;
3651 uint16_t plane_blocks, y_plane_blocks = 0;
3652
3653 if (plane_id == PLANE_CURSOR)
3654 continue;
3655
3656 data_rate = plane_data_rate[plane_id];
3657
3658
3659
3660
3661
3662
3663 plane_blocks = minimum[plane_id];
3664 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
3665 total_data_rate);
3666
3667
3668 if (data_rate) {
3669 ddb->plane[pipe][plane_id].start = start;
3670 ddb->plane[pipe][plane_id].end = start + plane_blocks;
3671 }
3672
3673 start += plane_blocks;
3674
3675
3676
3677
3678 y_data_rate = plane_y_data_rate[plane_id];
3679
3680 y_plane_blocks = y_minimum[plane_id];
3681 y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
3682 total_data_rate);
3683
3684 if (y_data_rate) {
3685 ddb->y_plane[pipe][plane_id].start = start;
3686 ddb->y_plane[pipe][plane_id].end = start + y_plane_blocks;
3687 }
3688
3689 start += y_plane_blocks;
3690 }
3691
3692 return 0;
3693}
3694
3695
3696
3697
3698
3699
3700
3701static uint_fixed_16_16_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp,
3702 uint32_t latency)
3703{
3704 uint32_t wm_intermediate_val;
3705 uint_fixed_16_16_t ret;
3706
3707 if (latency == 0)
3708 return FP_16_16_MAX;
3709
3710 wm_intermediate_val = latency * pixel_rate * cpp;
3711 ret = fixed_16_16_div_round_up_u64(wm_intermediate_val, 1000 * 512);
3712 return ret;
3713}
3714
3715static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
3716 uint32_t pipe_htotal,
3717 uint32_t latency,
3718 uint_fixed_16_16_t plane_blocks_per_line)
3719{
3720 uint32_t wm_intermediate_val;
3721 uint_fixed_16_16_t ret;
3722
3723 if (latency == 0)
3724 return FP_16_16_MAX;
3725
3726 wm_intermediate_val = latency * pixel_rate;
3727 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
3728 pipe_htotal * 1000);
3729 ret = mul_u32_fixed_16_16(wm_intermediate_val, plane_blocks_per_line);
3730 return ret;
3731}
3732
3733static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
3734 struct intel_plane_state *pstate)
3735{
3736 uint64_t adjusted_pixel_rate;
3737 uint64_t downscale_amount;
3738 uint64_t pixel_rate;
3739
3740
3741 if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
3742 return 0;
3743
3744
3745
3746
3747
3748 adjusted_pixel_rate = cstate->pixel_rate;
3749 downscale_amount = skl_plane_downscale_amount(cstate, pstate);
3750
3751 pixel_rate = adjusted_pixel_rate * downscale_amount >> 16;
3752 WARN_ON(pixel_rate != clamp_t(uint32_t, pixel_rate, 0, ~0));
3753
3754 return pixel_rate;
3755}
3756
3757static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3758 struct intel_crtc_state *cstate,
3759 struct intel_plane_state *intel_pstate,
3760 uint16_t ddb_allocation,
3761 int level,
3762 uint16_t *out_blocks,
3763 uint8_t *out_lines,
3764 bool *enabled )
3765{
3766 struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
3767 struct drm_plane_state *pstate = &intel_pstate->base;
3768 struct drm_framebuffer *fb = pstate->fb;
3769 uint32_t latency = dev_priv->wm.skl_latency[level];
3770 uint_fixed_16_16_t method1, method2;
3771 uint_fixed_16_16_t plane_blocks_per_line;
3772 uint_fixed_16_16_t selected_result;
3773 uint32_t interm_pbpl;
3774 uint32_t plane_bytes_per_line;
3775 uint32_t res_blocks, res_lines;
3776 uint8_t cpp;
3777 uint32_t width = 0, height = 0;
3778 uint32_t plane_pixel_rate;
3779 uint_fixed_16_16_t y_tile_minimum;
3780 uint32_t y_min_scanlines;
3781 struct intel_atomic_state *state =
3782 to_intel_atomic_state(cstate->base.state);
3783 bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
3784 bool y_tiled, x_tiled;
3785
3786 if (latency == 0 ||
3787 !intel_wm_plane_visible(cstate, intel_pstate)) {
3788 *enabled = false;
3789 return 0;
3790 }
3791
3792 y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
3793 fb->modifier == I915_FORMAT_MOD_Yf_TILED;
3794 x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
3795
3796
3797 if (IS_KABYLAKE(dev_priv) && dev_priv->ipc_enabled)
3798 latency += 4;
3799
3800 if (apply_memory_bw_wa && x_tiled)
3801 latency += 15;
3802
3803 if (plane->id == PLANE_CURSOR) {
3804 width = intel_pstate->base.crtc_w;
3805 height = intel_pstate->base.crtc_h;
3806 } else {
3807
3808
3809
3810
3811
3812 width = drm_rect_width(&intel_pstate->base.src) >> 16;
3813 height = drm_rect_height(&intel_pstate->base.src) >> 16;
3814 }
3815
3816 cpp = fb->format->cpp[0];
3817 plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
3818
3819 if (drm_rotation_90_or_270(pstate->rotation)) {
3820 int cpp = (fb->format->format == DRM_FORMAT_NV12) ?
3821 fb->format->cpp[1] :
3822 fb->format->cpp[0];
3823
3824 switch (cpp) {
3825 case 1:
3826 y_min_scanlines = 16;
3827 break;
3828 case 2:
3829 y_min_scanlines = 8;
3830 break;
3831 case 4:
3832 y_min_scanlines = 4;
3833 break;
3834 default:
3835 MISSING_CASE(cpp);
3836 return -EINVAL;
3837 }
3838 } else {
3839 y_min_scanlines = 4;
3840 }
3841
3842 if (apply_memory_bw_wa)
3843 y_min_scanlines *= 2;
3844
3845 plane_bytes_per_line = width * cpp;
3846 if (y_tiled) {
3847 interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line *
3848 y_min_scanlines, 512);
3849 plane_blocks_per_line =
3850 fixed_16_16_div_round_up(interm_pbpl, y_min_scanlines);
3851 } else if (x_tiled) {
3852 interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512);
3853 plane_blocks_per_line = u32_to_fixed_16_16(interm_pbpl);
3854 } else {
3855 interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512) + 1;
3856 plane_blocks_per_line = u32_to_fixed_16_16(interm_pbpl);
3857 }
3858
3859 method1 = skl_wm_method1(plane_pixel_rate, cpp, latency);
3860 method2 = skl_wm_method2(plane_pixel_rate,
3861 cstate->base.adjusted_mode.crtc_htotal,
3862 latency,
3863 plane_blocks_per_line);
3864
3865 y_tile_minimum = mul_u32_fixed_16_16(y_min_scanlines,
3866 plane_blocks_per_line);
3867
3868 if (y_tiled) {
3869 selected_result = max_fixed_16_16(method2, y_tile_minimum);
3870 } else {
3871 if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) &&
3872 (plane_bytes_per_line / 512 < 1))
3873 selected_result = method2;
3874 else if ((ddb_allocation /
3875 fixed_16_16_to_u32_round_up(plane_blocks_per_line)) >= 1)
3876 selected_result = min_fixed_16_16(method1, method2);
3877 else
3878 selected_result = method1;
3879 }
3880
3881 res_blocks = fixed_16_16_to_u32_round_up(selected_result) + 1;
3882 res_lines = DIV_ROUND_UP(selected_result.val,
3883 plane_blocks_per_line.val);
3884
3885 if (level >= 1 && level <= 7) {
3886 if (y_tiled) {
3887 res_blocks += fixed_16_16_to_u32_round_up(y_tile_minimum);
3888 res_lines += y_min_scanlines;
3889 } else {
3890 res_blocks++;
3891 }
3892 }
3893
3894 if (res_blocks >= ddb_allocation || res_lines > 31) {
3895 *enabled = false;
3896
3897
3898
3899
3900
3901 if (level) {
3902 return 0;
3903 } else {
3904 struct drm_plane *plane = pstate->plane;
3905
3906 DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
3907 DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n",
3908 plane->base.id, plane->name,
3909 res_blocks, ddb_allocation, res_lines);
3910 return -EINVAL;
3911 }
3912 }
3913
3914 *out_blocks = res_blocks;
3915 *out_lines = res_lines;
3916 *enabled = true;
3917
3918 return 0;
3919}
3920
3921static int
3922skl_compute_wm_level(const struct drm_i915_private *dev_priv,
3923 struct skl_ddb_allocation *ddb,
3924 struct intel_crtc_state *cstate,
3925 struct intel_plane *intel_plane,
3926 int level,
3927 struct skl_wm_level *result)
3928{
3929 struct drm_atomic_state *state = cstate->base.state;
3930 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3931 struct drm_plane *plane = &intel_plane->base;
3932 struct intel_plane_state *intel_pstate = NULL;
3933 uint16_t ddb_blocks;
3934 enum pipe pipe = intel_crtc->pipe;
3935 int ret;
3936
3937 if (state)
3938 intel_pstate =
3939 intel_atomic_get_existing_plane_state(state,
3940 intel_plane);
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953 if (!intel_pstate)
3954 intel_pstate = to_intel_plane_state(plane->state);
3955
3956 WARN_ON(!intel_pstate->base.fb);
3957
3958 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][intel_plane->id]);
3959
3960 ret = skl_compute_plane_wm(dev_priv,
3961 cstate,
3962 intel_pstate,
3963 ddb_blocks,
3964 level,
3965 &result->plane_res_b,
3966 &result->plane_res_l,
3967 &result->plane_en);
3968 if (ret)
3969 return ret;
3970
3971 return 0;
3972}
3973
3974static uint32_t
3975skl_compute_linetime_wm(struct intel_crtc_state *cstate)
3976{
3977 struct drm_atomic_state *state = cstate->base.state;
3978 struct drm_i915_private *dev_priv = to_i915(state->dev);
3979 uint32_t pixel_rate;
3980 uint32_t linetime_wm;
3981
3982 if (!cstate->base.active)
3983 return 0;
3984
3985 pixel_rate = cstate->pixel_rate;
3986
3987 if (WARN_ON(pixel_rate == 0))
3988 return 0;
3989
3990 linetime_wm = DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal *
3991 1000, pixel_rate);
3992
3993
3994 if (IS_BROXTON(dev_priv) && dev_priv->ipc_enabled)
3995 linetime_wm = DIV_ROUND_UP(linetime_wm, 2);
3996
3997 return linetime_wm;
3998}
3999
4000static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
4001 struct skl_wm_level *trans_wm )
4002{
4003 if (!cstate->base.active)
4004 return;
4005
4006
4007 trans_wm->plane_en = false;
4008}
4009
4010static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
4011 struct skl_ddb_allocation *ddb,
4012 struct skl_pipe_wm *pipe_wm)
4013{
4014 struct drm_device *dev = cstate->base.crtc->dev;
4015 const struct drm_i915_private *dev_priv = to_i915(dev);
4016 struct intel_plane *intel_plane;
4017 struct skl_plane_wm *wm;
4018 int level, max_level = ilk_wm_max_level(dev_priv);
4019 int ret;
4020
4021
4022
4023
4024
4025 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4026
4027 for_each_intel_plane_mask(&dev_priv->drm,
4028 intel_plane,
4029 cstate->base.plane_mask) {
4030 wm = &pipe_wm->planes[intel_plane->id];
4031
4032 for (level = 0; level <= max_level; level++) {
4033 ret = skl_compute_wm_level(dev_priv, ddb, cstate,
4034 intel_plane, level,
4035 &wm->wm[level]);
4036 if (ret)
4037 return ret;
4038 }
4039 skl_compute_transition_wm(cstate, &wm->trans_wm);
4040 }
4041 pipe_wm->linetime = skl_compute_linetime_wm(cstate);
4042
4043 return 0;
4044}
4045
4046static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
4047 i915_reg_t reg,
4048 const struct skl_ddb_entry *entry)
4049{
4050 if (entry->end)
4051 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
4052 else
4053 I915_WRITE(reg, 0);
4054}
4055
4056static void skl_write_wm_level(struct drm_i915_private *dev_priv,
4057 i915_reg_t reg,
4058 const struct skl_wm_level *level)
4059{
4060 uint32_t val = 0;
4061
4062 if (level->plane_en) {
4063 val |= PLANE_WM_EN;
4064 val |= level->plane_res_b;
4065 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
4066 }
4067
4068 I915_WRITE(reg, val);
4069}
4070
4071static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
4072 const struct skl_plane_wm *wm,
4073 const struct skl_ddb_allocation *ddb,
4074 enum plane_id plane_id)
4075{
4076 struct drm_crtc *crtc = &intel_crtc->base;
4077 struct drm_device *dev = crtc->dev;
4078 struct drm_i915_private *dev_priv = to_i915(dev);
4079 int level, max_level = ilk_wm_max_level(dev_priv);
4080 enum pipe pipe = intel_crtc->pipe;
4081
4082 for (level = 0; level <= max_level; level++) {
4083 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
4084 &wm->wm[level]);
4085 }
4086 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
4087 &wm->trans_wm);
4088
4089 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
4090 &ddb->plane[pipe][plane_id]);
4091 skl_ddb_entry_write(dev_priv, PLANE_NV12_BUF_CFG(pipe, plane_id),
4092 &ddb->y_plane[pipe][plane_id]);
4093}
4094
4095static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
4096 const struct skl_plane_wm *wm,
4097 const struct skl_ddb_allocation *ddb)
4098{
4099 struct drm_crtc *crtc = &intel_crtc->base;
4100 struct drm_device *dev = crtc->dev;
4101 struct drm_i915_private *dev_priv = to_i915(dev);
4102 int level, max_level = ilk_wm_max_level(dev_priv);
4103 enum pipe pipe = intel_crtc->pipe;
4104
4105 for (level = 0; level <= max_level; level++) {
4106 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
4107 &wm->wm[level]);
4108 }
4109 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
4110
4111 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
4112 &ddb->plane[pipe][PLANE_CURSOR]);
4113}
4114
4115bool skl_wm_level_equals(const struct skl_wm_level *l1,
4116 const struct skl_wm_level *l2)
4117{
4118 if (l1->plane_en != l2->plane_en)
4119 return false;
4120
4121
4122 if (!l1->plane_en)
4123 return true;
4124
4125 return (l1->plane_res_l == l2->plane_res_l &&
4126 l1->plane_res_b == l2->plane_res_b);
4127}
4128
4129static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
4130 const struct skl_ddb_entry *b)
4131{
4132 return a->start < b->end && b->start < a->end;
4133}
4134
4135bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries,
4136 const struct skl_ddb_entry *ddb,
4137 int ignore)
4138{
4139 int i;
4140
4141 for (i = 0; i < I915_MAX_PIPES; i++)
4142 if (i != ignore && entries[i] &&
4143 skl_ddb_entries_overlap(ddb, entries[i]))
4144 return true;
4145
4146 return false;
4147}
4148
4149static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
4150 const struct skl_pipe_wm *old_pipe_wm,
4151 struct skl_pipe_wm *pipe_wm,
4152 struct skl_ddb_allocation *ddb,
4153 bool *changed )
4154{
4155 struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
4156 int ret;
4157
4158 ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
4159 if (ret)
4160 return ret;
4161
4162 if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
4163 *changed = false;
4164 else
4165 *changed = true;
4166
4167 return 0;
4168}
4169
4170static uint32_t
4171pipes_modified(struct drm_atomic_state *state)
4172{
4173 struct drm_crtc *crtc;
4174 struct drm_crtc_state *cstate;
4175 uint32_t i, ret = 0;
4176
4177 for_each_new_crtc_in_state(state, crtc, cstate, i)
4178 ret |= drm_crtc_mask(crtc);
4179
4180 return ret;
4181}
4182
4183static int
4184skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
4185{
4186 struct drm_atomic_state *state = cstate->base.state;
4187 struct drm_device *dev = state->dev;
4188 struct drm_crtc *crtc = cstate->base.crtc;
4189 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4190 struct drm_i915_private *dev_priv = to_i915(dev);
4191 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4192 struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
4193 struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
4194 struct drm_plane_state *plane_state;
4195 struct drm_plane *plane;
4196 enum pipe pipe = intel_crtc->pipe;
4197
4198 WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc));
4199
4200 drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
4201 enum plane_id plane_id = to_intel_plane(plane)->id;
4202
4203 if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
4204 &new_ddb->plane[pipe][plane_id]) &&
4205 skl_ddb_entry_equal(&cur_ddb->y_plane[pipe][plane_id],
4206 &new_ddb->y_plane[pipe][plane_id]))
4207 continue;
4208
4209 plane_state = drm_atomic_get_plane_state(state, plane);
4210 if (IS_ERR(plane_state))
4211 return PTR_ERR(plane_state);
4212 }
4213
4214 return 0;
4215}
4216
4217static int
4218skl_compute_ddb(struct drm_atomic_state *state)
4219{
4220 struct drm_device *dev = state->dev;
4221 struct drm_i915_private *dev_priv = to_i915(dev);
4222 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4223 struct intel_crtc *intel_crtc;
4224 struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
4225 uint32_t realloc_pipes = pipes_modified(state);
4226 int ret;
4227
4228
4229
4230
4231
4232
4233
4234 if (dev_priv->wm.distrust_bios_wm) {
4235 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
4236 state->acquire_ctx);
4237 if (ret)
4238 return ret;
4239
4240 intel_state->active_pipe_changes = ~0;
4241
4242
4243
4244
4245
4246
4247
4248 if (!intel_state->modeset)
4249 intel_state->active_crtcs = dev_priv->active_crtcs;
4250 }
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265 if (intel_state->active_pipe_changes) {
4266 realloc_pipes = ~0;
4267 intel_state->wm_results.dirty_pipes = ~0;
4268 }
4269
4270
4271
4272
4273
4274 memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
4275
4276 for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
4277 struct intel_crtc_state *cstate;
4278
4279 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
4280 if (IS_ERR(cstate))
4281 return PTR_ERR(cstate);
4282
4283 ret = skl_allocate_pipe_ddb(cstate, ddb);
4284 if (ret)
4285 return ret;
4286
4287 ret = skl_ddb_add_affected_planes(cstate);
4288 if (ret)
4289 return ret;
4290 }
4291
4292 return 0;
4293}
4294
4295static void
4296skl_copy_wm_for_pipe(struct skl_wm_values *dst,
4297 struct skl_wm_values *src,
4298 enum pipe pipe)
4299{
4300 memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe],
4301 sizeof(dst->ddb.y_plane[pipe]));
4302 memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
4303 sizeof(dst->ddb.plane[pipe]));
4304}
4305
4306static void
4307skl_print_wm_changes(const struct drm_atomic_state *state)
4308{
4309 const struct drm_device *dev = state->dev;
4310 const struct drm_i915_private *dev_priv = to_i915(dev);
4311 const struct intel_atomic_state *intel_state =
4312 to_intel_atomic_state(state);
4313 const struct drm_crtc *crtc;
4314 const struct drm_crtc_state *cstate;
4315 const struct intel_plane *intel_plane;
4316 const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
4317 const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
4318 int i;
4319
4320 for_each_new_crtc_in_state(state, crtc, cstate, i) {
4321 const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4322 enum pipe pipe = intel_crtc->pipe;
4323
4324 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
4325 enum plane_id plane_id = intel_plane->id;
4326 const struct skl_ddb_entry *old, *new;
4327
4328 old = &old_ddb->plane[pipe][plane_id];
4329 new = &new_ddb->plane[pipe][plane_id];
4330
4331 if (skl_ddb_entry_equal(old, new))
4332 continue;
4333
4334 DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
4335 intel_plane->base.base.id,
4336 intel_plane->base.name,
4337 old->start, old->end,
4338 new->start, new->end);
4339 }
4340 }
4341}
4342
4343static int
4344skl_compute_wm(struct drm_atomic_state *state)
4345{
4346 struct drm_crtc *crtc;
4347 struct drm_crtc_state *cstate;
4348 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4349 struct skl_wm_values *results = &intel_state->wm_results;
4350 struct drm_device *dev = state->dev;
4351 struct skl_pipe_wm *pipe_wm;
4352 bool changed = false;
4353 int ret, i;
4354
4355
4356
4357
4358
4359 if (to_i915(dev)->wm.distrust_bios_wm)
4360 changed = true;
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370 for_each_new_crtc_in_state(state, crtc, cstate, i)
4371 changed = true;
4372
4373 if (!changed)
4374 return 0;
4375
4376
4377 results->dirty_pipes = 0;
4378
4379 ret = skl_compute_ddb(state);
4380 if (ret)
4381 return ret;
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393 for_each_new_crtc_in_state(state, crtc, cstate, i) {
4394 struct intel_crtc_state *intel_cstate =
4395 to_intel_crtc_state(cstate);
4396 const struct skl_pipe_wm *old_pipe_wm =
4397 &to_intel_crtc_state(crtc->state)->wm.skl.optimal;
4398
4399 pipe_wm = &intel_cstate->wm.skl.optimal;
4400 ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
4401 &results->ddb, &changed);
4402 if (ret)
4403 return ret;
4404
4405 if (changed)
4406 results->dirty_pipes |= drm_crtc_mask(crtc);
4407
4408 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
4409
4410 continue;
4411
4412 intel_cstate->update_wm_pre = true;
4413 }
4414
4415 skl_print_wm_changes(state);
4416
4417 return 0;
4418}
4419
4420static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
4421 struct intel_crtc_state *cstate)
4422{
4423 struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
4424 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4425 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
4426 const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
4427 enum pipe pipe = crtc->pipe;
4428 enum plane_id plane_id;
4429
4430 if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
4431 return;
4432
4433 I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
4434
4435 for_each_plane_id_on_crtc(crtc, plane_id) {
4436 if (plane_id != PLANE_CURSOR)
4437 skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id],
4438 ddb, plane_id);
4439 else
4440 skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id],
4441 ddb);
4442 }
4443}
4444
4445static void skl_initial_wm(struct intel_atomic_state *state,
4446 struct intel_crtc_state *cstate)
4447{
4448 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4449 struct drm_device *dev = intel_crtc->base.dev;
4450 struct drm_i915_private *dev_priv = to_i915(dev);
4451 struct skl_wm_values *results = &state->wm_results;
4452 struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
4453 enum pipe pipe = intel_crtc->pipe;
4454
4455 if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
4456 return;
4457
4458 mutex_lock(&dev_priv->wm.wm_mutex);
4459
4460 if (cstate->base.active_changed)
4461 skl_atomic_update_crtc_wm(state, cstate);
4462
4463 skl_copy_wm_for_pipe(hw_vals, results, pipe);
4464
4465 mutex_unlock(&dev_priv->wm.wm_mutex);
4466}
4467
4468static void ilk_compute_wm_config(struct drm_device *dev,
4469 struct intel_wm_config *config)
4470{
4471 struct intel_crtc *crtc;
4472
4473
4474 for_each_intel_crtc(dev, crtc) {
4475 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
4476
4477 if (!wm->pipe_enabled)
4478 continue;
4479
4480 config->sprites_enabled |= wm->sprites_enabled;
4481 config->sprites_scaled |= wm->sprites_scaled;
4482 config->num_pipes_active++;
4483 }
4484}
4485
4486static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
4487{
4488 struct drm_device *dev = &dev_priv->drm;
4489 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
4490 struct ilk_wm_maximums max;
4491 struct intel_wm_config config = {};
4492 struct ilk_wm_values results = {};
4493 enum intel_ddb_partitioning partitioning;
4494
4495 ilk_compute_wm_config(dev, &config);
4496
4497 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
4498 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
4499
4500
4501 if (INTEL_GEN(dev_priv) >= 7 &&
4502 config.num_pipes_active == 1 && config.sprites_enabled) {
4503 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
4504 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
4505
4506 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4507 } else {
4508 best_lp_wm = &lp_wm_1_2;
4509 }
4510
4511 partitioning = (best_lp_wm == &lp_wm_1_2) ?
4512 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
4513
4514 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
4515
4516 ilk_write_wm_values(dev_priv, &results);
4517}
4518
4519static void ilk_initial_watermarks(struct intel_atomic_state *state,
4520 struct intel_crtc_state *cstate)
4521{
4522 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
4523 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4524
4525 mutex_lock(&dev_priv->wm.wm_mutex);
4526 intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
4527 ilk_program_watermarks(dev_priv);
4528 mutex_unlock(&dev_priv->wm.wm_mutex);
4529}
4530
4531static void ilk_optimize_watermarks(struct intel_atomic_state *state,
4532 struct intel_crtc_state *cstate)
4533{
4534 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
4535 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4536
4537 mutex_lock(&dev_priv->wm.wm_mutex);
4538 if (cstate->wm.need_postvbl_update) {
4539 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
4540 ilk_program_watermarks(dev_priv);
4541 }
4542 mutex_unlock(&dev_priv->wm.wm_mutex);
4543}
4544
4545static inline void skl_wm_level_from_reg_val(uint32_t val,
4546 struct skl_wm_level *level)
4547{
4548 level->plane_en = val & PLANE_WM_EN;
4549 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
4550 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
4551 PLANE_WM_LINES_MASK;
4552}
4553
4554void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
4555 struct skl_pipe_wm *out)
4556{
4557 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
4558 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4559 enum pipe pipe = intel_crtc->pipe;
4560 int level, max_level;
4561 enum plane_id plane_id;
4562 uint32_t val;
4563
4564 max_level = ilk_wm_max_level(dev_priv);
4565
4566 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4567 struct skl_plane_wm *wm = &out->planes[plane_id];
4568
4569 for (level = 0; level <= max_level; level++) {
4570 if (plane_id != PLANE_CURSOR)
4571 val = I915_READ(PLANE_WM(pipe, plane_id, level));
4572 else
4573 val = I915_READ(CUR_WM(pipe, level));
4574
4575 skl_wm_level_from_reg_val(val, &wm->wm[level]);
4576 }
4577
4578 if (plane_id != PLANE_CURSOR)
4579 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
4580 else
4581 val = I915_READ(CUR_WM_TRANS(pipe));
4582
4583 skl_wm_level_from_reg_val(val, &wm->trans_wm);
4584 }
4585
4586 if (!intel_crtc->active)
4587 return;
4588
4589 out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
4590}
4591
4592void skl_wm_get_hw_state(struct drm_device *dev)
4593{
4594 struct drm_i915_private *dev_priv = to_i915(dev);
4595 struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
4596 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
4597 struct drm_crtc *crtc;
4598 struct intel_crtc *intel_crtc;
4599 struct intel_crtc_state *cstate;
4600
4601 skl_ddb_get_hw_state(dev_priv, ddb);
4602 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
4603 intel_crtc = to_intel_crtc(crtc);
4604 cstate = to_intel_crtc_state(crtc->state);
4605
4606 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
4607
4608 if (intel_crtc->active)
4609 hw->dirty_pipes |= drm_crtc_mask(crtc);
4610 }
4611
4612 if (dev_priv->active_crtcs) {
4613
4614 dev_priv->wm.distrust_bios_wm = true;
4615 } else {
4616
4617 memset(ddb, 0, sizeof(*ddb));
4618 }
4619}
4620
4621static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
4622{
4623 struct drm_device *dev = crtc->dev;
4624 struct drm_i915_private *dev_priv = to_i915(dev);
4625 struct ilk_wm_values *hw = &dev_priv->wm.hw;
4626 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4627 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
4628 struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
4629 enum pipe pipe = intel_crtc->pipe;
4630 static const i915_reg_t wm0_pipe_reg[] = {
4631 [PIPE_A] = WM0_PIPEA_ILK,
4632 [PIPE_B] = WM0_PIPEB_ILK,
4633 [PIPE_C] = WM0_PIPEC_IVB,
4634 };
4635
4636 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
4637 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4638 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
4639
4640 memset(active, 0, sizeof(*active));
4641
4642 active->pipe_enabled = intel_crtc->active;
4643
4644 if (active->pipe_enabled) {
4645 u32 tmp = hw->wm_pipe[pipe];
4646
4647
4648
4649
4650
4651
4652
4653 active->wm[0].enable = true;
4654 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
4655 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
4656 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
4657 active->linetime = hw->wm_linetime[pipe];
4658 } else {
4659 int level, max_level = ilk_wm_max_level(dev_priv);
4660
4661
4662
4663
4664
4665
4666 for (level = 0; level <= max_level; level++)
4667 active->wm[level].enable = true;
4668 }
4669
4670 intel_crtc->wm.active.ilk = *active;
4671}
4672
4673#define _FW_WM(value, plane) \
4674 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
4675#define _FW_WM_VLV(value, plane) \
4676 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
4677
4678static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
4679 struct vlv_wm_values *wm)
4680{
4681 enum pipe pipe;
4682 uint32_t tmp;
4683
4684 for_each_pipe(dev_priv, pipe) {
4685 tmp = I915_READ(VLV_DDL(pipe));
4686
4687 wm->ddl[pipe].plane[PLANE_PRIMARY] =
4688 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4689 wm->ddl[pipe].plane[PLANE_CURSOR] =
4690 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4691 wm->ddl[pipe].plane[PLANE_SPRITE0] =
4692 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4693 wm->ddl[pipe].plane[PLANE_SPRITE1] =
4694 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
4695 }
4696
4697 tmp = I915_READ(DSPFW1);
4698 wm->sr.plane = _FW_WM(tmp, SR);
4699 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
4700 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
4701 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
4702
4703 tmp = I915_READ(DSPFW2);
4704 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
4705 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
4706 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
4707
4708 tmp = I915_READ(DSPFW3);
4709 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
4710
4711 if (IS_CHERRYVIEW(dev_priv)) {
4712 tmp = I915_READ(DSPFW7_CHV);
4713 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
4714 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
4715
4716 tmp = I915_READ(DSPFW8_CHV);
4717 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
4718 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
4719
4720 tmp = I915_READ(DSPFW9_CHV);
4721 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
4722 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
4723
4724 tmp = I915_READ(DSPHOWM);
4725 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
4726 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
4727 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
4728 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
4729 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
4730 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
4731 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
4732 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
4733 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
4734 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
4735 } else {
4736 tmp = I915_READ(DSPFW7);
4737 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
4738 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
4739
4740 tmp = I915_READ(DSPHOWM);
4741 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
4742 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
4743 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
4744 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
4745 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
4746 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
4747 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
4748 }
4749}
4750
4751#undef _FW_WM
4752#undef _FW_WM_VLV
4753
4754void vlv_wm_get_hw_state(struct drm_device *dev)
4755{
4756 struct drm_i915_private *dev_priv = to_i915(dev);
4757 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
4758 struct intel_crtc *crtc;
4759 u32 val;
4760
4761 vlv_read_wm_values(dev_priv, wm);
4762
4763 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
4764 wm->level = VLV_WM_LEVEL_PM2;
4765
4766 if (IS_CHERRYVIEW(dev_priv)) {
4767 mutex_lock(&dev_priv->rps.hw_lock);
4768
4769 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
4770 if (val & DSP_MAXFIFO_PM5_ENABLE)
4771 wm->level = VLV_WM_LEVEL_PM5;
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
4783 val |= FORCE_DDR_FREQ_REQ_ACK;
4784 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
4785
4786 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
4787 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
4788 DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
4789 "assuming DDR DVFS is disabled\n");
4790 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
4791 } else {
4792 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
4793 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
4794 wm->level = VLV_WM_LEVEL_DDR_DVFS;
4795 }
4796
4797 mutex_unlock(&dev_priv->rps.hw_lock);
4798 }
4799
4800 for_each_intel_crtc(dev, crtc) {
4801 struct intel_crtc_state *crtc_state =
4802 to_intel_crtc_state(crtc->base.state);
4803 struct vlv_wm_state *active = &crtc->wm.active.vlv;
4804 const struct vlv_fifo_state *fifo_state =
4805 &crtc_state->wm.vlv.fifo_state;
4806 enum pipe pipe = crtc->pipe;
4807 enum plane_id plane_id;
4808 int level;
4809
4810 vlv_get_fifo_size(crtc_state);
4811
4812 active->num_levels = wm->level + 1;
4813 active->cxsr = wm->cxsr;
4814
4815 for (level = 0; level < active->num_levels; level++) {
4816 struct vlv_pipe_wm *raw =
4817 &crtc_state->wm.vlv.raw[level];
4818
4819 active->sr[level].plane = wm->sr.plane;
4820 active->sr[level].cursor = wm->sr.cursor;
4821
4822 for_each_plane_id_on_crtc(crtc, plane_id) {
4823 active->wm[level].plane[plane_id] =
4824 wm->pipe[pipe].plane[plane_id];
4825
4826 raw->plane[plane_id] =
4827 vlv_invert_wm_value(active->wm[level].plane[plane_id],
4828 fifo_state->plane[plane_id]);
4829 }
4830 }
4831
4832 for_each_plane_id_on_crtc(crtc, plane_id)
4833 vlv_raw_plane_wm_set(crtc_state, level,
4834 plane_id, USHRT_MAX);
4835 vlv_invalidate_wms(crtc, active, level);
4836
4837 crtc_state->wm.vlv.optimal = *active;
4838 crtc_state->wm.vlv.intermediate = *active;
4839
4840 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
4841 pipe_name(pipe),
4842 wm->pipe[pipe].plane[PLANE_PRIMARY],
4843 wm->pipe[pipe].plane[PLANE_CURSOR],
4844 wm->pipe[pipe].plane[PLANE_SPRITE0],
4845 wm->pipe[pipe].plane[PLANE_SPRITE1]);
4846 }
4847
4848 DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4849 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4850}
4851
4852void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
4853{
4854 struct intel_plane *plane;
4855 struct intel_crtc *crtc;
4856
4857 mutex_lock(&dev_priv->wm.wm_mutex);
4858
4859 for_each_intel_plane(&dev_priv->drm, plane) {
4860 struct intel_crtc *crtc =
4861 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
4862 struct intel_crtc_state *crtc_state =
4863 to_intel_crtc_state(crtc->base.state);
4864 struct intel_plane_state *plane_state =
4865 to_intel_plane_state(plane->base.state);
4866 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
4867 const struct vlv_fifo_state *fifo_state =
4868 &crtc_state->wm.vlv.fifo_state;
4869 enum plane_id plane_id = plane->id;
4870 int level;
4871
4872 if (plane_state->base.visible)
4873 continue;
4874
4875 for (level = 0; level < wm_state->num_levels; level++) {
4876 struct vlv_pipe_wm *raw =
4877 &crtc_state->wm.vlv.raw[level];
4878
4879 raw->plane[plane_id] = 0;
4880
4881 wm_state->wm[level].plane[plane_id] =
4882 vlv_invert_wm_value(raw->plane[plane_id],
4883 fifo_state->plane[plane_id]);
4884 }
4885 }
4886
4887 for_each_intel_crtc(&dev_priv->drm, crtc) {
4888 struct intel_crtc_state *crtc_state =
4889 to_intel_crtc_state(crtc->base.state);
4890
4891 crtc_state->wm.vlv.intermediate =
4892 crtc_state->wm.vlv.optimal;
4893 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4894 }
4895
4896 vlv_program_watermarks(dev_priv);
4897
4898 mutex_unlock(&dev_priv->wm.wm_mutex);
4899}
4900
4901void ilk_wm_get_hw_state(struct drm_device *dev)
4902{
4903 struct drm_i915_private *dev_priv = to_i915(dev);
4904 struct ilk_wm_values *hw = &dev_priv->wm.hw;
4905 struct drm_crtc *crtc;
4906
4907 for_each_crtc(dev, crtc)
4908 ilk_pipe_wm_get_hw_state(crtc);
4909
4910 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
4911 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
4912 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
4913
4914 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
4915 if (INTEL_GEN(dev_priv) >= 7) {
4916 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
4917 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
4918 }
4919
4920 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4921 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4922 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4923 else if (IS_IVYBRIDGE(dev_priv))
4924 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4925 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4926
4927 hw->enable_fbc_wm =
4928 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4929}
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963void intel_update_watermarks(struct intel_crtc *crtc)
4964{
4965 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4966
4967 if (dev_priv->display.update_wm)
4968 dev_priv->display.update_wm(crtc);
4969}
4970
4971
4972
4973
4974DEFINE_SPINLOCK(mchdev_lock);
4975
4976
4977
4978static struct drm_i915_private *i915_mch_dev;
4979
4980bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
4981{
4982 u16 rgvswctl;
4983
4984 lockdep_assert_held(&mchdev_lock);
4985
4986 rgvswctl = I915_READ16(MEMSWCTL);
4987 if (rgvswctl & MEMCTL_CMD_STS) {
4988 DRM_DEBUG("gpu busy, RCS change rejected\n");
4989 return false;
4990 }
4991
4992 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
4993 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
4994 I915_WRITE16(MEMSWCTL, rgvswctl);
4995 POSTING_READ16(MEMSWCTL);
4996
4997 rgvswctl |= MEMCTL_CMD_STS;
4998 I915_WRITE16(MEMSWCTL, rgvswctl);
4999
5000 return true;
5001}
5002
5003static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
5004{
5005 u32 rgvmodectl;
5006 u8 fmax, fmin, fstart, vstart;
5007
5008 spin_lock_irq(&mchdev_lock);
5009
5010 rgvmodectl = I915_READ(MEMMODECTL);
5011
5012
5013 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
5014 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
5015
5016
5017 I915_WRITE(RCUPEI, 100000);
5018 I915_WRITE(RCDNEI, 100000);
5019
5020
5021 I915_WRITE(RCBMAXAVG, 90000);
5022 I915_WRITE(RCBMINAVG, 80000);
5023
5024 I915_WRITE(MEMIHYST, 1);
5025
5026
5027 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
5028 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
5029 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
5030 MEMMODE_FSTART_SHIFT;
5031
5032 vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
5033 PXVFREQ_PX_SHIFT;
5034
5035 dev_priv->ips.fmax = fmax;
5036 dev_priv->ips.fstart = fstart;
5037
5038 dev_priv->ips.max_delay = fstart;
5039 dev_priv->ips.min_delay = fmin;
5040 dev_priv->ips.cur_delay = fstart;
5041
5042 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
5043 fmax, fmin, fstart);
5044
5045 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
5046
5047
5048
5049
5050
5051 I915_WRITE(VIDSTART, vstart);
5052 POSTING_READ(VIDSTART);
5053
5054 rgvmodectl |= MEMMODE_SWMODE_EN;
5055 I915_WRITE(MEMMODECTL, rgvmodectl);
5056
5057 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
5058 DRM_ERROR("stuck trying to change perf mode\n");
5059 mdelay(1);
5060
5061 ironlake_set_drps(dev_priv, fstart);
5062
5063 dev_priv->ips.last_count1 = I915_READ(DMIEC) +
5064 I915_READ(DDREC) + I915_READ(CSIEC);
5065 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
5066 dev_priv->ips.last_count2 = I915_READ(GFXEC);
5067 dev_priv->ips.last_time2 = ktime_get_raw_ns();
5068
5069 spin_unlock_irq(&mchdev_lock);
5070}
5071
5072static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
5073{
5074 u16 rgvswctl;
5075
5076 spin_lock_irq(&mchdev_lock);
5077
5078 rgvswctl = I915_READ16(MEMSWCTL);
5079
5080
5081 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
5082 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
5083 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
5084 I915_WRITE(DEIIR, DE_PCU_EVENT);
5085 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
5086
5087
5088 ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
5089 mdelay(1);
5090 rgvswctl |= MEMCTL_CMD_STS;
5091 I915_WRITE(MEMSWCTL, rgvswctl);
5092 mdelay(1);
5093
5094 spin_unlock_irq(&mchdev_lock);
5095}
5096
5097
5098
5099
5100
5101
5102static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
5103{
5104 u32 limits;
5105
5106
5107
5108
5109
5110
5111
5112 if (IS_GEN9(dev_priv)) {
5113 limits = (dev_priv->rps.max_freq_softlimit) << 23;
5114 if (val <= dev_priv->rps.min_freq_softlimit)
5115 limits |= (dev_priv->rps.min_freq_softlimit) << 14;
5116 } else {
5117 limits = dev_priv->rps.max_freq_softlimit << 24;
5118 if (val <= dev_priv->rps.min_freq_softlimit)
5119 limits |= dev_priv->rps.min_freq_softlimit << 16;
5120 }
5121
5122 return limits;
5123}
5124
5125static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
5126{
5127 int new_power;
5128 u32 threshold_up = 0, threshold_down = 0;
5129 u32 ei_up = 0, ei_down = 0;
5130
5131 new_power = dev_priv->rps.power;
5132 switch (dev_priv->rps.power) {
5133 case LOW_POWER:
5134 if (val > dev_priv->rps.efficient_freq + 1 &&
5135 val > dev_priv->rps.cur_freq)
5136 new_power = BETWEEN;
5137 break;
5138
5139 case BETWEEN:
5140 if (val <= dev_priv->rps.efficient_freq &&
5141 val < dev_priv->rps.cur_freq)
5142 new_power = LOW_POWER;
5143 else if (val >= dev_priv->rps.rp0_freq &&
5144 val > dev_priv->rps.cur_freq)
5145 new_power = HIGH_POWER;
5146 break;
5147
5148 case HIGH_POWER:
5149 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 &&
5150 val < dev_priv->rps.cur_freq)
5151 new_power = BETWEEN;
5152 break;
5153 }
5154
5155 if (val <= dev_priv->rps.min_freq_softlimit)
5156 new_power = LOW_POWER;
5157 if (val >= dev_priv->rps.max_freq_softlimit)
5158 new_power = HIGH_POWER;
5159 if (new_power == dev_priv->rps.power)
5160 return;
5161
5162
5163 switch (new_power) {
5164 case LOW_POWER:
5165
5166 ei_up = 16000;
5167 threshold_up = 95;
5168
5169
5170 ei_down = 32000;
5171 threshold_down = 85;
5172 break;
5173
5174 case BETWEEN:
5175
5176 ei_up = 13000;
5177 threshold_up = 90;
5178
5179
5180 ei_down = 32000;
5181 threshold_down = 75;
5182 break;
5183
5184 case HIGH_POWER:
5185
5186 ei_up = 10000;
5187 threshold_up = 85;
5188
5189
5190 ei_down = 32000;
5191 threshold_down = 60;
5192 break;
5193 }
5194
5195
5196
5197
5198 if (IS_VALLEYVIEW(dev_priv))
5199 goto skip_hw_write;
5200
5201 I915_WRITE(GEN6_RP_UP_EI,
5202 GT_INTERVAL_FROM_US(dev_priv, ei_up));
5203 I915_WRITE(GEN6_RP_UP_THRESHOLD,
5204 GT_INTERVAL_FROM_US(dev_priv,
5205 ei_up * threshold_up / 100));
5206
5207 I915_WRITE(GEN6_RP_DOWN_EI,
5208 GT_INTERVAL_FROM_US(dev_priv, ei_down));
5209 I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
5210 GT_INTERVAL_FROM_US(dev_priv,
5211 ei_down * threshold_down / 100));
5212
5213 I915_WRITE(GEN6_RP_CONTROL,
5214 GEN6_RP_MEDIA_TURBO |
5215 GEN6_RP_MEDIA_HW_NORMAL_MODE |
5216 GEN6_RP_MEDIA_IS_GFX |
5217 GEN6_RP_ENABLE |
5218 GEN6_RP_UP_BUSY_AVG |
5219 GEN6_RP_DOWN_IDLE_AVG);
5220
5221skip_hw_write:
5222 dev_priv->rps.power = new_power;
5223 dev_priv->rps.up_threshold = threshold_up;
5224 dev_priv->rps.down_threshold = threshold_down;
5225 dev_priv->rps.last_adj = 0;
5226}
5227
5228static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
5229{
5230 u32 mask = 0;
5231
5232
5233 if (val > dev_priv->rps.min_freq_softlimit)
5234 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
5235 if (val < dev_priv->rps.max_freq_softlimit)
5236 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
5237
5238 mask &= dev_priv->pm_rps_events;
5239
5240 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
5241}
5242
5243
5244
5245
5246static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
5247{
5248
5249
5250
5251 if (val != dev_priv->rps.cur_freq) {
5252 gen6_set_rps_thresholds(dev_priv, val);
5253
5254 if (IS_GEN9(dev_priv))
5255 I915_WRITE(GEN6_RPNSWREQ,
5256 GEN9_FREQUENCY(val));
5257 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5258 I915_WRITE(GEN6_RPNSWREQ,
5259 HSW_FREQUENCY(val));
5260 else
5261 I915_WRITE(GEN6_RPNSWREQ,
5262 GEN6_FREQUENCY(val) |
5263 GEN6_OFFSET(0) |
5264 GEN6_AGGRESSIVE_TURBO);
5265 }
5266
5267
5268
5269
5270 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
5271 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5272
5273 dev_priv->rps.cur_freq = val;
5274 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
5275
5276 return 0;
5277}
5278
5279static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
5280{
5281 int err;
5282
5283 if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
5284 "Odd GPU freq value\n"))
5285 val &= ~1;
5286
5287 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5288
5289 if (val != dev_priv->rps.cur_freq) {
5290 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
5291 if (err)
5292 return err;
5293
5294 gen6_set_rps_thresholds(dev_priv, val);
5295 }
5296
5297 dev_priv->rps.cur_freq = val;
5298 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
5299
5300 return 0;
5301}
5302
5303
5304
5305
5306
5307
5308
5309
5310static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
5311{
5312 u32 val = dev_priv->rps.idle_freq;
5313 int err;
5314
5315 if (dev_priv->rps.cur_freq <= val)
5316 return;
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
5331 err = valleyview_set_rps(dev_priv, val);
5332 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
5333
5334 if (err)
5335 DRM_ERROR("Failed to set RPS for idle\n");
5336}
5337
5338void gen6_rps_busy(struct drm_i915_private *dev_priv)
5339{
5340 mutex_lock(&dev_priv->rps.hw_lock);
5341 if (dev_priv->rps.enabled) {
5342 u8 freq;
5343
5344 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
5345 gen6_rps_reset_ei(dev_priv);
5346 I915_WRITE(GEN6_PMINTRMSK,
5347 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
5348
5349 gen6_enable_rps_interrupts(dev_priv);
5350
5351
5352
5353
5354 freq = max(dev_priv->rps.cur_freq,
5355 dev_priv->rps.efficient_freq);
5356
5357 if (intel_set_rps(dev_priv,
5358 clamp(freq,
5359 dev_priv->rps.min_freq_softlimit,
5360 dev_priv->rps.max_freq_softlimit)))
5361 DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
5362 }
5363 mutex_unlock(&dev_priv->rps.hw_lock);
5364}
5365
5366void gen6_rps_idle(struct drm_i915_private *dev_priv)
5367{
5368
5369
5370
5371
5372
5373 gen6_disable_rps_interrupts(dev_priv);
5374
5375 mutex_lock(&dev_priv->rps.hw_lock);
5376 if (dev_priv->rps.enabled) {
5377 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
5378 vlv_set_rps_idle(dev_priv);
5379 else
5380 gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
5381 dev_priv->rps.last_adj = 0;
5382 I915_WRITE(GEN6_PMINTRMSK,
5383 gen6_sanitize_rps_pm_mask(dev_priv, ~0));
5384 }
5385 mutex_unlock(&dev_priv->rps.hw_lock);
5386
5387 spin_lock(&dev_priv->rps.client_lock);
5388 while (!list_empty(&dev_priv->rps.clients))
5389 list_del_init(dev_priv->rps.clients.next);
5390 spin_unlock(&dev_priv->rps.client_lock);
5391}
5392
5393void gen6_rps_boost(struct drm_i915_private *dev_priv,
5394 struct intel_rps_client *rps,
5395 unsigned long submitted)
5396{
5397
5398
5399
5400 if (!(dev_priv->gt.awake &&
5401 dev_priv->rps.enabled &&
5402 dev_priv->rps.cur_freq < dev_priv->rps.boost_freq))
5403 return;
5404
5405
5406
5407
5408 if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
5409 rps = NULL;
5410
5411 spin_lock(&dev_priv->rps.client_lock);
5412 if (rps == NULL || list_empty(&rps->link)) {
5413 spin_lock_irq(&dev_priv->irq_lock);
5414 if (dev_priv->rps.interrupts_enabled) {
5415 dev_priv->rps.client_boost = true;
5416 schedule_work(&dev_priv->rps.work);
5417 }
5418 spin_unlock_irq(&dev_priv->irq_lock);
5419
5420 if (rps != NULL) {
5421 list_add(&rps->link, &dev_priv->rps.clients);
5422 rps->boosts++;
5423 } else
5424 dev_priv->rps.boosts++;
5425 }
5426 spin_unlock(&dev_priv->rps.client_lock);
5427}
5428
5429int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
5430{
5431 int err;
5432
5433 lockdep_assert_held(&dev_priv->rps.hw_lock);
5434 GEM_BUG_ON(val > dev_priv->rps.max_freq);
5435 GEM_BUG_ON(val < dev_priv->rps.min_freq);
5436
5437 if (!dev_priv->rps.enabled) {
5438 dev_priv->rps.cur_freq = val;
5439 return 0;
5440 }
5441
5442 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
5443 err = valleyview_set_rps(dev_priv, val);
5444 else
5445 err = gen6_set_rps(dev_priv, val);
5446
5447 return err;
5448}
5449
5450static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
5451{
5452 I915_WRITE(GEN6_RC_CONTROL, 0);
5453 I915_WRITE(GEN9_PG_ENABLE, 0);
5454}
5455
5456static void gen9_disable_rps(struct drm_i915_private *dev_priv)
5457{
5458 I915_WRITE(GEN6_RP_CONTROL, 0);
5459}
5460
5461static void gen6_disable_rps(struct drm_i915_private *dev_priv)
5462{
5463 I915_WRITE(GEN6_RC_CONTROL, 0);
5464 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
5465 I915_WRITE(GEN6_RP_CONTROL, 0);
5466}
5467
5468static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
5469{
5470 I915_WRITE(GEN6_RC_CONTROL, 0);
5471}
5472
5473static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
5474{
5475
5476
5477 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5478
5479 I915_WRITE(GEN6_RC_CONTROL, 0);
5480
5481 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5482}
5483
5484static void intel_print_rc6_info(struct drm_i915_private *dev_priv, u32 mode)
5485{
5486 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
5487 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
5488 mode = GEN6_RC_CTL_RC6_ENABLE;
5489 else
5490 mode = 0;
5491 }
5492 if (HAS_RC6p(dev_priv))
5493 DRM_DEBUG_DRIVER("Enabling RC6 states: "
5494 "RC6 %s RC6p %s RC6pp %s\n",
5495 onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
5496 onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
5497 onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
5498
5499 else
5500 DRM_DEBUG_DRIVER("Enabling RC6 states: RC6 %s\n",
5501 onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
5502}
5503
5504static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
5505{
5506 struct i915_ggtt *ggtt = &dev_priv->ggtt;
5507 bool enable_rc6 = true;
5508 unsigned long rc6_ctx_base;
5509 u32 rc_ctl;
5510 int rc_sw_target;
5511
5512 rc_ctl = I915_READ(GEN6_RC_CONTROL);
5513 rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
5514 RC_SW_TARGET_STATE_SHIFT;
5515 DRM_DEBUG_DRIVER("BIOS enabled RC states: "
5516 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
5517 onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
5518 onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
5519 rc_sw_target);
5520
5521 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
5522 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
5523 enable_rc6 = false;
5524 }
5525
5526
5527
5528
5529
5530 rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
5531 if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) &&
5532 (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base +
5533 ggtt->stolen_reserved_size))) {
5534 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
5535 enable_rc6 = false;
5536 }
5537
5538 if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
5539 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
5540 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
5541 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
5542 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
5543 enable_rc6 = false;
5544 }
5545
5546 if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
5547 !I915_READ(GEN8_PUSHBUS_ENABLE) ||
5548 !I915_READ(GEN8_PUSHBUS_SHIFT)) {
5549 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
5550 enable_rc6 = false;
5551 }
5552
5553 if (!I915_READ(GEN6_GFXPAUSE)) {
5554 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
5555 enable_rc6 = false;
5556 }
5557
5558 if (!I915_READ(GEN8_MISC_CTRL0)) {
5559 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
5560 enable_rc6 = false;
5561 }
5562
5563 return enable_rc6;
5564}
5565
5566int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6)
5567{
5568
5569 if (INTEL_INFO(dev_priv)->gen < 6)
5570 return 0;
5571
5572 if (!enable_rc6)
5573 return 0;
5574
5575 if (IS_GEN9_LP(dev_priv) && !bxt_check_bios_rc6_setup(dev_priv)) {
5576 DRM_INFO("RC6 disabled by BIOS\n");
5577 return 0;
5578 }
5579
5580
5581 if (enable_rc6 >= 0) {
5582 int mask;
5583
5584 if (HAS_RC6p(dev_priv))
5585 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
5586 INTEL_RC6pp_ENABLE;
5587 else
5588 mask = INTEL_RC6_ENABLE;
5589
5590 if ((enable_rc6 & mask) != enable_rc6)
5591 DRM_DEBUG_DRIVER("Adjusting RC6 mask to %d "
5592 "(requested %d, valid %d)\n",
5593 enable_rc6 & mask, enable_rc6, mask);
5594
5595 return enable_rc6 & mask;
5596 }
5597
5598 if (IS_IVYBRIDGE(dev_priv))
5599 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
5600
5601 return INTEL_RC6_ENABLE;
5602}
5603
5604static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
5605{
5606
5607
5608
5609 if (IS_GEN9_LP(dev_priv)) {
5610 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
5611 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
5612 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
5613 dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
5614 } else {
5615 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
5616 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
5617 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
5618 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
5619 }
5620
5621 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
5622
5623 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
5624 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
5625 IS_GEN9_BC(dev_priv)) {
5626 u32 ddcc_status = 0;
5627
5628 if (sandybridge_pcode_read(dev_priv,
5629 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
5630 &ddcc_status) == 0)
5631 dev_priv->rps.efficient_freq =
5632 clamp_t(u8,
5633 ((ddcc_status >> 8) & 0xff),
5634 dev_priv->rps.min_freq,
5635 dev_priv->rps.max_freq);
5636 }
5637
5638 if (IS_GEN9_BC(dev_priv)) {
5639
5640
5641
5642 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
5643 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
5644 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
5645 dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
5646 dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
5647 }
5648}
5649
5650static void reset_rps(struct drm_i915_private *dev_priv,
5651 int (*set)(struct drm_i915_private *, u8))
5652{
5653 u8 freq = dev_priv->rps.cur_freq;
5654
5655
5656 dev_priv->rps.power = -1;
5657 dev_priv->rps.cur_freq = -1;
5658
5659 if (set(dev_priv, freq))
5660 DRM_ERROR("Failed to reset RPS to initial values\n");
5661}
5662
5663
5664static void gen9_enable_rps(struct drm_i915_private *dev_priv)
5665{
5666 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5667
5668
5669 I915_WRITE(GEN6_RC_VIDEO_FREQ,
5670 GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
5671
5672
5673 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
5674 GT_INTERVAL_FROM_US(dev_priv, 1000000));
5675
5676 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
5677
5678
5679
5680
5681 reset_rps(dev_priv, gen6_set_rps);
5682
5683 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5684}
5685
5686static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
5687{
5688 struct intel_engine_cs *engine;
5689 enum intel_engine_id id;
5690 uint32_t rc6_mask = 0;
5691
5692
5693 I915_WRITE(GEN6_RC_STATE, 0);
5694
5695
5696
5697 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5698
5699
5700 I915_WRITE(GEN6_RC_CONTROL, 0);
5701
5702
5703
5704
5705 if (IS_SKYLAKE(dev_priv))
5706 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
5707 else
5708 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
5709 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5710 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5711 for_each_engine(engine, dev_priv, id)
5712 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
5713
5714 if (HAS_GUC(dev_priv))
5715 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
5716
5717 I915_WRITE(GEN6_RC_SLEEP, 0);
5718
5719
5720 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
5721 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
5722
5723
5724 if (intel_enable_rc6() & INTEL_RC6_ENABLE)
5725 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5726 DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
5727 I915_WRITE(GEN6_RC6_THRESHOLD, 37500);
5728 I915_WRITE(GEN6_RC_CONTROL,
5729 GEN6_RC_CTL_HW_ENABLE | GEN6_RC_CTL_EI_MODE(1) | rc6_mask);
5730
5731
5732
5733
5734
5735 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
5736 I915_WRITE(GEN9_PG_ENABLE, 0);
5737 else
5738 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
5739 (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
5740
5741 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5742}
5743
5744static void gen8_enable_rps(struct drm_i915_private *dev_priv)
5745{
5746 struct intel_engine_cs *engine;
5747 enum intel_engine_id id;
5748 uint32_t rc6_mask = 0;
5749
5750
5751 I915_WRITE(GEN6_RC_STATE, 0);
5752
5753
5754
5755 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5756
5757
5758 I915_WRITE(GEN6_RC_CONTROL, 0);
5759
5760
5761 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
5762 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5763 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5764 for_each_engine(engine, dev_priv, id)
5765 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
5766 I915_WRITE(GEN6_RC_SLEEP, 0);
5767 if (IS_BROADWELL(dev_priv))
5768 I915_WRITE(GEN6_RC6_THRESHOLD, 625);
5769 else
5770 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
5771
5772
5773 if (intel_enable_rc6() & INTEL_RC6_ENABLE)
5774 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5775 intel_print_rc6_info(dev_priv, rc6_mask);
5776 if (IS_BROADWELL(dev_priv))
5777 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
5778 GEN7_RC_CTL_TO_MODE |
5779 rc6_mask);
5780 else
5781 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
5782 GEN6_RC_CTL_EI_MODE(1) |
5783 rc6_mask);
5784
5785
5786 I915_WRITE(GEN6_RPNSWREQ,
5787 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
5788 I915_WRITE(GEN6_RC_VIDEO_FREQ,
5789 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
5790
5791 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128);
5792
5793
5794 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5795 dev_priv->rps.max_freq_softlimit << 24 |
5796 dev_priv->rps.min_freq_softlimit << 16);
5797
5798 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128);
5799 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128);
5800 I915_WRITE(GEN6_RP_UP_EI, 66000);
5801 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5802
5803 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5804
5805
5806 I915_WRITE(GEN6_RP_CONTROL,
5807 GEN6_RP_MEDIA_TURBO |
5808 GEN6_RP_MEDIA_HW_NORMAL_MODE |
5809 GEN6_RP_MEDIA_IS_GFX |
5810 GEN6_RP_ENABLE |
5811 GEN6_RP_UP_BUSY_AVG |
5812 GEN6_RP_DOWN_IDLE_AVG);
5813
5814
5815
5816 reset_rps(dev_priv, gen6_set_rps);
5817
5818 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5819}
5820
5821static void gen6_enable_rps(struct drm_i915_private *dev_priv)
5822{
5823 struct intel_engine_cs *engine;
5824 enum intel_engine_id id;
5825 u32 rc6vids, rc6_mask = 0;
5826 u32 gtfifodbg;
5827 int rc6_mode;
5828 int ret;
5829
5830 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5831
5832
5833
5834
5835
5836
5837
5838 I915_WRITE(GEN6_RC_STATE, 0);
5839
5840
5841 gtfifodbg = I915_READ(GTFIFODBG);
5842 if (gtfifodbg) {
5843 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
5844 I915_WRITE(GTFIFODBG, gtfifodbg);
5845 }
5846
5847 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5848
5849
5850 I915_WRITE(GEN6_RC_CONTROL, 0);
5851
5852 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
5853 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
5854 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
5855 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5856 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5857
5858 for_each_engine(engine, dev_priv, id)
5859 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
5860
5861 I915_WRITE(GEN6_RC_SLEEP, 0);
5862 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
5863 if (IS_IVYBRIDGE(dev_priv))
5864 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
5865 else
5866 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
5867 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
5868 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000);
5869
5870
5871 rc6_mode = intel_enable_rc6();
5872 if (rc6_mode & INTEL_RC6_ENABLE)
5873 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
5874
5875
5876 if (!IS_HASWELL(dev_priv)) {
5877 if (rc6_mode & INTEL_RC6p_ENABLE)
5878 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
5879
5880 if (rc6_mode & INTEL_RC6pp_ENABLE)
5881 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
5882 }
5883
5884 intel_print_rc6_info(dev_priv, rc6_mask);
5885
5886 I915_WRITE(GEN6_RC_CONTROL,
5887 rc6_mask |
5888 GEN6_RC_CTL_EI_MODE(1) |
5889 GEN6_RC_CTL_HW_ENABLE);
5890
5891
5892 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
5893 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5894
5895 reset_rps(dev_priv, gen6_set_rps);
5896
5897 rc6vids = 0;
5898 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
5899 if (IS_GEN6(dev_priv) && ret) {
5900 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
5901 } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
5902 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
5903 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
5904 rc6vids &= 0xffff00;
5905 rc6vids |= GEN6_ENCODE_RC6_VID(450);
5906 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
5907 if (ret)
5908 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
5909 }
5910
5911 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5912}
5913
5914static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
5915{
5916 int min_freq = 15;
5917 unsigned int gpu_freq;
5918 unsigned int max_ia_freq, min_ring_freq;
5919 unsigned int max_gpu_freq, min_gpu_freq;
5920 int scaling_factor = 180;
5921 struct cpufreq_policy *policy;
5922
5923 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5924
5925 policy = cpufreq_cpu_get(0);
5926 if (policy) {
5927 max_ia_freq = policy->cpuinfo.max_freq;
5928 cpufreq_cpu_put(policy);
5929 } else {
5930
5931
5932
5933
5934 max_ia_freq = tsc_khz;
5935 }
5936
5937
5938 max_ia_freq /= 1000;
5939
5940 min_ring_freq = I915_READ(DCLK) & 0xf;
5941
5942 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
5943
5944 if (IS_GEN9_BC(dev_priv)) {
5945
5946 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
5947 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
5948 } else {
5949 min_gpu_freq = dev_priv->rps.min_freq;
5950 max_gpu_freq = dev_priv->rps.max_freq;
5951 }
5952
5953
5954
5955
5956
5957
5958 for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
5959 int diff = max_gpu_freq - gpu_freq;
5960 unsigned int ia_freq = 0, ring_freq = 0;
5961
5962 if (IS_GEN9_BC(dev_priv)) {
5963
5964
5965
5966
5967 ring_freq = gpu_freq;
5968 } else if (INTEL_INFO(dev_priv)->gen >= 8) {
5969
5970 ring_freq = max(min_ring_freq, gpu_freq);
5971 } else if (IS_HASWELL(dev_priv)) {
5972 ring_freq = mult_frac(gpu_freq, 5, 4);
5973 ring_freq = max(min_ring_freq, ring_freq);
5974
5975 } else {
5976
5977
5978
5979
5980
5981
5982
5983 if (gpu_freq < min_freq)
5984 ia_freq = 800;
5985 else
5986 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
5987 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
5988 }
5989
5990 sandybridge_pcode_write(dev_priv,
5991 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
5992 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
5993 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
5994 gpu_freq);
5995 }
5996}
5997
5998static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
5999{
6000 u32 val, rp0;
6001
6002 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
6003
6004 switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
6005 case 8:
6006
6007 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
6008 break;
6009 case 12:
6010
6011 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
6012 break;
6013 case 16:
6014
6015 default:
6016
6017 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
6018 break;
6019 }
6020
6021 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
6022
6023 return rp0;
6024}
6025
6026static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
6027{
6028 u32 val, rpe;
6029
6030 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
6031 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
6032
6033 return rpe;
6034}
6035
6036static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
6037{
6038 u32 val, rp1;
6039
6040 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
6041 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
6042
6043 return rp1;
6044}
6045
6046static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
6047{
6048 u32 val, rpn;
6049
6050 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
6051 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
6052 FB_GFX_FREQ_FUSE_MASK);
6053
6054 return rpn;
6055}
6056
6057static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
6058{
6059 u32 val, rp1;
6060
6061 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
6062
6063 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
6064
6065 return rp1;
6066}
6067
6068static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
6069{
6070 u32 val, rp0;
6071
6072 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
6073
6074 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
6075
6076 rp0 = min_t(u32, rp0, 0xea);
6077
6078 return rp0;
6079}
6080
6081static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
6082{
6083 u32 val, rpe;
6084
6085 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
6086 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
6087 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
6088 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
6089
6090 return rpe;
6091}
6092
6093static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
6094{
6095 u32 val;
6096
6097 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
6098
6099
6100
6101
6102
6103
6104
6105 return max_t(u32, val, 0xc0);
6106}
6107
6108
6109static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
6110{
6111 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
6112
6113 WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
6114 dev_priv->vlv_pctx->stolen->start);
6115}
6116
6117
6118
6119static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
6120{
6121 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
6122
6123 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
6124}
6125
6126static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
6127{
6128 struct i915_ggtt *ggtt = &dev_priv->ggtt;
6129 unsigned long pctx_paddr, paddr;
6130 u32 pcbr;
6131 int pctx_size = 32*1024;
6132
6133 pcbr = I915_READ(VLV_PCBR);
6134 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
6135 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
6136 paddr = (dev_priv->mm.stolen_base +
6137 (ggtt->stolen_size - pctx_size));
6138
6139 pctx_paddr = (paddr & (~4095));
6140 I915_WRITE(VLV_PCBR, pctx_paddr);
6141 }
6142
6143 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
6144}
6145
6146static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
6147{
6148 struct drm_i915_gem_object *pctx;
6149 unsigned long pctx_paddr;
6150 u32 pcbr;
6151 int pctx_size = 24*1024;
6152
6153 pcbr = I915_READ(VLV_PCBR);
6154 if (pcbr) {
6155
6156 int pcbr_offset;
6157
6158 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
6159 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
6160 pcbr_offset,
6161 I915_GTT_OFFSET_NONE,
6162 pctx_size);
6163 goto out;
6164 }
6165
6166 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176 pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
6177 if (!pctx) {
6178 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
6179 goto out;
6180 }
6181
6182 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
6183 I915_WRITE(VLV_PCBR, pctx_paddr);
6184
6185out:
6186 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
6187 dev_priv->vlv_pctx = pctx;
6188}
6189
6190static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
6191{
6192 if (WARN_ON(!dev_priv->vlv_pctx))
6193 return;
6194
6195 i915_gem_object_put(dev_priv->vlv_pctx);
6196 dev_priv->vlv_pctx = NULL;
6197}
6198
6199static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
6200{
6201 dev_priv->rps.gpll_ref_freq =
6202 vlv_get_cck_clock(dev_priv, "GPLL ref",
6203 CCK_GPLL_CLOCK_CONTROL,
6204 dev_priv->czclk_freq);
6205
6206 DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
6207 dev_priv->rps.gpll_ref_freq);
6208}
6209
6210static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
6211{
6212 u32 val;
6213
6214 valleyview_setup_pctx(dev_priv);
6215
6216 vlv_init_gpll_ref_freq(dev_priv);
6217
6218 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
6219 switch ((val >> 6) & 3) {
6220 case 0:
6221 case 1:
6222 dev_priv->mem_freq = 800;
6223 break;
6224 case 2:
6225 dev_priv->mem_freq = 1066;
6226 break;
6227 case 3:
6228 dev_priv->mem_freq = 1333;
6229 break;
6230 }
6231 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
6232
6233 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
6234 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
6235 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6236 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
6237 dev_priv->rps.max_freq);
6238
6239 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
6240 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6241 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
6242 dev_priv->rps.efficient_freq);
6243
6244 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
6245 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
6246 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
6247 dev_priv->rps.rp1_freq);
6248
6249 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
6250 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6251 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
6252 dev_priv->rps.min_freq);
6253}
6254
6255static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
6256{
6257 u32 val;
6258
6259 cherryview_setup_pctx(dev_priv);
6260
6261 vlv_init_gpll_ref_freq(dev_priv);
6262
6263 mutex_lock(&dev_priv->sb_lock);
6264 val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
6265 mutex_unlock(&dev_priv->sb_lock);
6266
6267 switch ((val >> 2) & 0x7) {
6268 case 3:
6269 dev_priv->mem_freq = 2000;
6270 break;
6271 default:
6272 dev_priv->mem_freq = 1600;
6273 break;
6274 }
6275 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
6276
6277 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
6278 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
6279 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6280 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
6281 dev_priv->rps.max_freq);
6282
6283 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
6284 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6285 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
6286 dev_priv->rps.efficient_freq);
6287
6288 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
6289 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
6290 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
6291 dev_priv->rps.rp1_freq);
6292
6293 dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
6294 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6295 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
6296 dev_priv->rps.min_freq);
6297
6298 WARN_ONCE((dev_priv->rps.max_freq |
6299 dev_priv->rps.efficient_freq |
6300 dev_priv->rps.rp1_freq |
6301 dev_priv->rps.min_freq) & 1,
6302 "Odd GPU freq values\n");
6303}
6304
6305static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
6306{
6307 valleyview_cleanup_pctx(dev_priv);
6308}
6309
6310static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
6311{
6312 struct intel_engine_cs *engine;
6313 enum intel_engine_id id;
6314 u32 gtfifodbg, val, rc6_mode = 0, pcbr;
6315
6316 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6317
6318 gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
6319 GT_FIFO_FREE_ENTRIES_CHV);
6320 if (gtfifodbg) {
6321 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
6322 gtfifodbg);
6323 I915_WRITE(GTFIFODBG, gtfifodbg);
6324 }
6325
6326 cherryview_check_pctx(dev_priv);
6327
6328
6329
6330 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6331
6332
6333 I915_WRITE(GEN6_RC_CONTROL, 0);
6334
6335
6336 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
6337 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
6338 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
6339
6340 for_each_engine(engine, dev_priv, id)
6341 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6342 I915_WRITE(GEN6_RC_SLEEP, 0);
6343
6344
6345 I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
6346
6347
6348 I915_WRITE(VLV_COUNTER_CONTROL,
6349 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
6350 VLV_MEDIA_RC6_COUNT_EN |
6351 VLV_RENDER_RC6_COUNT_EN));
6352
6353
6354 pcbr = I915_READ(VLV_PCBR);
6355
6356
6357 if ((intel_enable_rc6() & INTEL_RC6_ENABLE) &&
6358 (pcbr >> VLV_PCBR_ADDR_SHIFT))
6359 rc6_mode = GEN7_RC_CTL_TO_MODE;
6360
6361 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
6362
6363
6364 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
6365 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
6366 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
6367 I915_WRITE(GEN6_RP_UP_EI, 66000);
6368 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
6369
6370 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6371
6372
6373 I915_WRITE(GEN6_RP_CONTROL,
6374 GEN6_RP_MEDIA_HW_NORMAL_MODE |
6375 GEN6_RP_MEDIA_IS_GFX |
6376 GEN6_RP_ENABLE |
6377 GEN6_RP_UP_BUSY_AVG |
6378 GEN6_RP_DOWN_IDLE_AVG);
6379
6380
6381 val = VLV_OVERRIDE_EN |
6382 VLV_SOC_TDP_EN |
6383 CHV_BIAS_CPU_50_SOC_50;
6384 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
6385
6386 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
6387
6388
6389 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
6390
6391 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
6392 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
6393
6394 reset_rps(dev_priv, valleyview_set_rps);
6395
6396 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6397}
6398
6399static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
6400{
6401 struct intel_engine_cs *engine;
6402 enum intel_engine_id id;
6403 u32 gtfifodbg, val, rc6_mode = 0;
6404
6405 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6406
6407 valleyview_check_pctx(dev_priv);
6408
6409 gtfifodbg = I915_READ(GTFIFODBG);
6410 if (gtfifodbg) {
6411 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
6412 gtfifodbg);
6413 I915_WRITE(GTFIFODBG, gtfifodbg);
6414 }
6415
6416
6417 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6418
6419
6420 I915_WRITE(GEN6_RC_CONTROL, 0);
6421
6422 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
6423 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
6424 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
6425 I915_WRITE(GEN6_RP_UP_EI, 66000);
6426 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
6427
6428 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6429
6430 I915_WRITE(GEN6_RP_CONTROL,
6431 GEN6_RP_MEDIA_TURBO |
6432 GEN6_RP_MEDIA_HW_NORMAL_MODE |
6433 GEN6_RP_MEDIA_IS_GFX |
6434 GEN6_RP_ENABLE |
6435 GEN6_RP_UP_BUSY_AVG |
6436 GEN6_RP_DOWN_IDLE_CONT);
6437
6438 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
6439 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
6440 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
6441
6442 for_each_engine(engine, dev_priv, id)
6443 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6444
6445 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
6446
6447
6448 I915_WRITE(VLV_COUNTER_CONTROL,
6449 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
6450 VLV_MEDIA_RC0_COUNT_EN |
6451 VLV_RENDER_RC0_COUNT_EN |
6452 VLV_MEDIA_RC6_COUNT_EN |
6453 VLV_RENDER_RC6_COUNT_EN));
6454
6455 if (intel_enable_rc6() & INTEL_RC6_ENABLE)
6456 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
6457
6458 intel_print_rc6_info(dev_priv, rc6_mode);
6459
6460 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
6461
6462
6463 val = VLV_OVERRIDE_EN |
6464 VLV_SOC_TDP_EN |
6465 VLV_BIAS_CPU_125_SOC_875;
6466 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
6467
6468 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
6469
6470
6471 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
6472
6473 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
6474 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
6475
6476 reset_rps(dev_priv, valleyview_set_rps);
6477
6478 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6479}
6480
6481static unsigned long intel_pxfreq(u32 vidfreq)
6482{
6483 unsigned long freq;
6484 int div = (vidfreq & 0x3f0000) >> 16;
6485 int post = (vidfreq & 0x3000) >> 12;
6486 int pre = (vidfreq & 0x7);
6487
6488 if (!pre)
6489 return 0;
6490
6491 freq = ((div * 133333) / ((1<<post) * pre));
6492
6493 return freq;
6494}
6495
6496static const struct cparams {
6497 u16 i;
6498 u16 t;
6499 u16 m;
6500 u16 c;
6501} cparams[] = {
6502 { 1, 1333, 301, 28664 },
6503 { 1, 1066, 294, 24460 },
6504 { 1, 800, 294, 25192 },
6505 { 0, 1333, 276, 27605 },
6506 { 0, 1066, 276, 27605 },
6507 { 0, 800, 231, 23784 },
6508};
6509
6510static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
6511{
6512 u64 total_count, diff, ret;
6513 u32 count1, count2, count3, m = 0, c = 0;
6514 unsigned long now = jiffies_to_msecs(jiffies), diff1;
6515 int i;
6516
6517 lockdep_assert_held(&mchdev_lock);
6518
6519 diff1 = now - dev_priv->ips.last_time1;
6520
6521
6522
6523
6524
6525
6526 if (diff1 <= 10)
6527 return dev_priv->ips.chipset_power;
6528
6529 count1 = I915_READ(DMIEC);
6530 count2 = I915_READ(DDREC);
6531 count3 = I915_READ(CSIEC);
6532
6533 total_count = count1 + count2 + count3;
6534
6535
6536 if (total_count < dev_priv->ips.last_count1) {
6537 diff = ~0UL - dev_priv->ips.last_count1;
6538 diff += total_count;
6539 } else {
6540 diff = total_count - dev_priv->ips.last_count1;
6541 }
6542
6543 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
6544 if (cparams[i].i == dev_priv->ips.c_m &&
6545 cparams[i].t == dev_priv->ips.r_t) {
6546 m = cparams[i].m;
6547 c = cparams[i].c;
6548 break;
6549 }
6550 }
6551
6552 diff = div_u64(diff, diff1);
6553 ret = ((m * diff) + c);
6554 ret = div_u64(ret, 10);
6555
6556 dev_priv->ips.last_count1 = total_count;
6557 dev_priv->ips.last_time1 = now;
6558
6559 dev_priv->ips.chipset_power = ret;
6560
6561 return ret;
6562}
6563
6564unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
6565{
6566 unsigned long val;
6567
6568 if (INTEL_INFO(dev_priv)->gen != 5)
6569 return 0;
6570
6571 spin_lock_irq(&mchdev_lock);
6572
6573 val = __i915_chipset_val(dev_priv);
6574
6575 spin_unlock_irq(&mchdev_lock);
6576
6577 return val;
6578}
6579
6580unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
6581{
6582 unsigned long m, x, b;
6583 u32 tsfs;
6584
6585 tsfs = I915_READ(TSFS);
6586
6587 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
6588 x = I915_READ8(TR1);
6589
6590 b = tsfs & TSFS_INTR_MASK;
6591
6592 return ((m * x) / 127) - b;
6593}
6594
6595static int _pxvid_to_vd(u8 pxvid)
6596{
6597 if (pxvid == 0)
6598 return 0;
6599
6600 if (pxvid >= 8 && pxvid < 31)
6601 pxvid = 31;
6602
6603 return (pxvid + 2) * 125;
6604}
6605
6606static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
6607{
6608 const int vd = _pxvid_to_vd(pxvid);
6609 const int vm = vd - 1125;
6610
6611 if (INTEL_INFO(dev_priv)->is_mobile)
6612 return vm > 0 ? vm : 0;
6613
6614 return vd;
6615}
6616
6617static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
6618{
6619 u64 now, diff, diffms;
6620 u32 count;
6621
6622 lockdep_assert_held(&mchdev_lock);
6623
6624 now = ktime_get_raw_ns();
6625 diffms = now - dev_priv->ips.last_time2;
6626 do_div(diffms, NSEC_PER_MSEC);
6627
6628
6629 if (!diffms)
6630 return;
6631
6632 count = I915_READ(GFXEC);
6633
6634 if (count < dev_priv->ips.last_count2) {
6635 diff = ~0UL - dev_priv->ips.last_count2;
6636 diff += count;
6637 } else {
6638 diff = count - dev_priv->ips.last_count2;
6639 }
6640
6641 dev_priv->ips.last_count2 = count;
6642 dev_priv->ips.last_time2 = now;
6643
6644
6645 diff = diff * 1181;
6646 diff = div_u64(diff, diffms * 10);
6647 dev_priv->ips.gfx_power = diff;
6648}
6649
6650void i915_update_gfx_val(struct drm_i915_private *dev_priv)
6651{
6652 if (INTEL_INFO(dev_priv)->gen != 5)
6653 return;
6654
6655 spin_lock_irq(&mchdev_lock);
6656
6657 __i915_update_gfx_val(dev_priv);
6658
6659 spin_unlock_irq(&mchdev_lock);
6660}
6661
6662static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
6663{
6664 unsigned long t, corr, state1, corr2, state2;
6665 u32 pxvid, ext_v;
6666
6667 lockdep_assert_held(&mchdev_lock);
6668
6669 pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
6670 pxvid = (pxvid >> 24) & 0x7f;
6671 ext_v = pvid_to_extvid(dev_priv, pxvid);
6672
6673 state1 = ext_v;
6674
6675 t = i915_mch_val(dev_priv);
6676
6677
6678
6679
6680 if (t > 80)
6681 corr = ((t * 2349) + 135940);
6682 else if (t >= 50)
6683 corr = ((t * 964) + 29317);
6684 else
6685 corr = ((t * 301) + 1004);
6686
6687 corr = corr * ((150142 * state1) / 10000 - 78642);
6688 corr /= 100000;
6689 corr2 = (corr * dev_priv->ips.corr);
6690
6691 state2 = (corr2 * state1) / 10000;
6692 state2 /= 100;
6693
6694 __i915_update_gfx_val(dev_priv);
6695
6696 return dev_priv->ips.gfx_power + state2;
6697}
6698
6699unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
6700{
6701 unsigned long val;
6702
6703 if (INTEL_INFO(dev_priv)->gen != 5)
6704 return 0;
6705
6706 spin_lock_irq(&mchdev_lock);
6707
6708 val = __i915_gfx_val(dev_priv);
6709
6710 spin_unlock_irq(&mchdev_lock);
6711
6712 return val;
6713}
6714
6715
6716
6717
6718
6719
6720
6721unsigned long i915_read_mch_val(void)
6722{
6723 struct drm_i915_private *dev_priv;
6724 unsigned long chipset_val, graphics_val, ret = 0;
6725
6726 spin_lock_irq(&mchdev_lock);
6727 if (!i915_mch_dev)
6728 goto out_unlock;
6729 dev_priv = i915_mch_dev;
6730
6731 chipset_val = __i915_chipset_val(dev_priv);
6732 graphics_val = __i915_gfx_val(dev_priv);
6733
6734 ret = chipset_val + graphics_val;
6735
6736out_unlock:
6737 spin_unlock_irq(&mchdev_lock);
6738
6739 return ret;
6740}
6741EXPORT_SYMBOL_GPL(i915_read_mch_val);
6742
6743
6744
6745
6746
6747
6748bool i915_gpu_raise(void)
6749{
6750 struct drm_i915_private *dev_priv;
6751 bool ret = true;
6752
6753 spin_lock_irq(&mchdev_lock);
6754 if (!i915_mch_dev) {
6755 ret = false;
6756 goto out_unlock;
6757 }
6758 dev_priv = i915_mch_dev;
6759
6760 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
6761 dev_priv->ips.max_delay--;
6762
6763out_unlock:
6764 spin_unlock_irq(&mchdev_lock);
6765
6766 return ret;
6767}
6768EXPORT_SYMBOL_GPL(i915_gpu_raise);
6769
6770
6771
6772
6773
6774
6775
6776bool i915_gpu_lower(void)
6777{
6778 struct drm_i915_private *dev_priv;
6779 bool ret = true;
6780
6781 spin_lock_irq(&mchdev_lock);
6782 if (!i915_mch_dev) {
6783 ret = false;
6784 goto out_unlock;
6785 }
6786 dev_priv = i915_mch_dev;
6787
6788 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
6789 dev_priv->ips.max_delay++;
6790
6791out_unlock:
6792 spin_unlock_irq(&mchdev_lock);
6793
6794 return ret;
6795}
6796EXPORT_SYMBOL_GPL(i915_gpu_lower);
6797
6798
6799
6800
6801
6802
6803bool i915_gpu_busy(void)
6804{
6805 bool ret = false;
6806
6807 spin_lock_irq(&mchdev_lock);
6808 if (i915_mch_dev)
6809 ret = i915_mch_dev->gt.awake;
6810 spin_unlock_irq(&mchdev_lock);
6811
6812 return ret;
6813}
6814EXPORT_SYMBOL_GPL(i915_gpu_busy);
6815
6816
6817
6818
6819
6820
6821
6822bool i915_gpu_turbo_disable(void)
6823{
6824 struct drm_i915_private *dev_priv;
6825 bool ret = true;
6826
6827 spin_lock_irq(&mchdev_lock);
6828 if (!i915_mch_dev) {
6829 ret = false;
6830 goto out_unlock;
6831 }
6832 dev_priv = i915_mch_dev;
6833
6834 dev_priv->ips.max_delay = dev_priv->ips.fstart;
6835
6836 if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
6837 ret = false;
6838
6839out_unlock:
6840 spin_unlock_irq(&mchdev_lock);
6841
6842 return ret;
6843}
6844EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854static void
6855ips_ping_for_i915_load(void)
6856{
6857 void (*link)(void);
6858
6859 link = symbol_get(ips_link_to_i915_driver);
6860 if (link) {
6861 link();
6862 symbol_put(ips_link_to_i915_driver);
6863 }
6864}
6865
6866void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
6867{
6868
6869
6870 spin_lock_irq(&mchdev_lock);
6871 i915_mch_dev = dev_priv;
6872 spin_unlock_irq(&mchdev_lock);
6873
6874 ips_ping_for_i915_load();
6875}
6876
6877void intel_gpu_ips_teardown(void)
6878{
6879 spin_lock_irq(&mchdev_lock);
6880 i915_mch_dev = NULL;
6881 spin_unlock_irq(&mchdev_lock);
6882}
6883
6884static void intel_init_emon(struct drm_i915_private *dev_priv)
6885{
6886 u32 lcfuse;
6887 u8 pxw[16];
6888 int i;
6889
6890
6891 I915_WRITE(ECR, 0);
6892 POSTING_READ(ECR);
6893
6894
6895 I915_WRITE(SDEW, 0x15040d00);
6896 I915_WRITE(CSIEW0, 0x007f0000);
6897 I915_WRITE(CSIEW1, 0x1e220004);
6898 I915_WRITE(CSIEW2, 0x04000004);
6899
6900 for (i = 0; i < 5; i++)
6901 I915_WRITE(PEW(i), 0);
6902 for (i = 0; i < 3; i++)
6903 I915_WRITE(DEW(i), 0);
6904
6905
6906 for (i = 0; i < 16; i++) {
6907 u32 pxvidfreq = I915_READ(PXVFREQ(i));
6908 unsigned long freq = intel_pxfreq(pxvidfreq);
6909 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
6910 PXVFREQ_PX_SHIFT;
6911 unsigned long val;
6912
6913 val = vid * vid;
6914 val *= (freq / 1000);
6915 val *= 255;
6916 val /= (127*127*900);
6917 if (val > 0xff)
6918 DRM_ERROR("bad pxval: %ld\n", val);
6919 pxw[i] = val;
6920 }
6921
6922 pxw[14] = 0;
6923 pxw[15] = 0;
6924
6925 for (i = 0; i < 4; i++) {
6926 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
6927 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
6928 I915_WRITE(PXW(i), val);
6929 }
6930
6931
6932 I915_WRITE(OGW0, 0);
6933 I915_WRITE(OGW1, 0);
6934 I915_WRITE(EG0, 0x00007f00);
6935 I915_WRITE(EG1, 0x0000000e);
6936 I915_WRITE(EG2, 0x000e0000);
6937 I915_WRITE(EG3, 0x68000300);
6938 I915_WRITE(EG4, 0x42000000);
6939 I915_WRITE(EG5, 0x00140031);
6940 I915_WRITE(EG6, 0);
6941 I915_WRITE(EG7, 0);
6942
6943 for (i = 0; i < 8; i++)
6944 I915_WRITE(PXWL(i), 0);
6945
6946
6947 I915_WRITE(ECR, 0x80000019);
6948
6949 lcfuse = I915_READ(LCFUSE02);
6950
6951 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
6952}
6953
6954void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
6955{
6956
6957
6958
6959
6960 if (!i915.enable_rc6) {
6961 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
6962 intel_runtime_pm_get(dev_priv);
6963 }
6964
6965 mutex_lock(&dev_priv->drm.struct_mutex);
6966 mutex_lock(&dev_priv->rps.hw_lock);
6967
6968
6969 if (IS_CHERRYVIEW(dev_priv))
6970 cherryview_init_gt_powersave(dev_priv);
6971 else if (IS_VALLEYVIEW(dev_priv))
6972 valleyview_init_gt_powersave(dev_priv);
6973 else if (INTEL_GEN(dev_priv) >= 6)
6974 gen6_init_rps_frequencies(dev_priv);
6975
6976
6977 dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
6978 dev_priv->rps.cur_freq = dev_priv->rps.idle_freq;
6979
6980 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
6981 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
6982
6983 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6984 dev_priv->rps.min_freq_softlimit =
6985 max_t(int,
6986 dev_priv->rps.efficient_freq,
6987 intel_freq_opcode(dev_priv, 450));
6988
6989
6990 if (IS_GEN6(dev_priv) ||
6991 IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
6992 u32 params = 0;
6993
6994 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, ¶ms);
6995 if (params & BIT(31)) {
6996 DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
6997 (dev_priv->rps.max_freq & 0xff) * 50,
6998 (params & 0xff) * 50);
6999 dev_priv->rps.max_freq = params & 0xff;
7000 }
7001 }
7002
7003
7004 dev_priv->rps.boost_freq = dev_priv->rps.max_freq;
7005
7006 mutex_unlock(&dev_priv->rps.hw_lock);
7007 mutex_unlock(&dev_priv->drm.struct_mutex);
7008
7009 intel_autoenable_gt_powersave(dev_priv);
7010}
7011
7012void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7013{
7014 if (IS_VALLEYVIEW(dev_priv))
7015 valleyview_cleanup_gt_powersave(dev_priv);
7016
7017 if (!i915.enable_rc6)
7018 intel_runtime_pm_put(dev_priv);
7019}
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
7030{
7031 if (INTEL_GEN(dev_priv) < 6)
7032 return;
7033
7034 if (cancel_delayed_work_sync(&dev_priv->rps.autoenable_work))
7035 intel_runtime_pm_put(dev_priv);
7036
7037
7038}
7039
7040void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
7041{
7042 dev_priv->rps.enabled = true;
7043 intel_disable_gt_powersave(dev_priv);
7044
7045 gen6_reset_rps_interrupts(dev_priv);
7046}
7047
7048void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
7049{
7050 if (!READ_ONCE(dev_priv->rps.enabled))
7051 return;
7052
7053 mutex_lock(&dev_priv->rps.hw_lock);
7054
7055 if (INTEL_GEN(dev_priv) >= 9) {
7056 gen9_disable_rc6(dev_priv);
7057 gen9_disable_rps(dev_priv);
7058 } else if (IS_CHERRYVIEW(dev_priv)) {
7059 cherryview_disable_rps(dev_priv);
7060 } else if (IS_VALLEYVIEW(dev_priv)) {
7061 valleyview_disable_rps(dev_priv);
7062 } else if (INTEL_GEN(dev_priv) >= 6) {
7063 gen6_disable_rps(dev_priv);
7064 } else if (IS_IRONLAKE_M(dev_priv)) {
7065 ironlake_disable_drps(dev_priv);
7066 }
7067
7068 dev_priv->rps.enabled = false;
7069 mutex_unlock(&dev_priv->rps.hw_lock);
7070}
7071
7072void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
7073{
7074
7075
7076
7077 if (READ_ONCE(dev_priv->rps.enabled))
7078 return;
7079
7080
7081 if (intel_vgpu_active(dev_priv))
7082 return;
7083
7084 mutex_lock(&dev_priv->rps.hw_lock);
7085
7086 if (IS_CHERRYVIEW(dev_priv)) {
7087 cherryview_enable_rps(dev_priv);
7088 } else if (IS_VALLEYVIEW(dev_priv)) {
7089 valleyview_enable_rps(dev_priv);
7090 } else if (INTEL_GEN(dev_priv) >= 9) {
7091 gen9_enable_rc6(dev_priv);
7092 gen9_enable_rps(dev_priv);
7093 if (IS_GEN9_BC(dev_priv))
7094 gen6_update_ring_freq(dev_priv);
7095 } else if (IS_BROADWELL(dev_priv)) {
7096 gen8_enable_rps(dev_priv);
7097 gen6_update_ring_freq(dev_priv);
7098 } else if (INTEL_GEN(dev_priv) >= 6) {
7099 gen6_enable_rps(dev_priv);
7100 gen6_update_ring_freq(dev_priv);
7101 } else if (IS_IRONLAKE_M(dev_priv)) {
7102 ironlake_enable_drps(dev_priv);
7103 intel_init_emon(dev_priv);
7104 }
7105
7106 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
7107 WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
7108
7109 WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
7110 WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
7111
7112 dev_priv->rps.enabled = true;
7113 mutex_unlock(&dev_priv->rps.hw_lock);
7114}
7115
7116static void __intel_autoenable_gt_powersave(struct work_struct *work)
7117{
7118 struct drm_i915_private *dev_priv =
7119 container_of(work, typeof(*dev_priv), rps.autoenable_work.work);
7120 struct intel_engine_cs *rcs;
7121 struct drm_i915_gem_request *req;
7122
7123 if (READ_ONCE(dev_priv->rps.enabled))
7124 goto out;
7125
7126 rcs = dev_priv->engine[RCS];
7127 if (rcs->last_retired_context)
7128 goto out;
7129
7130 if (!rcs->init_context)
7131 goto out;
7132
7133 mutex_lock(&dev_priv->drm.struct_mutex);
7134
7135 req = i915_gem_request_alloc(rcs, dev_priv->kernel_context);
7136 if (IS_ERR(req))
7137 goto unlock;
7138
7139 if (!i915.enable_execlists && i915_switch_context(req) == 0)
7140 rcs->init_context(req);
7141
7142
7143 i915_add_request(req);
7144
7145unlock:
7146 mutex_unlock(&dev_priv->drm.struct_mutex);
7147out:
7148 intel_runtime_pm_put(dev_priv);
7149}
7150
7151void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv)
7152{
7153 if (READ_ONCE(dev_priv->rps.enabled))
7154 return;
7155
7156 if (IS_IRONLAKE_M(dev_priv)) {
7157 ironlake_enable_drps(dev_priv);
7158 intel_init_emon(dev_priv);
7159 } else if (INTEL_INFO(dev_priv)->gen >= 6) {
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172 if (queue_delayed_work(dev_priv->wq,
7173 &dev_priv->rps.autoenable_work,
7174 round_jiffies_up_relative(HZ)))
7175 intel_runtime_pm_get_noresume(dev_priv);
7176 }
7177}
7178
7179static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7180{
7181
7182
7183
7184
7185
7186 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7187}
7188
7189static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7190{
7191 enum pipe pipe;
7192
7193 for_each_pipe(dev_priv, pipe) {
7194 I915_WRITE(DSPCNTR(pipe),
7195 I915_READ(DSPCNTR(pipe)) |
7196 DISPPLANE_TRICKLE_FEED_DISABLE);
7197
7198 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
7199 POSTING_READ(DSPSURF(pipe));
7200 }
7201}
7202
7203static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7204{
7205 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
7206 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
7207 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
7208
7209
7210
7211
7212
7213}
7214
7215static void ironlake_init_clock_gating(struct drm_i915_private *dev_priv)
7216{
7217 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7218
7219
7220
7221
7222
7223 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7224 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7225 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7226
7227 I915_WRITE(PCH_3DCGDIS0,
7228 MARIUNIT_CLOCK_GATE_DISABLE |
7229 SVSMUNIT_CLOCK_GATE_DISABLE);
7230 I915_WRITE(PCH_3DCGDIS1,
7231 VFMUNIT_CLOCK_GATE_DISABLE);
7232
7233
7234
7235
7236
7237
7238
7239
7240 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7241 (I915_READ(ILK_DISPLAY_CHICKEN2) |
7242 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7243 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7244 I915_WRITE(DISP_ARB_CTL,
7245 (I915_READ(DISP_ARB_CTL) |
7246 DISP_FBC_WM_DIS));
7247
7248 ilk_init_lp_watermarks(dev_priv);
7249
7250
7251
7252
7253
7254
7255
7256
7257 if (IS_IRONLAKE_M(dev_priv)) {
7258
7259 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7260 I915_READ(ILK_DISPLAY_CHICKEN1) |
7261 ILK_FBCQ_DIS);
7262 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7263 I915_READ(ILK_DISPLAY_CHICKEN2) |
7264 ILK_DPARB_GATE);
7265 }
7266
7267 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
7268
7269 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7270 I915_READ(ILK_DISPLAY_CHICKEN2) |
7271 ILK_ELPIN_409_SELECT);
7272 I915_WRITE(_3D_CHICKEN2,
7273 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
7274 _3D_CHICKEN2_WM_READ_PIPELINED);
7275
7276
7277 I915_WRITE(CACHE_MODE_0,
7278 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
7279
7280
7281 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7282
7283 g4x_disable_trickle_feed(dev_priv);
7284
7285 ibx_init_clock_gating(dev_priv);
7286}
7287
7288static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7289{
7290 int pipe;
7291 uint32_t val;
7292
7293
7294
7295
7296
7297
7298 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7299 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7300 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7301 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
7302 DPLS_EDP_PPS_FIX_DIS);
7303
7304
7305
7306 for_each_pipe(dev_priv, pipe) {
7307 val = I915_READ(TRANS_CHICKEN2(pipe));
7308 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7309 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7310 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7311 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7312 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
7313 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7314 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7315 I915_WRITE(TRANS_CHICKEN2(pipe), val);
7316 }
7317
7318 for_each_pipe(dev_priv, pipe) {
7319 I915_WRITE(TRANS_CHICKEN1(pipe),
7320 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7321 }
7322}
7323
7324static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7325{
7326 uint32_t tmp;
7327
7328 tmp = I915_READ(MCH_SSKPD);
7329 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
7330 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7331 tmp);
7332}
7333
7334static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7335{
7336 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7337
7338 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
7339
7340 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7341 I915_READ(ILK_DISPLAY_CHICKEN2) |
7342 ILK_ELPIN_409_SELECT);
7343
7344
7345 I915_WRITE(_3D_CHICKEN,
7346 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
7347
7348
7349 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359 I915_WRITE(GEN6_GT_MODE,
7360 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7361
7362 ilk_init_lp_watermarks(dev_priv);
7363
7364 I915_WRITE(CACHE_MODE_0,
7365 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
7366
7367 I915_WRITE(GEN6_UCGCTL1,
7368 I915_READ(GEN6_UCGCTL1) |
7369 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7370 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385 I915_WRITE(GEN6_UCGCTL2,
7386 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7387 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7388
7389
7390 I915_WRITE(_3D_CHICKEN3,
7391 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
7392
7393
7394
7395
7396
7397
7398 I915_WRITE(_3D_CHICKEN3,
7399 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7413 I915_READ(ILK_DISPLAY_CHICKEN1) |
7414 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7415 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7416 I915_READ(ILK_DISPLAY_CHICKEN2) |
7417 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7418 I915_WRITE(ILK_DSPCLK_GATE_D,
7419 I915_READ(ILK_DSPCLK_GATE_D) |
7420 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7421 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7422
7423 g4x_disable_trickle_feed(dev_priv);
7424
7425 cpt_init_clock_gating(dev_priv);
7426
7427 gen6_check_mch_setup(dev_priv);
7428}
7429
7430static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
7431{
7432 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
7433
7434
7435
7436
7437
7438
7439
7440 reg &= ~GEN7_FF_SCHED_MASK;
7441 reg |= GEN7_FF_TS_SCHED_HW;
7442 reg |= GEN7_FF_VS_SCHED_HW;
7443 reg |= GEN7_FF_DS_SCHED_HW;
7444
7445 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
7446}
7447
7448static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7449{
7450
7451
7452
7453
7454 if (HAS_PCH_LPT_LP(dev_priv))
7455 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7456 I915_READ(SOUTH_DSPCLK_GATE_D) |
7457 PCH_LP_PARTITION_LEVEL_DISABLE);
7458
7459
7460 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7461 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7462 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7463}
7464
7465static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7466{
7467 if (HAS_PCH_LPT_LP(dev_priv)) {
7468 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
7469
7470 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7471 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7472 }
7473}
7474
7475static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7476 int general_prio_credits,
7477 int high_prio_credits)
7478{
7479 u32 misccpctl;
7480
7481
7482 misccpctl = I915_READ(GEN7_MISCCPCTL);
7483 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7484
7485 I915_WRITE(GEN8_L3SQCREG1,
7486 L3_GENERAL_PRIO_CREDITS(general_prio_credits) |
7487 L3_HIGH_PRIO_CREDITS(high_prio_credits));
7488
7489
7490
7491
7492
7493 POSTING_READ(GEN8_L3SQCREG1);
7494 udelay(1);
7495 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7496}
7497
7498static void kabylake_init_clock_gating(struct drm_i915_private *dev_priv)
7499{
7500 gen9_init_clock_gating(dev_priv);
7501
7502
7503 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7504 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7505 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7506
7507
7508 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7509 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7510 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7511
7512
7513 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7514 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7515}
7516
7517static void skylake_init_clock_gating(struct drm_i915_private *dev_priv)
7518{
7519 gen9_init_clock_gating(dev_priv);
7520
7521
7522 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7523 FBC_LLC_FULLY_OPEN);
7524
7525
7526 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7527 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7528}
7529
7530static void broadwell_init_clock_gating(struct drm_i915_private *dev_priv)
7531{
7532 enum pipe pipe;
7533
7534 ilk_init_lp_watermarks(dev_priv);
7535
7536
7537 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7538
7539
7540 I915_WRITE(CHICKEN_PAR1_1,
7541 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7542
7543
7544 for_each_pipe(dev_priv, pipe) {
7545 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7546 I915_READ(CHICKEN_PIPESL_1(pipe)) |
7547 BDW_DPRS_MASK_VBLANK_SRD);
7548 }
7549
7550
7551
7552 I915_WRITE(GEN7_FF_THREAD_MODE,
7553 I915_READ(GEN7_FF_THREAD_MODE) &
7554 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7555
7556 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7557 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7558
7559
7560 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7561 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7562
7563
7564 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7565
7566
7567
7568
7569
7570
7571 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
7572
7573
7574 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7575 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7576
7577 lpt_init_clock_gating(dev_priv);
7578
7579
7580
7581
7582
7583
7584 I915_WRITE(GEN6_UCGCTL1,
7585 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7586}
7587
7588static void haswell_init_clock_gating(struct drm_i915_private *dev_priv)
7589{
7590 ilk_init_lp_watermarks(dev_priv);
7591
7592
7593 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
7594 I915_WRITE(HSW_ROW_CHICKEN3,
7595 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
7596
7597
7598 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7599 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7600 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7601
7602
7603 I915_WRITE(GEN7_FF_THREAD_MODE,
7604 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
7605
7606
7607 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7608
7609
7610 I915_WRITE(CACHE_MODE_0_GEN7,
7611 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
7612
7613
7614 I915_WRITE(CACHE_MODE_1,
7615 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625 I915_WRITE(GEN7_GT_MODE,
7626 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7627
7628
7629 I915_WRITE(HALF_SLICE_CHICKEN3,
7630 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
7631
7632
7633 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7634
7635
7636 I915_WRITE(CHICKEN_PAR1_1,
7637 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
7638
7639 lpt_init_clock_gating(dev_priv);
7640}
7641
7642static void ivybridge_init_clock_gating(struct drm_i915_private *dev_priv)
7643{
7644 uint32_t snpcr;
7645
7646 ilk_init_lp_watermarks(dev_priv);
7647
7648 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7649
7650
7651 I915_WRITE(_3D_CHICKEN3,
7652 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
7653
7654
7655 I915_WRITE(IVB_CHICKEN3,
7656 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7657 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7658
7659
7660 if (IS_IVB_GT1(dev_priv))
7661 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
7662 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
7663
7664
7665 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7666
7667
7668 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
7669 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
7670
7671
7672 I915_WRITE(GEN7_L3CNTLREG1,
7673 GEN7_WA_FOR_GEN7_L3_CONTROL);
7674 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
7675 GEN7_WA_L3_CHICKEN_MODE);
7676 if (IS_IVB_GT1(dev_priv))
7677 I915_WRITE(GEN7_ROW_CHICKEN2,
7678 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7679 else {
7680
7681 I915_WRITE(GEN7_ROW_CHICKEN2,
7682 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7683 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7684 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7685 }
7686
7687
7688 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
7689 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
7690
7691
7692
7693
7694
7695 I915_WRITE(GEN6_UCGCTL2,
7696 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7697
7698
7699 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7700 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7701 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7702
7703 g4x_disable_trickle_feed(dev_priv);
7704
7705 gen7_setup_fixed_func_scheduler(dev_priv);
7706
7707 if (0) {
7708
7709 I915_WRITE(CACHE_MODE_0_GEN7,
7710 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
7711 }
7712
7713
7714 I915_WRITE(CACHE_MODE_1,
7715 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725 I915_WRITE(GEN7_GT_MODE,
7726 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7727
7728 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7729 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7730 snpcr |= GEN6_MBC_SNPCR_MED;
7731 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7732
7733 if (!HAS_PCH_NOP(dev_priv))
7734 cpt_init_clock_gating(dev_priv);
7735
7736 gen6_check_mch_setup(dev_priv);
7737}
7738
7739static void valleyview_init_clock_gating(struct drm_i915_private *dev_priv)
7740{
7741
7742 I915_WRITE(_3D_CHICKEN3,
7743 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
7744
7745
7746 I915_WRITE(IVB_CHICKEN3,
7747 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7748 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7749
7750
7751
7752 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
7753 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
7754 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
7755
7756
7757 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7758
7759
7760 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
7761 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
7762
7763
7764 I915_WRITE(GEN7_ROW_CHICKEN2,
7765 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7766
7767
7768 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7769 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7770 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7771
7772 gen7_setup_fixed_func_scheduler(dev_priv);
7773
7774
7775
7776
7777
7778 I915_WRITE(GEN6_UCGCTL2,
7779 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7780
7781
7782
7783
7784 I915_WRITE(GEN7_UCGCTL4,
7785 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7786
7787
7788
7789
7790
7791 I915_WRITE(CACHE_MODE_1,
7792 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802 I915_WRITE(GEN7_GT_MODE,
7803 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7804
7805
7806
7807
7808
7809 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
7810
7811
7812
7813
7814
7815
7816 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7817}
7818
7819static void cherryview_init_clock_gating(struct drm_i915_private *dev_priv)
7820{
7821
7822
7823 I915_WRITE(GEN7_FF_THREAD_MODE,
7824 I915_READ(GEN7_FF_THREAD_MODE) &
7825 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7826
7827
7828 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7829 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7830
7831
7832 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7833 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7834
7835
7836 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7837 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7838
7839
7840
7841
7842
7843
7844 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7845
7846
7847
7848
7849
7850 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
7851}
7852
7853static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7854{
7855 uint32_t dspclk_gate;
7856
7857 I915_WRITE(RENCLK_GATE_D1, 0);
7858 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7859 GS_UNIT_CLOCK_GATE_DISABLE |
7860 CL_UNIT_CLOCK_GATE_DISABLE);
7861 I915_WRITE(RAMCLK_GATE_D, 0);
7862 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7863 OVRUNIT_CLOCK_GATE_DISABLE |
7864 OVCUNIT_CLOCK_GATE_DISABLE;
7865 if (IS_GM45(dev_priv))
7866 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7867 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7868
7869
7870 I915_WRITE(CACHE_MODE_0,
7871 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
7872
7873
7874 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7875
7876 g4x_disable_trickle_feed(dev_priv);
7877}
7878
7879static void crestline_init_clock_gating(struct drm_i915_private *dev_priv)
7880{
7881 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7882 I915_WRITE(RENCLK_GATE_D2, 0);
7883 I915_WRITE(DSPCLK_GATE_D, 0);
7884 I915_WRITE(RAMCLK_GATE_D, 0);
7885 I915_WRITE16(DEUC, 0);
7886 I915_WRITE(MI_ARB_STATE,
7887 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7888
7889
7890 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7891}
7892
7893static void broadwater_init_clock_gating(struct drm_i915_private *dev_priv)
7894{
7895 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7896 I965_RCC_CLOCK_GATE_DISABLE |
7897 I965_RCPB_CLOCK_GATE_DISABLE |
7898 I965_ISC_CLOCK_GATE_DISABLE |
7899 I965_FBC_CLOCK_GATE_DISABLE);
7900 I915_WRITE(RENCLK_GATE_D2, 0);
7901 I915_WRITE(MI_ARB_STATE,
7902 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7903
7904
7905 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7906}
7907
7908static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7909{
7910 u32 dstate = I915_READ(D_STATE);
7911
7912 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7913 DSTATE_DOT_CLOCK_GATING;
7914 I915_WRITE(D_STATE, dstate);
7915
7916 if (IS_PINEVIEW(dev_priv))
7917 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7918
7919
7920 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7921
7922
7923 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7924
7925
7926 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7927
7928 I915_WRITE(MI_ARB_STATE,
7929 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7930}
7931
7932static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7933{
7934 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7935
7936
7937 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7938 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7939
7940 I915_WRITE(MEM_MODE,
7941 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7942}
7943
7944static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7945{
7946 I915_WRITE(MEM_MODE,
7947 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7948 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7949}
7950
7951void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7952{
7953 dev_priv->display.init_clock_gating(dev_priv);
7954}
7955
7956void intel_suspend_hw(struct drm_i915_private *dev_priv)
7957{
7958 if (HAS_PCH_LPT(dev_priv))
7959 lpt_suspend_hw(dev_priv);
7960}
7961
7962static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7963{
7964 DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
7965}
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7977{
7978 if (IS_SKYLAKE(dev_priv))
7979 dev_priv->display.init_clock_gating = skylake_init_clock_gating;
7980 else if (IS_KABYLAKE(dev_priv))
7981 dev_priv->display.init_clock_gating = kabylake_init_clock_gating;
7982 else if (IS_BROXTON(dev_priv))
7983 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7984 else if (IS_GEMINILAKE(dev_priv))
7985 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7986 else if (IS_BROADWELL(dev_priv))
7987 dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
7988 else if (IS_CHERRYVIEW(dev_priv))
7989 dev_priv->display.init_clock_gating = cherryview_init_clock_gating;
7990 else if (IS_HASWELL(dev_priv))
7991 dev_priv->display.init_clock_gating = haswell_init_clock_gating;
7992 else if (IS_IVYBRIDGE(dev_priv))
7993 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
7994 else if (IS_VALLEYVIEW(dev_priv))
7995 dev_priv->display.init_clock_gating = valleyview_init_clock_gating;
7996 else if (IS_GEN6(dev_priv))
7997 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7998 else if (IS_GEN5(dev_priv))
7999 dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
8000 else if (IS_G4X(dev_priv))
8001 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
8002 else if (IS_I965GM(dev_priv))
8003 dev_priv->display.init_clock_gating = crestline_init_clock_gating;
8004 else if (IS_I965G(dev_priv))
8005 dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
8006 else if (IS_GEN3(dev_priv))
8007 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
8008 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
8009 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
8010 else if (IS_GEN2(dev_priv))
8011 dev_priv->display.init_clock_gating = i830_init_clock_gating;
8012 else {
8013 MISSING_CASE(INTEL_DEVID(dev_priv));
8014 dev_priv->display.init_clock_gating = nop_init_clock_gating;
8015 }
8016}
8017
8018
8019void intel_init_pm(struct drm_i915_private *dev_priv)
8020{
8021 intel_fbc_init(dev_priv);
8022
8023
8024 if (IS_PINEVIEW(dev_priv))
8025 i915_pineview_get_mem_freq(dev_priv);
8026 else if (IS_GEN5(dev_priv))
8027 i915_ironlake_get_mem_freq(dev_priv);
8028
8029
8030 if (INTEL_GEN(dev_priv) >= 9) {
8031 skl_setup_wm_latency(dev_priv);
8032 dev_priv->display.initial_watermarks = skl_initial_wm;
8033 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
8034 dev_priv->display.compute_global_watermarks = skl_compute_wm;
8035 } else if (HAS_PCH_SPLIT(dev_priv)) {
8036 ilk_setup_wm_latency(dev_priv);
8037
8038 if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
8039 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8040 (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
8041 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8042 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
8043 dev_priv->display.compute_intermediate_wm =
8044 ilk_compute_intermediate_wm;
8045 dev_priv->display.initial_watermarks =
8046 ilk_initial_watermarks;
8047 dev_priv->display.optimize_watermarks =
8048 ilk_optimize_watermarks;
8049 } else {
8050 DRM_DEBUG_KMS("Failed to read display plane latency. "
8051 "Disable CxSR\n");
8052 }
8053 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8054 vlv_setup_wm_latency(dev_priv);
8055 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
8056 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
8057 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
8058 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
8059 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
8060 } else if (IS_PINEVIEW(dev_priv)) {
8061 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
8062 dev_priv->is_ddr3,
8063 dev_priv->fsb_freq,
8064 dev_priv->mem_freq)) {
8065 DRM_INFO("failed to find known CxSR latency "
8066 "(found ddr%s fsb freq %d, mem freq %d), "
8067 "disabling CxSR\n",
8068 (dev_priv->is_ddr3 == 1) ? "3" : "2",
8069 dev_priv->fsb_freq, dev_priv->mem_freq);
8070
8071 intel_set_memory_cxsr(dev_priv, false);
8072 dev_priv->display.update_wm = NULL;
8073 } else
8074 dev_priv->display.update_wm = pineview_update_wm;
8075 } else if (IS_G4X(dev_priv)) {
8076 dev_priv->display.update_wm = g4x_update_wm;
8077 } else if (IS_GEN4(dev_priv)) {
8078 dev_priv->display.update_wm = i965_update_wm;
8079 } else if (IS_GEN3(dev_priv)) {
8080 dev_priv->display.update_wm = i9xx_update_wm;
8081 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
8082 } else if (IS_GEN2(dev_priv)) {
8083 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
8084 dev_priv->display.update_wm = i845_update_wm;
8085 dev_priv->display.get_fifo_size = i845_get_fifo_size;
8086 } else {
8087 dev_priv->display.update_wm = i9xx_update_wm;
8088 dev_priv->display.get_fifo_size = i830_get_fifo_size;
8089 }
8090 } else {
8091 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
8092 }
8093}
8094
8095static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
8096{
8097 uint32_t flags =
8098 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
8099
8100 switch (flags) {
8101 case GEN6_PCODE_SUCCESS:
8102 return 0;
8103 case GEN6_PCODE_UNIMPLEMENTED_CMD:
8104 case GEN6_PCODE_ILLEGAL_CMD:
8105 return -ENXIO;
8106 case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
8107 case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
8108 return -EOVERFLOW;
8109 case GEN6_PCODE_TIMEOUT:
8110 return -ETIMEDOUT;
8111 default:
8112 MISSING_CASE(flags);
8113 return 0;
8114 }
8115}
8116
8117static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
8118{
8119 uint32_t flags =
8120 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
8121
8122 switch (flags) {
8123 case GEN6_PCODE_SUCCESS:
8124 return 0;
8125 case GEN6_PCODE_ILLEGAL_CMD:
8126 return -ENXIO;
8127 case GEN7_PCODE_TIMEOUT:
8128 return -ETIMEDOUT;
8129 case GEN7_PCODE_ILLEGAL_DATA:
8130 return -EINVAL;
8131 case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
8132 return -EOVERFLOW;
8133 default:
8134 MISSING_CASE(flags);
8135 return 0;
8136 }
8137}
8138
8139int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
8140{
8141 int status;
8142
8143 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
8144
8145
8146
8147
8148
8149
8150 if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
8151 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
8152 return -EAGAIN;
8153 }
8154
8155 I915_WRITE_FW(GEN6_PCODE_DATA, *val);
8156 I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
8157 I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
8158
8159 if (intel_wait_for_register_fw(dev_priv,
8160 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
8161 500)) {
8162 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
8163 return -ETIMEDOUT;
8164 }
8165
8166 *val = I915_READ_FW(GEN6_PCODE_DATA);
8167 I915_WRITE_FW(GEN6_PCODE_DATA, 0);
8168
8169 if (INTEL_GEN(dev_priv) > 6)
8170 status = gen7_check_mailbox_status(dev_priv);
8171 else
8172 status = gen6_check_mailbox_status(dev_priv);
8173
8174 if (status) {
8175 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed: %d\n",
8176 status);
8177 return status;
8178 }
8179
8180 return 0;
8181}
8182
8183int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
8184 u32 mbox, u32 val)
8185{
8186 int status;
8187
8188 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
8189
8190
8191
8192
8193
8194
8195 if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
8196 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
8197 return -EAGAIN;
8198 }
8199
8200 I915_WRITE_FW(GEN6_PCODE_DATA, val);
8201 I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
8202 I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
8203
8204 if (intel_wait_for_register_fw(dev_priv,
8205 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
8206 500)) {
8207 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
8208 return -ETIMEDOUT;
8209 }
8210
8211 I915_WRITE_FW(GEN6_PCODE_DATA, 0);
8212
8213 if (INTEL_GEN(dev_priv) > 6)
8214 status = gen7_check_mailbox_status(dev_priv);
8215 else
8216 status = gen6_check_mailbox_status(dev_priv);
8217
8218 if (status) {
8219 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed: %d\n",
8220 status);
8221 return status;
8222 }
8223
8224 return 0;
8225}
8226
8227static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox,
8228 u32 request, u32 reply_mask, u32 reply,
8229 u32 *status)
8230{
8231 u32 val = request;
8232
8233 *status = sandybridge_pcode_read(dev_priv, mbox, &val);
8234
8235 return *status || ((val & reply_mask) == reply);
8236}
8237
8238
8239
8240
8241
8242
8243
8244
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
8258 u32 reply_mask, u32 reply, int timeout_base_ms)
8259{
8260 u32 status;
8261 int ret;
8262
8263 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
8264
8265#define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \
8266 &status)
8267
8268
8269
8270
8271
8272
8273
8274 if (COND) {
8275 ret = 0;
8276 goto out;
8277 }
8278 ret = _wait_for(COND, timeout_base_ms * 1000, 10);
8279 if (!ret)
8280 goto out;
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292 DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n");
8293 WARN_ON_ONCE(timeout_base_ms > 3);
8294 preempt_disable();
8295 ret = wait_for_atomic(COND, 50);
8296 preempt_enable();
8297
8298out:
8299 return ret ? ret : status;
8300#undef COND
8301}
8302
8303static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
8304{
8305
8306
8307
8308
8309 return DIV_ROUND_CLOSEST(dev_priv->rps.gpll_ref_freq * (val - 0xb7), 1000);
8310}
8311
8312static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
8313{
8314 return DIV_ROUND_CLOSEST(1000 * val, dev_priv->rps.gpll_ref_freq) + 0xb7;
8315}
8316
8317static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
8318{
8319
8320
8321
8322
8323 return DIV_ROUND_CLOSEST(dev_priv->rps.gpll_ref_freq * val, 2 * 2 * 1000);
8324}
8325
8326static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
8327{
8328
8329 return DIV_ROUND_CLOSEST(2 * 1000 * val, dev_priv->rps.gpll_ref_freq) * 2;
8330}
8331
8332int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
8333{
8334 if (IS_GEN9(dev_priv))
8335 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
8336 GEN9_FREQ_SCALER);
8337 else if (IS_CHERRYVIEW(dev_priv))
8338 return chv_gpu_freq(dev_priv, val);
8339 else if (IS_VALLEYVIEW(dev_priv))
8340 return byt_gpu_freq(dev_priv, val);
8341 else
8342 return val * GT_FREQUENCY_MULTIPLIER;
8343}
8344
8345int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
8346{
8347 if (IS_GEN9(dev_priv))
8348 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
8349 GT_FREQUENCY_MULTIPLIER);
8350 else if (IS_CHERRYVIEW(dev_priv))
8351 return chv_freq_opcode(dev_priv, val);
8352 else if (IS_VALLEYVIEW(dev_priv))
8353 return byt_freq_opcode(dev_priv, val);
8354 else
8355 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
8356}
8357
8358struct request_boost {
8359 struct work_struct work;
8360 struct drm_i915_gem_request *req;
8361};
8362
8363static void __intel_rps_boost_work(struct work_struct *work)
8364{
8365 struct request_boost *boost = container_of(work, struct request_boost, work);
8366 struct drm_i915_gem_request *req = boost->req;
8367
8368 if (!i915_gem_request_completed(req))
8369 gen6_rps_boost(req->i915, NULL, req->emitted_jiffies);
8370
8371 i915_gem_request_put(req);
8372 kfree(boost);
8373}
8374
8375void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req)
8376{
8377 struct request_boost *boost;
8378
8379 if (req == NULL || INTEL_GEN(req->i915) < 6)
8380 return;
8381
8382 if (i915_gem_request_completed(req))
8383 return;
8384
8385 boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
8386 if (boost == NULL)
8387 return;
8388
8389 boost->req = i915_gem_request_get(req);
8390
8391 INIT_WORK(&boost->work, __intel_rps_boost_work);
8392 queue_work(req->i915->wq, &boost->work);
8393}
8394
8395void intel_pm_setup(struct drm_i915_private *dev_priv)
8396{
8397 mutex_init(&dev_priv->rps.hw_lock);
8398 spin_lock_init(&dev_priv->rps.client_lock);
8399
8400 INIT_DELAYED_WORK(&dev_priv->rps.autoenable_work,
8401 __intel_autoenable_gt_powersave);
8402 INIT_LIST_HEAD(&dev_priv->rps.clients);
8403
8404 dev_priv->pm.suspended = false;
8405 atomic_set(&dev_priv->pm.wakeref_count, 0);
8406}
8407
8408static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
8409 const i915_reg_t reg)
8410{
8411 u32 lower, upper, tmp;
8412 int loop = 2;
8413
8414
8415
8416
8417 spin_lock_irq(&dev_priv->uncore.lock);
8418
8419
8420
8421
8422
8423
8424
8425
8426
8427
8428 I915_WRITE_FW(VLV_COUNTER_CONTROL,
8429 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
8430 upper = I915_READ_FW(reg);
8431 do {
8432 tmp = upper;
8433
8434 I915_WRITE_FW(VLV_COUNTER_CONTROL,
8435 _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
8436 lower = I915_READ_FW(reg);
8437
8438 I915_WRITE_FW(VLV_COUNTER_CONTROL,
8439 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
8440 upper = I915_READ_FW(reg);
8441 } while (upper != tmp && --loop);
8442
8443
8444
8445
8446
8447
8448 spin_unlock_irq(&dev_priv->uncore.lock);
8449
8450 return lower | (u64)upper << 8;
8451}
8452
8453u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
8454 const i915_reg_t reg)
8455{
8456 u64 time_hw, units, div;
8457
8458 if (!intel_enable_rc6())
8459 return 0;
8460
8461 intel_runtime_pm_get(dev_priv);
8462
8463
8464 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8465 units = 1000;
8466 div = dev_priv->czclk_freq;
8467
8468 time_hw = vlv_residency_raw(dev_priv, reg);
8469 } else if (IS_GEN9_LP(dev_priv)) {
8470 units = 1000;
8471 div = 1200;
8472
8473 time_hw = I915_READ(reg);
8474 } else {
8475 units = 128000;
8476 div = 100000;
8477
8478 time_hw = I915_READ(reg);
8479 }
8480
8481 intel_runtime_pm_put(dev_priv);
8482 return DIV_ROUND_UP_ULL(time_hw * units, div);
8483}
8484