1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/cpufreq.h>
29#include "i915_drv.h"
30#include "intel_drv.h"
31#include "../../../platform/x86/intel_ips.h"
32#include <linux/module.h>
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53#define INTEL_RC6_ENABLE (1<<0)
54#define INTEL_RC6p_ENABLE (1<<1)
55#define INTEL_RC6pp_ENABLE (1<<2)
56
57static void bxt_init_clock_gating(struct drm_device *dev)
58{
59 struct drm_i915_private *dev_priv = dev->dev_private;
60
61
62 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
63 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
64
65
66
67
68
69 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
70 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
71
72
73
74
75
76 if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
77 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
78 PWM1_GATING_DIS | PWM2_GATING_DIS);
79}
80
81static void i915_pineview_get_mem_freq(struct drm_device *dev)
82{
83 struct drm_i915_private *dev_priv = dev->dev_private;
84 u32 tmp;
85
86 tmp = I915_READ(CLKCFG);
87
88 switch (tmp & CLKCFG_FSB_MASK) {
89 case CLKCFG_FSB_533:
90 dev_priv->fsb_freq = 533;
91 break;
92 case CLKCFG_FSB_800:
93 dev_priv->fsb_freq = 800;
94 break;
95 case CLKCFG_FSB_667:
96 dev_priv->fsb_freq = 667;
97 break;
98 case CLKCFG_FSB_400:
99 dev_priv->fsb_freq = 400;
100 break;
101 }
102
103 switch (tmp & CLKCFG_MEM_MASK) {
104 case CLKCFG_MEM_533:
105 dev_priv->mem_freq = 533;
106 break;
107 case CLKCFG_MEM_667:
108 dev_priv->mem_freq = 667;
109 break;
110 case CLKCFG_MEM_800:
111 dev_priv->mem_freq = 800;
112 break;
113 }
114
115
116 tmp = I915_READ(CSHRDDR3CTL);
117 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
118}
119
120static void i915_ironlake_get_mem_freq(struct drm_device *dev)
121{
122 struct drm_i915_private *dev_priv = dev->dev_private;
123 u16 ddrpll, csipll;
124
125 ddrpll = I915_READ16(DDRMPLL1);
126 csipll = I915_READ16(CSIPLL0);
127
128 switch (ddrpll & 0xff) {
129 case 0xc:
130 dev_priv->mem_freq = 800;
131 break;
132 case 0x10:
133 dev_priv->mem_freq = 1066;
134 break;
135 case 0x14:
136 dev_priv->mem_freq = 1333;
137 break;
138 case 0x18:
139 dev_priv->mem_freq = 1600;
140 break;
141 default:
142 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
143 ddrpll & 0xff);
144 dev_priv->mem_freq = 0;
145 break;
146 }
147
148 dev_priv->ips.r_t = dev_priv->mem_freq;
149
150 switch (csipll & 0x3ff) {
151 case 0x00c:
152 dev_priv->fsb_freq = 3200;
153 break;
154 case 0x00e:
155 dev_priv->fsb_freq = 3733;
156 break;
157 case 0x010:
158 dev_priv->fsb_freq = 4266;
159 break;
160 case 0x012:
161 dev_priv->fsb_freq = 4800;
162 break;
163 case 0x014:
164 dev_priv->fsb_freq = 5333;
165 break;
166 case 0x016:
167 dev_priv->fsb_freq = 5866;
168 break;
169 case 0x018:
170 dev_priv->fsb_freq = 6400;
171 break;
172 default:
173 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
174 csipll & 0x3ff);
175 dev_priv->fsb_freq = 0;
176 break;
177 }
178
179 if (dev_priv->fsb_freq == 3200) {
180 dev_priv->ips.c_m = 0;
181 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
182 dev_priv->ips.c_m = 1;
183 } else {
184 dev_priv->ips.c_m = 2;
185 }
186}
187
188static const struct cxsr_latency cxsr_latency_table[] = {
189 {1, 0, 800, 400, 3382, 33382, 3983, 33983},
190 {1, 0, 800, 667, 3354, 33354, 3807, 33807},
191 {1, 0, 800, 800, 3347, 33347, 3763, 33763},
192 {1, 1, 800, 667, 6420, 36420, 6873, 36873},
193 {1, 1, 800, 800, 5902, 35902, 6318, 36318},
194
195 {1, 0, 667, 400, 3400, 33400, 4021, 34021},
196 {1, 0, 667, 667, 3372, 33372, 3845, 33845},
197 {1, 0, 667, 800, 3386, 33386, 3822, 33822},
198 {1, 1, 667, 667, 6438, 36438, 6911, 36911},
199 {1, 1, 667, 800, 5941, 35941, 6377, 36377},
200
201 {1, 0, 400, 400, 3472, 33472, 4173, 34173},
202 {1, 0, 400, 667, 3443, 33443, 3996, 33996},
203 {1, 0, 400, 800, 3430, 33430, 3946, 33946},
204 {1, 1, 400, 667, 6509, 36509, 7062, 37062},
205 {1, 1, 400, 800, 5985, 35985, 6501, 36501},
206
207 {0, 0, 800, 400, 3438, 33438, 4065, 34065},
208 {0, 0, 800, 667, 3410, 33410, 3889, 33889},
209 {0, 0, 800, 800, 3403, 33403, 3845, 33845},
210 {0, 1, 800, 667, 6476, 36476, 6955, 36955},
211 {0, 1, 800, 800, 5958, 35958, 6400, 36400},
212
213 {0, 0, 667, 400, 3456, 33456, 4103, 34106},
214 {0, 0, 667, 667, 3428, 33428, 3927, 33927},
215 {0, 0, 667, 800, 3443, 33443, 3905, 33905},
216 {0, 1, 667, 667, 6494, 36494, 6993, 36993},
217 {0, 1, 667, 800, 5998, 35998, 6460, 36460},
218
219 {0, 0, 400, 400, 3528, 33528, 4255, 34255},
220 {0, 0, 400, 667, 3500, 33500, 4079, 34079},
221 {0, 0, 400, 800, 3487, 33487, 4029, 34029},
222 {0, 1, 400, 667, 6566, 36566, 7145, 37145},
223 {0, 1, 400, 800, 6042, 36042, 6584, 36584},
224};
225
226static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
227 int is_ddr3,
228 int fsb,
229 int mem)
230{
231 const struct cxsr_latency *latency;
232 int i;
233
234 if (fsb == 0 || mem == 0)
235 return NULL;
236
237 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
238 latency = &cxsr_latency_table[i];
239 if (is_desktop == latency->is_desktop &&
240 is_ddr3 == latency->is_ddr3 &&
241 fsb == latency->fsb_freq && mem == latency->mem_freq)
242 return latency;
243 }
244
245 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
246
247 return NULL;
248}
249
250static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
251{
252 u32 val;
253
254 mutex_lock(&dev_priv->rps.hw_lock);
255
256 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
257 if (enable)
258 val &= ~FORCE_DDR_HIGH_FREQ;
259 else
260 val |= FORCE_DDR_HIGH_FREQ;
261 val &= ~FORCE_DDR_LOW_FREQ;
262 val |= FORCE_DDR_FREQ_REQ_ACK;
263 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
264
265 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
266 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
267 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
268
269 mutex_unlock(&dev_priv->rps.hw_lock);
270}
271
272static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
273{
274 u32 val;
275
276 mutex_lock(&dev_priv->rps.hw_lock);
277
278 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
279 if (enable)
280 val |= DSP_MAXFIFO_PM5_ENABLE;
281 else
282 val &= ~DSP_MAXFIFO_PM5_ENABLE;
283 vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
284
285 mutex_unlock(&dev_priv->rps.hw_lock);
286}
287
288#define FW_WM(value, plane) \
289 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
290
291void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
292{
293 struct drm_device *dev = dev_priv->dev;
294 u32 val;
295
296 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
297 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
298 POSTING_READ(FW_BLC_SELF_VLV);
299 dev_priv->wm.vlv.cxsr = enable;
300 } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
301 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
302 POSTING_READ(FW_BLC_SELF);
303 } else if (IS_PINEVIEW(dev)) {
304 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
305 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
306 I915_WRITE(DSPFW3, val);
307 POSTING_READ(DSPFW3);
308 } else if (IS_I945G(dev) || IS_I945GM(dev)) {
309 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
310 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
311 I915_WRITE(FW_BLC_SELF, val);
312 POSTING_READ(FW_BLC_SELF);
313 } else if (IS_I915GM(dev)) {
314 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
315 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
316 I915_WRITE(INSTPM, val);
317 POSTING_READ(INSTPM);
318 } else {
319 return;
320 }
321
322 DRM_DEBUG_KMS("memory self-refresh is %s\n",
323 enable ? "enabled" : "disabled");
324}
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341static const int pessimal_latency_ns = 5000;
342
343#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
344 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
345
346static int vlv_get_fifo_size(struct drm_device *dev,
347 enum pipe pipe, int plane)
348{
349 struct drm_i915_private *dev_priv = dev->dev_private;
350 int sprite0_start, sprite1_start, size;
351
352 switch (pipe) {
353 uint32_t dsparb, dsparb2, dsparb3;
354 case PIPE_A:
355 dsparb = I915_READ(DSPARB);
356 dsparb2 = I915_READ(DSPARB2);
357 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
358 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
359 break;
360 case PIPE_B:
361 dsparb = I915_READ(DSPARB);
362 dsparb2 = I915_READ(DSPARB2);
363 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
364 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
365 break;
366 case PIPE_C:
367 dsparb2 = I915_READ(DSPARB2);
368 dsparb3 = I915_READ(DSPARB3);
369 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
370 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
371 break;
372 default:
373 return 0;
374 }
375
376 switch (plane) {
377 case 0:
378 size = sprite0_start;
379 break;
380 case 1:
381 size = sprite1_start - sprite0_start;
382 break;
383 case 2:
384 size = 512 - 1 - sprite1_start;
385 break;
386 default:
387 return 0;
388 }
389
390 DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
391 pipe_name(pipe), plane == 0 ? "primary" : "sprite",
392 plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
393 size);
394
395 return size;
396}
397
398static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
399{
400 struct drm_i915_private *dev_priv = dev->dev_private;
401 uint32_t dsparb = I915_READ(DSPARB);
402 int size;
403
404 size = dsparb & 0x7f;
405 if (plane)
406 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
407
408 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
409 plane ? "B" : "A", size);
410
411 return size;
412}
413
414static int i830_get_fifo_size(struct drm_device *dev, int plane)
415{
416 struct drm_i915_private *dev_priv = dev->dev_private;
417 uint32_t dsparb = I915_READ(DSPARB);
418 int size;
419
420 size = dsparb & 0x1ff;
421 if (plane)
422 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
423 size >>= 1;
424
425 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
426 plane ? "B" : "A", size);
427
428 return size;
429}
430
431static int i845_get_fifo_size(struct drm_device *dev, int plane)
432{
433 struct drm_i915_private *dev_priv = dev->dev_private;
434 uint32_t dsparb = I915_READ(DSPARB);
435 int size;
436
437 size = dsparb & 0x7f;
438 size >>= 2;
439
440 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
441 plane ? "B" : "A",
442 size);
443
444 return size;
445}
446
447
448static const struct intel_watermark_params pineview_display_wm = {
449 .fifo_size = PINEVIEW_DISPLAY_FIFO,
450 .max_wm = PINEVIEW_MAX_WM,
451 .default_wm = PINEVIEW_DFT_WM,
452 .guard_size = PINEVIEW_GUARD_WM,
453 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
454};
455static const struct intel_watermark_params pineview_display_hplloff_wm = {
456 .fifo_size = PINEVIEW_DISPLAY_FIFO,
457 .max_wm = PINEVIEW_MAX_WM,
458 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
459 .guard_size = PINEVIEW_GUARD_WM,
460 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
461};
462static const struct intel_watermark_params pineview_cursor_wm = {
463 .fifo_size = PINEVIEW_CURSOR_FIFO,
464 .max_wm = PINEVIEW_CURSOR_MAX_WM,
465 .default_wm = PINEVIEW_CURSOR_DFT_WM,
466 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
467 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
468};
469static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
470 .fifo_size = PINEVIEW_CURSOR_FIFO,
471 .max_wm = PINEVIEW_CURSOR_MAX_WM,
472 .default_wm = PINEVIEW_CURSOR_DFT_WM,
473 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
474 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
475};
476static const struct intel_watermark_params g4x_wm_info = {
477 .fifo_size = G4X_FIFO_SIZE,
478 .max_wm = G4X_MAX_WM,
479 .default_wm = G4X_MAX_WM,
480 .guard_size = 2,
481 .cacheline_size = G4X_FIFO_LINE_SIZE,
482};
483static const struct intel_watermark_params g4x_cursor_wm_info = {
484 .fifo_size = I965_CURSOR_FIFO,
485 .max_wm = I965_CURSOR_MAX_WM,
486 .default_wm = I965_CURSOR_DFT_WM,
487 .guard_size = 2,
488 .cacheline_size = G4X_FIFO_LINE_SIZE,
489};
490static const struct intel_watermark_params valleyview_wm_info = {
491 .fifo_size = VALLEYVIEW_FIFO_SIZE,
492 .max_wm = VALLEYVIEW_MAX_WM,
493 .default_wm = VALLEYVIEW_MAX_WM,
494 .guard_size = 2,
495 .cacheline_size = G4X_FIFO_LINE_SIZE,
496};
497static const struct intel_watermark_params valleyview_cursor_wm_info = {
498 .fifo_size = I965_CURSOR_FIFO,
499 .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
500 .default_wm = I965_CURSOR_DFT_WM,
501 .guard_size = 2,
502 .cacheline_size = G4X_FIFO_LINE_SIZE,
503};
504static const struct intel_watermark_params i965_cursor_wm_info = {
505 .fifo_size = I965_CURSOR_FIFO,
506 .max_wm = I965_CURSOR_MAX_WM,
507 .default_wm = I965_CURSOR_DFT_WM,
508 .guard_size = 2,
509 .cacheline_size = I915_FIFO_LINE_SIZE,
510};
511static const struct intel_watermark_params i945_wm_info = {
512 .fifo_size = I945_FIFO_SIZE,
513 .max_wm = I915_MAX_WM,
514 .default_wm = 1,
515 .guard_size = 2,
516 .cacheline_size = I915_FIFO_LINE_SIZE,
517};
518static const struct intel_watermark_params i915_wm_info = {
519 .fifo_size = I915_FIFO_SIZE,
520 .max_wm = I915_MAX_WM,
521 .default_wm = 1,
522 .guard_size = 2,
523 .cacheline_size = I915_FIFO_LINE_SIZE,
524};
525static const struct intel_watermark_params i830_a_wm_info = {
526 .fifo_size = I855GM_FIFO_SIZE,
527 .max_wm = I915_MAX_WM,
528 .default_wm = 1,
529 .guard_size = 2,
530 .cacheline_size = I830_FIFO_LINE_SIZE,
531};
532static const struct intel_watermark_params i830_bc_wm_info = {
533 .fifo_size = I855GM_FIFO_SIZE,
534 .max_wm = I915_MAX_WM/2,
535 .default_wm = 1,
536 .guard_size = 2,
537 .cacheline_size = I830_FIFO_LINE_SIZE,
538};
539static const struct intel_watermark_params i845_wm_info = {
540 .fifo_size = I830_FIFO_SIZE,
541 .max_wm = I915_MAX_WM,
542 .default_wm = 1,
543 .guard_size = 2,
544 .cacheline_size = I830_FIFO_LINE_SIZE,
545};
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
566 const struct intel_watermark_params *wm,
567 int fifo_size, int cpp,
568 unsigned long latency_ns)
569{
570 long entries_required, wm_size;
571
572
573
574
575
576
577
578 entries_required = ((clock_in_khz / 1000) * cpp * latency_ns) /
579 1000;
580 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
581
582 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
583
584 wm_size = fifo_size - (entries_required + wm->guard_size);
585
586 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
587
588
589 if (wm_size > (long)wm->max_wm)
590 wm_size = wm->max_wm;
591 if (wm_size <= 0)
592 wm_size = wm->default_wm;
593
594
595
596
597
598
599
600
601 if (wm_size <= 8)
602 wm_size = 8;
603
604 return wm_size;
605}
606
607static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
608{
609 struct drm_crtc *crtc, *enabled = NULL;
610
611 for_each_crtc(dev, crtc) {
612 if (intel_crtc_active(crtc)) {
613 if (enabled)
614 return NULL;
615 enabled = crtc;
616 }
617 }
618
619 return enabled;
620}
621
622static void pineview_update_wm(struct drm_crtc *unused_crtc)
623{
624 struct drm_device *dev = unused_crtc->dev;
625 struct drm_i915_private *dev_priv = dev->dev_private;
626 struct drm_crtc *crtc;
627 const struct cxsr_latency *latency;
628 u32 reg;
629 unsigned long wm;
630
631 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
632 dev_priv->fsb_freq, dev_priv->mem_freq);
633 if (!latency) {
634 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
635 intel_set_memory_cxsr(dev_priv, false);
636 return;
637 }
638
639 crtc = single_enabled_crtc(dev);
640 if (crtc) {
641 const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
642 int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
643 int clock = adjusted_mode->crtc_clock;
644
645
646 wm = intel_calculate_wm(clock, &pineview_display_wm,
647 pineview_display_wm.fifo_size,
648 cpp, latency->display_sr);
649 reg = I915_READ(DSPFW1);
650 reg &= ~DSPFW_SR_MASK;
651 reg |= FW_WM(wm, SR);
652 I915_WRITE(DSPFW1, reg);
653 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
654
655
656 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
657 pineview_display_wm.fifo_size,
658 cpp, latency->cursor_sr);
659 reg = I915_READ(DSPFW3);
660 reg &= ~DSPFW_CURSOR_SR_MASK;
661 reg |= FW_WM(wm, CURSOR_SR);
662 I915_WRITE(DSPFW3, reg);
663
664
665 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
666 pineview_display_hplloff_wm.fifo_size,
667 cpp, latency->display_hpll_disable);
668 reg = I915_READ(DSPFW3);
669 reg &= ~DSPFW_HPLL_SR_MASK;
670 reg |= FW_WM(wm, HPLL_SR);
671 I915_WRITE(DSPFW3, reg);
672
673
674 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
675 pineview_display_hplloff_wm.fifo_size,
676 cpp, latency->cursor_hpll_disable);
677 reg = I915_READ(DSPFW3);
678 reg &= ~DSPFW_HPLL_CURSOR_MASK;
679 reg |= FW_WM(wm, HPLL_CURSOR);
680 I915_WRITE(DSPFW3, reg);
681 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
682
683 intel_set_memory_cxsr(dev_priv, true);
684 } else {
685 intel_set_memory_cxsr(dev_priv, false);
686 }
687}
688
689static bool g4x_compute_wm0(struct drm_device *dev,
690 int plane,
691 const struct intel_watermark_params *display,
692 int display_latency_ns,
693 const struct intel_watermark_params *cursor,
694 int cursor_latency_ns,
695 int *plane_wm,
696 int *cursor_wm)
697{
698 struct drm_crtc *crtc;
699 const struct drm_display_mode *adjusted_mode;
700 int htotal, hdisplay, clock, cpp;
701 int line_time_us, line_count;
702 int entries, tlb_miss;
703
704 crtc = intel_get_crtc_for_plane(dev, plane);
705 if (!intel_crtc_active(crtc)) {
706 *cursor_wm = cursor->guard_size;
707 *plane_wm = display->guard_size;
708 return false;
709 }
710
711 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
712 clock = adjusted_mode->crtc_clock;
713 htotal = adjusted_mode->crtc_htotal;
714 hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
715 cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
716
717
718 entries = ((clock * cpp / 1000) * display_latency_ns) / 1000;
719 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
720 if (tlb_miss > 0)
721 entries += tlb_miss;
722 entries = DIV_ROUND_UP(entries, display->cacheline_size);
723 *plane_wm = entries + display->guard_size;
724 if (*plane_wm > (int)display->max_wm)
725 *plane_wm = display->max_wm;
726
727
728 line_time_us = max(htotal * 1000 / clock, 1);
729 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
730 entries = line_count * crtc->cursor->state->crtc_w * cpp;
731 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
732 if (tlb_miss > 0)
733 entries += tlb_miss;
734 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
735 *cursor_wm = entries + cursor->guard_size;
736 if (*cursor_wm > (int)cursor->max_wm)
737 *cursor_wm = (int)cursor->max_wm;
738
739 return true;
740}
741
742
743
744
745
746
747
748
749static bool g4x_check_srwm(struct drm_device *dev,
750 int display_wm, int cursor_wm,
751 const struct intel_watermark_params *display,
752 const struct intel_watermark_params *cursor)
753{
754 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
755 display_wm, cursor_wm);
756
757 if (display_wm > display->max_wm) {
758 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
759 display_wm, display->max_wm);
760 return false;
761 }
762
763 if (cursor_wm > cursor->max_wm) {
764 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
765 cursor_wm, cursor->max_wm);
766 return false;
767 }
768
769 if (!(display_wm || cursor_wm)) {
770 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
771 return false;
772 }
773
774 return true;
775}
776
777static bool g4x_compute_srwm(struct drm_device *dev,
778 int plane,
779 int latency_ns,
780 const struct intel_watermark_params *display,
781 const struct intel_watermark_params *cursor,
782 int *display_wm, int *cursor_wm)
783{
784 struct drm_crtc *crtc;
785 const struct drm_display_mode *adjusted_mode;
786 int hdisplay, htotal, cpp, clock;
787 unsigned long line_time_us;
788 int line_count, line_size;
789 int small, large;
790 int entries;
791
792 if (!latency_ns) {
793 *display_wm = *cursor_wm = 0;
794 return false;
795 }
796
797 crtc = intel_get_crtc_for_plane(dev, plane);
798 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
799 clock = adjusted_mode->crtc_clock;
800 htotal = adjusted_mode->crtc_htotal;
801 hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
802 cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
803
804 line_time_us = max(htotal * 1000 / clock, 1);
805 line_count = (latency_ns / line_time_us + 1000) / 1000;
806 line_size = hdisplay * cpp;
807
808
809 small = ((clock * cpp / 1000) * latency_ns) / 1000;
810 large = line_count * line_size;
811
812 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
813 *display_wm = entries + display->guard_size;
814
815
816 entries = line_count * cpp * crtc->cursor->state->crtc_w;
817 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
818 *cursor_wm = entries + cursor->guard_size;
819
820 return g4x_check_srwm(dev,
821 *display_wm, *cursor_wm,
822 display, cursor);
823}
824
825#define FW_WM_VLV(value, plane) \
826 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
827
828static void vlv_write_wm_values(struct intel_crtc *crtc,
829 const struct vlv_wm_values *wm)
830{
831 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
832 enum pipe pipe = crtc->pipe;
833
834 I915_WRITE(VLV_DDL(pipe),
835 (wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) |
836 (wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) |
837 (wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) |
838 (wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
839
840 I915_WRITE(DSPFW1,
841 FW_WM(wm->sr.plane, SR) |
842 FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) |
843 FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) |
844 FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
845 I915_WRITE(DSPFW2,
846 FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) |
847 FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) |
848 FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
849 I915_WRITE(DSPFW3,
850 FW_WM(wm->sr.cursor, CURSOR_SR));
851
852 if (IS_CHERRYVIEW(dev_priv)) {
853 I915_WRITE(DSPFW7_CHV,
854 FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) |
855 FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
856 I915_WRITE(DSPFW8_CHV,
857 FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) |
858 FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
859 I915_WRITE(DSPFW9_CHV,
860 FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) |
861 FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
862 I915_WRITE(DSPHOWM,
863 FW_WM(wm->sr.plane >> 9, SR_HI) |
864 FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) |
865 FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) |
866 FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) |
867 FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) |
868 FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) |
869 FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) |
870 FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) |
871 FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) |
872 FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
873 } else {
874 I915_WRITE(DSPFW7,
875 FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) |
876 FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
877 I915_WRITE(DSPHOWM,
878 FW_WM(wm->sr.plane >> 9, SR_HI) |
879 FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) |
880 FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) |
881 FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) |
882 FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) |
883 FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) |
884 FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
885 }
886
887
888 I915_WRITE(DSPFW4, 0);
889 I915_WRITE(DSPFW5, 0);
890 I915_WRITE(DSPFW6, 0);
891 I915_WRITE(DSPHOWM1, 0);
892
893 POSTING_READ(DSPFW1);
894}
895
896#undef FW_WM_VLV
897
898enum vlv_wm_level {
899 VLV_WM_LEVEL_PM2,
900 VLV_WM_LEVEL_PM5,
901 VLV_WM_LEVEL_DDR_DVFS,
902};
903
904
905static unsigned int vlv_wm_method2(unsigned int pixel_rate,
906 unsigned int pipe_htotal,
907 unsigned int horiz_pixels,
908 unsigned int cpp,
909 unsigned int latency)
910{
911 unsigned int ret;
912
913 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
914 ret = (ret + 1) * horiz_pixels * cpp;
915 ret = DIV_ROUND_UP(ret, 64);
916
917 return ret;
918}
919
920static void vlv_setup_wm_latency(struct drm_device *dev)
921{
922 struct drm_i915_private *dev_priv = dev->dev_private;
923
924
925 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
926
927 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
928
929 if (IS_CHERRYVIEW(dev_priv)) {
930 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
931 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
932
933 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
934 }
935}
936
937static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
938 struct intel_crtc *crtc,
939 const struct intel_plane_state *state,
940 int level)
941{
942 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
943 int clock, htotal, cpp, width, wm;
944
945 if (dev_priv->wm.pri_latency[level] == 0)
946 return USHRT_MAX;
947
948 if (!state->visible)
949 return 0;
950
951 cpp = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
952 clock = crtc->config->base.adjusted_mode.crtc_clock;
953 htotal = crtc->config->base.adjusted_mode.crtc_htotal;
954 width = crtc->config->pipe_src_w;
955 if (WARN_ON(htotal == 0))
956 htotal = 1;
957
958 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
959
960
961
962
963
964
965 wm = 63;
966 } else {
967 wm = vlv_wm_method2(clock, htotal, width, cpp,
968 dev_priv->wm.pri_latency[level] * 10);
969 }
970
971 return min_t(int, wm, USHRT_MAX);
972}
973
974static void vlv_compute_fifo(struct intel_crtc *crtc)
975{
976 struct drm_device *dev = crtc->base.dev;
977 struct vlv_wm_state *wm_state = &crtc->wm_state;
978 struct intel_plane *plane;
979 unsigned int total_rate = 0;
980 const int fifo_size = 512 - 1;
981 int fifo_extra, fifo_left = fifo_size;
982
983 for_each_intel_plane_on_crtc(dev, crtc, plane) {
984 struct intel_plane_state *state =
985 to_intel_plane_state(plane->base.state);
986
987 if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
988 continue;
989
990 if (state->visible) {
991 wm_state->num_active_planes++;
992 total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
993 }
994 }
995
996 for_each_intel_plane_on_crtc(dev, crtc, plane) {
997 struct intel_plane_state *state =
998 to_intel_plane_state(plane->base.state);
999 unsigned int rate;
1000
1001 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
1002 plane->wm.fifo_size = 63;
1003 continue;
1004 }
1005
1006 if (!state->visible) {
1007 plane->wm.fifo_size = 0;
1008 continue;
1009 }
1010
1011 rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
1012 plane->wm.fifo_size = fifo_size * rate / total_rate;
1013 fifo_left -= plane->wm.fifo_size;
1014 }
1015
1016 fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
1017
1018
1019 for_each_intel_plane_on_crtc(dev, crtc, plane) {
1020 int plane_extra;
1021
1022 if (fifo_left == 0)
1023 break;
1024
1025 if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
1026 continue;
1027
1028
1029 if (plane->wm.fifo_size == 0 &&
1030 wm_state->num_active_planes)
1031 continue;
1032
1033 plane_extra = min(fifo_extra, fifo_left);
1034 plane->wm.fifo_size += plane_extra;
1035 fifo_left -= plane_extra;
1036 }
1037
1038 WARN_ON(fifo_left != 0);
1039}
1040
1041static void vlv_invert_wms(struct intel_crtc *crtc)
1042{
1043 struct vlv_wm_state *wm_state = &crtc->wm_state;
1044 int level;
1045
1046 for (level = 0; level < wm_state->num_levels; level++) {
1047 struct drm_device *dev = crtc->base.dev;
1048 const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
1049 struct intel_plane *plane;
1050
1051 wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
1052 wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
1053
1054 for_each_intel_plane_on_crtc(dev, crtc, plane) {
1055 switch (plane->base.type) {
1056 int sprite;
1057 case DRM_PLANE_TYPE_CURSOR:
1058 wm_state->wm[level].cursor = plane->wm.fifo_size -
1059 wm_state->wm[level].cursor;
1060 break;
1061 case DRM_PLANE_TYPE_PRIMARY:
1062 wm_state->wm[level].primary = plane->wm.fifo_size -
1063 wm_state->wm[level].primary;
1064 break;
1065 case DRM_PLANE_TYPE_OVERLAY:
1066 sprite = plane->plane;
1067 wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
1068 wm_state->wm[level].sprite[sprite];
1069 break;
1070 }
1071 }
1072 }
1073}
1074
1075static void vlv_compute_wm(struct intel_crtc *crtc)
1076{
1077 struct drm_device *dev = crtc->base.dev;
1078 struct vlv_wm_state *wm_state = &crtc->wm_state;
1079 struct intel_plane *plane;
1080 int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
1081 int level;
1082
1083 memset(wm_state, 0, sizeof(*wm_state));
1084
1085 wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
1086 wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
1087
1088 wm_state->num_active_planes = 0;
1089
1090 vlv_compute_fifo(crtc);
1091
1092 if (wm_state->num_active_planes != 1)
1093 wm_state->cxsr = false;
1094
1095 if (wm_state->cxsr) {
1096 for (level = 0; level < wm_state->num_levels; level++) {
1097 wm_state->sr[level].plane = sr_fifo_size;
1098 wm_state->sr[level].cursor = 63;
1099 }
1100 }
1101
1102 for_each_intel_plane_on_crtc(dev, crtc, plane) {
1103 struct intel_plane_state *state =
1104 to_intel_plane_state(plane->base.state);
1105
1106 if (!state->visible)
1107 continue;
1108
1109
1110 for (level = 0; level < wm_state->num_levels; level++) {
1111 int wm = vlv_compute_wm_level(plane, crtc, state, level);
1112 int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
1113
1114
1115 if (WARN_ON(level == 0 && wm > max_wm))
1116 wm = max_wm;
1117
1118 if (wm > plane->wm.fifo_size)
1119 break;
1120
1121 switch (plane->base.type) {
1122 int sprite;
1123 case DRM_PLANE_TYPE_CURSOR:
1124 wm_state->wm[level].cursor = wm;
1125 break;
1126 case DRM_PLANE_TYPE_PRIMARY:
1127 wm_state->wm[level].primary = wm;
1128 break;
1129 case DRM_PLANE_TYPE_OVERLAY:
1130 sprite = plane->plane;
1131 wm_state->wm[level].sprite[sprite] = wm;
1132 break;
1133 }
1134 }
1135
1136 wm_state->num_levels = level;
1137
1138 if (!wm_state->cxsr)
1139 continue;
1140
1141
1142 switch (plane->base.type) {
1143 int sprite, level;
1144 case DRM_PLANE_TYPE_CURSOR:
1145 for (level = 0; level < wm_state->num_levels; level++)
1146 wm_state->sr[level].cursor =
1147 wm_state->wm[level].cursor;
1148 break;
1149 case DRM_PLANE_TYPE_PRIMARY:
1150 for (level = 0; level < wm_state->num_levels; level++)
1151 wm_state->sr[level].plane =
1152 min(wm_state->sr[level].plane,
1153 wm_state->wm[level].primary);
1154 break;
1155 case DRM_PLANE_TYPE_OVERLAY:
1156 sprite = plane->plane;
1157 for (level = 0; level < wm_state->num_levels; level++)
1158 wm_state->sr[level].plane =
1159 min(wm_state->sr[level].plane,
1160 wm_state->wm[level].sprite[sprite]);
1161 break;
1162 }
1163 }
1164
1165
1166 for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
1167 memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
1168 memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
1169 }
1170
1171 vlv_invert_wms(crtc);
1172}
1173
1174#define VLV_FIFO(plane, value) \
1175 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1176
1177static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
1178{
1179 struct drm_device *dev = crtc->base.dev;
1180 struct drm_i915_private *dev_priv = to_i915(dev);
1181 struct intel_plane *plane;
1182 int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
1183
1184 for_each_intel_plane_on_crtc(dev, crtc, plane) {
1185 if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
1186 WARN_ON(plane->wm.fifo_size != 63);
1187 continue;
1188 }
1189
1190 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
1191 sprite0_start = plane->wm.fifo_size;
1192 else if (plane->plane == 0)
1193 sprite1_start = sprite0_start + plane->wm.fifo_size;
1194 else
1195 fifo_size = sprite1_start + plane->wm.fifo_size;
1196 }
1197
1198 WARN_ON(fifo_size != 512 - 1);
1199
1200 DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
1201 pipe_name(crtc->pipe), sprite0_start,
1202 sprite1_start, fifo_size);
1203
1204 switch (crtc->pipe) {
1205 uint32_t dsparb, dsparb2, dsparb3;
1206 case PIPE_A:
1207 dsparb = I915_READ(DSPARB);
1208 dsparb2 = I915_READ(DSPARB2);
1209
1210 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1211 VLV_FIFO(SPRITEB, 0xff));
1212 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1213 VLV_FIFO(SPRITEB, sprite1_start));
1214
1215 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1216 VLV_FIFO(SPRITEB_HI, 0x1));
1217 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1218 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1219
1220 I915_WRITE(DSPARB, dsparb);
1221 I915_WRITE(DSPARB2, dsparb2);
1222 break;
1223 case PIPE_B:
1224 dsparb = I915_READ(DSPARB);
1225 dsparb2 = I915_READ(DSPARB2);
1226
1227 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1228 VLV_FIFO(SPRITED, 0xff));
1229 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1230 VLV_FIFO(SPRITED, sprite1_start));
1231
1232 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1233 VLV_FIFO(SPRITED_HI, 0xff));
1234 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1235 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1236
1237 I915_WRITE(DSPARB, dsparb);
1238 I915_WRITE(DSPARB2, dsparb2);
1239 break;
1240 case PIPE_C:
1241 dsparb3 = I915_READ(DSPARB3);
1242 dsparb2 = I915_READ(DSPARB2);
1243
1244 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1245 VLV_FIFO(SPRITEF, 0xff));
1246 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1247 VLV_FIFO(SPRITEF, sprite1_start));
1248
1249 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1250 VLV_FIFO(SPRITEF_HI, 0xff));
1251 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1252 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1253
1254 I915_WRITE(DSPARB3, dsparb3);
1255 I915_WRITE(DSPARB2, dsparb2);
1256 break;
1257 default:
1258 break;
1259 }
1260}
1261
1262#undef VLV_FIFO
1263
1264static void vlv_merge_wm(struct drm_device *dev,
1265 struct vlv_wm_values *wm)
1266{
1267 struct intel_crtc *crtc;
1268 int num_active_crtcs = 0;
1269
1270 wm->level = to_i915(dev)->wm.max_level;
1271 wm->cxsr = true;
1272
1273 for_each_intel_crtc(dev, crtc) {
1274 const struct vlv_wm_state *wm_state = &crtc->wm_state;
1275
1276 if (!crtc->active)
1277 continue;
1278
1279 if (!wm_state->cxsr)
1280 wm->cxsr = false;
1281
1282 num_active_crtcs++;
1283 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1284 }
1285
1286 if (num_active_crtcs != 1)
1287 wm->cxsr = false;
1288
1289 if (num_active_crtcs > 1)
1290 wm->level = VLV_WM_LEVEL_PM2;
1291
1292 for_each_intel_crtc(dev, crtc) {
1293 struct vlv_wm_state *wm_state = &crtc->wm_state;
1294 enum pipe pipe = crtc->pipe;
1295
1296 if (!crtc->active)
1297 continue;
1298
1299 wm->pipe[pipe] = wm_state->wm[wm->level];
1300 if (wm->cxsr)
1301 wm->sr = wm_state->sr[wm->level];
1302
1303 wm->ddl[pipe].primary = DDL_PRECISION_HIGH | 2;
1304 wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH | 2;
1305 wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH | 2;
1306 wm->ddl[pipe].cursor = DDL_PRECISION_HIGH | 2;
1307 }
1308}
1309
1310static void vlv_update_wm(struct drm_crtc *crtc)
1311{
1312 struct drm_device *dev = crtc->dev;
1313 struct drm_i915_private *dev_priv = dev->dev_private;
1314 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1315 enum pipe pipe = intel_crtc->pipe;
1316 struct vlv_wm_values wm = {};
1317
1318 vlv_compute_wm(intel_crtc);
1319 vlv_merge_wm(dev, &wm);
1320
1321 if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
1322
1323 vlv_pipe_set_fifo_size(intel_crtc);
1324 return;
1325 }
1326
1327 if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
1328 dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
1329 chv_set_memory_dvfs(dev_priv, false);
1330
1331 if (wm.level < VLV_WM_LEVEL_PM5 &&
1332 dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
1333 chv_set_memory_pm5(dev_priv, false);
1334
1335 if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
1336 intel_set_memory_cxsr(dev_priv, false);
1337
1338
1339 vlv_pipe_set_fifo_size(intel_crtc);
1340
1341 vlv_write_wm_values(intel_crtc, &wm);
1342
1343 DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
1344 "sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
1345 pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
1346 wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
1347 wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
1348
1349 if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
1350 intel_set_memory_cxsr(dev_priv, true);
1351
1352 if (wm.level >= VLV_WM_LEVEL_PM5 &&
1353 dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
1354 chv_set_memory_pm5(dev_priv, true);
1355
1356 if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
1357 dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
1358 chv_set_memory_dvfs(dev_priv, true);
1359
1360 dev_priv->wm.vlv = wm;
1361}
1362
1363#define single_plane_enabled(mask) is_power_of_2(mask)
1364
1365static void g4x_update_wm(struct drm_crtc *crtc)
1366{
1367 struct drm_device *dev = crtc->dev;
1368 static const int sr_latency_ns = 12000;
1369 struct drm_i915_private *dev_priv = dev->dev_private;
1370 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1371 int plane_sr, cursor_sr;
1372 unsigned int enabled = 0;
1373 bool cxsr_enabled;
1374
1375 if (g4x_compute_wm0(dev, PIPE_A,
1376 &g4x_wm_info, pessimal_latency_ns,
1377 &g4x_cursor_wm_info, pessimal_latency_ns,
1378 &planea_wm, &cursora_wm))
1379 enabled |= 1 << PIPE_A;
1380
1381 if (g4x_compute_wm0(dev, PIPE_B,
1382 &g4x_wm_info, pessimal_latency_ns,
1383 &g4x_cursor_wm_info, pessimal_latency_ns,
1384 &planeb_wm, &cursorb_wm))
1385 enabled |= 1 << PIPE_B;
1386
1387 if (single_plane_enabled(enabled) &&
1388 g4x_compute_srwm(dev, ffs(enabled) - 1,
1389 sr_latency_ns,
1390 &g4x_wm_info,
1391 &g4x_cursor_wm_info,
1392 &plane_sr, &cursor_sr)) {
1393 cxsr_enabled = true;
1394 } else {
1395 cxsr_enabled = false;
1396 intel_set_memory_cxsr(dev_priv, false);
1397 plane_sr = cursor_sr = 0;
1398 }
1399
1400 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
1401 "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1402 planea_wm, cursora_wm,
1403 planeb_wm, cursorb_wm,
1404 plane_sr, cursor_sr);
1405
1406 I915_WRITE(DSPFW1,
1407 FW_WM(plane_sr, SR) |
1408 FW_WM(cursorb_wm, CURSORB) |
1409 FW_WM(planeb_wm, PLANEB) |
1410 FW_WM(planea_wm, PLANEA));
1411 I915_WRITE(DSPFW2,
1412 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1413 FW_WM(cursora_wm, CURSORA));
1414
1415 I915_WRITE(DSPFW3,
1416 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1417 FW_WM(cursor_sr, CURSOR_SR));
1418
1419 if (cxsr_enabled)
1420 intel_set_memory_cxsr(dev_priv, true);
1421}
1422
1423static void i965_update_wm(struct drm_crtc *unused_crtc)
1424{
1425 struct drm_device *dev = unused_crtc->dev;
1426 struct drm_i915_private *dev_priv = dev->dev_private;
1427 struct drm_crtc *crtc;
1428 int srwm = 1;
1429 int cursor_sr = 16;
1430 bool cxsr_enabled;
1431
1432
1433 crtc = single_enabled_crtc(dev);
1434 if (crtc) {
1435
1436 static const int sr_latency_ns = 12000;
1437 const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1438 int clock = adjusted_mode->crtc_clock;
1439 int htotal = adjusted_mode->crtc_htotal;
1440 int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
1441 int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
1442 unsigned long line_time_us;
1443 int entries;
1444
1445 line_time_us = max(htotal * 1000 / clock, 1);
1446
1447
1448 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1449 cpp * hdisplay;
1450 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1451 srwm = I965_FIFO_SIZE - entries;
1452 if (srwm < 0)
1453 srwm = 1;
1454 srwm &= 0x1ff;
1455 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1456 entries, srwm);
1457
1458 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1459 cpp * crtc->cursor->state->crtc_w;
1460 entries = DIV_ROUND_UP(entries,
1461 i965_cursor_wm_info.cacheline_size);
1462 cursor_sr = i965_cursor_wm_info.fifo_size -
1463 (entries + i965_cursor_wm_info.guard_size);
1464
1465 if (cursor_sr > i965_cursor_wm_info.max_wm)
1466 cursor_sr = i965_cursor_wm_info.max_wm;
1467
1468 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1469 "cursor %d\n", srwm, cursor_sr);
1470
1471 cxsr_enabled = true;
1472 } else {
1473 cxsr_enabled = false;
1474
1475 intel_set_memory_cxsr(dev_priv, false);
1476 }
1477
1478 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1479 srwm);
1480
1481
1482 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
1483 FW_WM(8, CURSORB) |
1484 FW_WM(8, PLANEB) |
1485 FW_WM(8, PLANEA));
1486 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
1487 FW_WM(8, PLANEC_OLD));
1488
1489 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
1490
1491 if (cxsr_enabled)
1492 intel_set_memory_cxsr(dev_priv, true);
1493}
1494
1495#undef FW_WM
1496
1497static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1498{
1499 struct drm_device *dev = unused_crtc->dev;
1500 struct drm_i915_private *dev_priv = dev->dev_private;
1501 const struct intel_watermark_params *wm_info;
1502 uint32_t fwater_lo;
1503 uint32_t fwater_hi;
1504 int cwm, srwm = 1;
1505 int fifo_size;
1506 int planea_wm, planeb_wm;
1507 struct drm_crtc *crtc, *enabled = NULL;
1508
1509 if (IS_I945GM(dev))
1510 wm_info = &i945_wm_info;
1511 else if (!IS_GEN2(dev))
1512 wm_info = &i915_wm_info;
1513 else
1514 wm_info = &i830_a_wm_info;
1515
1516 fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1517 crtc = intel_get_crtc_for_plane(dev, 0);
1518 if (intel_crtc_active(crtc)) {
1519 const struct drm_display_mode *adjusted_mode;
1520 int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
1521 if (IS_GEN2(dev))
1522 cpp = 4;
1523
1524 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1525 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1526 wm_info, fifo_size, cpp,
1527 pessimal_latency_ns);
1528 enabled = crtc;
1529 } else {
1530 planea_wm = fifo_size - wm_info->guard_size;
1531 if (planea_wm > (long)wm_info->max_wm)
1532 planea_wm = wm_info->max_wm;
1533 }
1534
1535 if (IS_GEN2(dev))
1536 wm_info = &i830_bc_wm_info;
1537
1538 fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1539 crtc = intel_get_crtc_for_plane(dev, 1);
1540 if (intel_crtc_active(crtc)) {
1541 const struct drm_display_mode *adjusted_mode;
1542 int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
1543 if (IS_GEN2(dev))
1544 cpp = 4;
1545
1546 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1547 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1548 wm_info, fifo_size, cpp,
1549 pessimal_latency_ns);
1550 if (enabled == NULL)
1551 enabled = crtc;
1552 else
1553 enabled = NULL;
1554 } else {
1555 planeb_wm = fifo_size - wm_info->guard_size;
1556 if (planeb_wm > (long)wm_info->max_wm)
1557 planeb_wm = wm_info->max_wm;
1558 }
1559
1560 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1561
1562 if (IS_I915GM(dev) && enabled) {
1563 struct drm_i915_gem_object *obj;
1564
1565 obj = intel_fb_obj(enabled->primary->state->fb);
1566
1567
1568 if (obj->tiling_mode == I915_TILING_NONE)
1569 enabled = NULL;
1570 }
1571
1572
1573
1574
1575 cwm = 2;
1576
1577
1578 intel_set_memory_cxsr(dev_priv, false);
1579
1580
1581 if (HAS_FW_BLC(dev) && enabled) {
1582
1583 static const int sr_latency_ns = 6000;
1584 const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
1585 int clock = adjusted_mode->crtc_clock;
1586 int htotal = adjusted_mode->crtc_htotal;
1587 int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
1588 int cpp = drm_format_plane_cpp(enabled->primary->state->fb->pixel_format, 0);
1589 unsigned long line_time_us;
1590 int entries;
1591
1592 line_time_us = max(htotal * 1000 / clock, 1);
1593
1594
1595 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1596 cpp * hdisplay;
1597 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1598 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1599 srwm = wm_info->fifo_size - entries;
1600 if (srwm < 0)
1601 srwm = 1;
1602
1603 if (IS_I945G(dev) || IS_I945GM(dev))
1604 I915_WRITE(FW_BLC_SELF,
1605 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1606 else if (IS_I915GM(dev))
1607 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1608 }
1609
1610 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1611 planea_wm, planeb_wm, cwm, srwm);
1612
1613 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1614 fwater_hi = (cwm & 0x1f);
1615
1616
1617 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1618 fwater_hi = fwater_hi | (1 << 8);
1619
1620 I915_WRITE(FW_BLC, fwater_lo);
1621 I915_WRITE(FW_BLC2, fwater_hi);
1622
1623 if (enabled)
1624 intel_set_memory_cxsr(dev_priv, true);
1625}
1626
1627static void i845_update_wm(struct drm_crtc *unused_crtc)
1628{
1629 struct drm_device *dev = unused_crtc->dev;
1630 struct drm_i915_private *dev_priv = dev->dev_private;
1631 struct drm_crtc *crtc;
1632 const struct drm_display_mode *adjusted_mode;
1633 uint32_t fwater_lo;
1634 int planea_wm;
1635
1636 crtc = single_enabled_crtc(dev);
1637 if (crtc == NULL)
1638 return;
1639
1640 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1641 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1642 &i845_wm_info,
1643 dev_priv->display.get_fifo_size(dev, 0),
1644 4, pessimal_latency_ns);
1645 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1646 fwater_lo |= (3<<8) | planea_wm;
1647
1648 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1649
1650 I915_WRITE(FW_BLC, fwater_lo);
1651}
1652
1653uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
1654{
1655 uint32_t pixel_rate;
1656
1657 pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
1658
1659
1660
1661
1662 if (pipe_config->pch_pfit.enabled) {
1663 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
1664 uint32_t pfit_size = pipe_config->pch_pfit.size;
1665
1666 pipe_w = pipe_config->pipe_src_w;
1667 pipe_h = pipe_config->pipe_src_h;
1668
1669 pfit_w = (pfit_size >> 16) & 0xFFFF;
1670 pfit_h = pfit_size & 0xFFFF;
1671 if (pipe_w < pfit_w)
1672 pipe_w = pfit_w;
1673 if (pipe_h < pfit_h)
1674 pipe_h = pfit_h;
1675
1676 if (WARN_ON(!pfit_w || !pfit_h))
1677 return pixel_rate;
1678
1679 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1680 pfit_w * pfit_h);
1681 }
1682
1683 return pixel_rate;
1684}
1685
1686
1687static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latency)
1688{
1689 uint64_t ret;
1690
1691 if (WARN(latency == 0, "Latency value missing\n"))
1692 return UINT_MAX;
1693
1694 ret = (uint64_t) pixel_rate * cpp * latency;
1695 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1696
1697 return ret;
1698}
1699
1700
1701static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
1702 uint32_t horiz_pixels, uint8_t cpp,
1703 uint32_t latency)
1704{
1705 uint32_t ret;
1706
1707 if (WARN(latency == 0, "Latency value missing\n"))
1708 return UINT_MAX;
1709 if (WARN_ON(!pipe_htotal))
1710 return UINT_MAX;
1711
1712 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1713 ret = (ret + 1) * horiz_pixels * cpp;
1714 ret = DIV_ROUND_UP(ret, 64) + 2;
1715 return ret;
1716}
1717
1718static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
1719 uint8_t cpp)
1720{
1721
1722
1723
1724
1725
1726
1727 if (WARN_ON(!cpp))
1728 return 0;
1729 if (WARN_ON(!horiz_pixels))
1730 return 0;
1731
1732 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
1733}
1734
1735struct ilk_wm_maximums {
1736 uint16_t pri;
1737 uint16_t spr;
1738 uint16_t cur;
1739 uint16_t fbc;
1740};
1741
1742
1743
1744
1745
1746static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
1747 const struct intel_plane_state *pstate,
1748 uint32_t mem_value,
1749 bool is_lp)
1750{
1751 int cpp = pstate->base.fb ?
1752 drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
1753 uint32_t method1, method2;
1754
1755 if (!cstate->base.active || !pstate->visible)
1756 return 0;
1757
1758 method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), cpp, mem_value);
1759
1760 if (!is_lp)
1761 return method1;
1762
1763 method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
1764 cstate->base.adjusted_mode.crtc_htotal,
1765 drm_rect_width(&pstate->dst),
1766 cpp, mem_value);
1767
1768 return min(method1, method2);
1769}
1770
1771
1772
1773
1774
1775static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
1776 const struct intel_plane_state *pstate,
1777 uint32_t mem_value)
1778{
1779 int cpp = pstate->base.fb ?
1780 drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
1781 uint32_t method1, method2;
1782
1783 if (!cstate->base.active || !pstate->visible)
1784 return 0;
1785
1786 method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), cpp, mem_value);
1787 method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
1788 cstate->base.adjusted_mode.crtc_htotal,
1789 drm_rect_width(&pstate->dst),
1790 cpp, mem_value);
1791 return min(method1, method2);
1792}
1793
1794
1795
1796
1797
1798static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
1799 const struct intel_plane_state *pstate,
1800 uint32_t mem_value)
1801{
1802
1803
1804
1805
1806
1807 int cpp = 4;
1808 int width = pstate->visible ? pstate->base.crtc_w : 64;
1809
1810 if (!cstate->base.active)
1811 return 0;
1812
1813 return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
1814 cstate->base.adjusted_mode.crtc_htotal,
1815 width, cpp, mem_value);
1816}
1817
1818
1819static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1820 const struct intel_plane_state *pstate,
1821 uint32_t pri_val)
1822{
1823 int cpp = pstate->base.fb ?
1824 drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
1825
1826 if (!cstate->base.active || !pstate->visible)
1827 return 0;
1828
1829 return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), cpp);
1830}
1831
1832static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1833{
1834 if (INTEL_INFO(dev)->gen >= 8)
1835 return 3072;
1836 else if (INTEL_INFO(dev)->gen >= 7)
1837 return 768;
1838 else
1839 return 512;
1840}
1841
1842static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1843 int level, bool is_sprite)
1844{
1845 if (INTEL_INFO(dev)->gen >= 8)
1846
1847 return level == 0 ? 255 : 2047;
1848 else if (INTEL_INFO(dev)->gen >= 7)
1849
1850 return level == 0 ? 127 : 1023;
1851 else if (!is_sprite)
1852
1853 return level == 0 ? 127 : 511;
1854 else
1855
1856 return level == 0 ? 63 : 255;
1857}
1858
1859static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
1860 int level)
1861{
1862 if (INTEL_INFO(dev)->gen >= 7)
1863 return level == 0 ? 63 : 255;
1864 else
1865 return level == 0 ? 31 : 63;
1866}
1867
1868static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
1869{
1870 if (INTEL_INFO(dev)->gen >= 8)
1871 return 31;
1872 else
1873 return 15;
1874}
1875
1876
1877static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1878 int level,
1879 const struct intel_wm_config *config,
1880 enum intel_ddb_partitioning ddb_partitioning,
1881 bool is_sprite)
1882{
1883 unsigned int fifo_size = ilk_display_fifo_size(dev);
1884
1885
1886 if (is_sprite && !config->sprites_enabled)
1887 return 0;
1888
1889
1890 if (level == 0 || config->num_pipes_active > 1) {
1891 fifo_size /= INTEL_INFO(dev)->num_pipes;
1892
1893
1894
1895
1896
1897
1898 if (INTEL_INFO(dev)->gen <= 6)
1899 fifo_size /= 2;
1900 }
1901
1902 if (config->sprites_enabled) {
1903
1904 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1905 if (is_sprite)
1906 fifo_size *= 5;
1907 fifo_size /= 6;
1908 } else {
1909 fifo_size /= 2;
1910 }
1911 }
1912
1913
1914 return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
1915}
1916
1917
1918static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1919 int level,
1920 const struct intel_wm_config *config)
1921{
1922
1923 if (level > 0 && config->num_pipes_active > 1)
1924 return 64;
1925
1926
1927 return ilk_cursor_wm_reg_max(dev, level);
1928}
1929
1930static void ilk_compute_wm_maximums(const struct drm_device *dev,
1931 int level,
1932 const struct intel_wm_config *config,
1933 enum intel_ddb_partitioning ddb_partitioning,
1934 struct ilk_wm_maximums *max)
1935{
1936 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1937 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1938 max->cur = ilk_cursor_wm_max(dev, level, config);
1939 max->fbc = ilk_fbc_wm_reg_max(dev);
1940}
1941
1942static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
1943 int level,
1944 struct ilk_wm_maximums *max)
1945{
1946 max->pri = ilk_plane_wm_reg_max(dev, level, false);
1947 max->spr = ilk_plane_wm_reg_max(dev, level, true);
1948 max->cur = ilk_cursor_wm_reg_max(dev, level);
1949 max->fbc = ilk_fbc_wm_reg_max(dev);
1950}
1951
1952static bool ilk_validate_wm_level(int level,
1953 const struct ilk_wm_maximums *max,
1954 struct intel_wm_level *result)
1955{
1956 bool ret;
1957
1958
1959 if (!result->enable)
1960 return false;
1961
1962 result->enable = result->pri_val <= max->pri &&
1963 result->spr_val <= max->spr &&
1964 result->cur_val <= max->cur;
1965
1966 ret = result->enable;
1967
1968
1969
1970
1971
1972
1973 if (level == 0 && !result->enable) {
1974 if (result->pri_val > max->pri)
1975 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
1976 level, result->pri_val, max->pri);
1977 if (result->spr_val > max->spr)
1978 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
1979 level, result->spr_val, max->spr);
1980 if (result->cur_val > max->cur)
1981 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
1982 level, result->cur_val, max->cur);
1983
1984 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
1985 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
1986 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
1987 result->enable = true;
1988 }
1989
1990 return ret;
1991}
1992
1993static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
1994 const struct intel_crtc *intel_crtc,
1995 int level,
1996 struct intel_crtc_state *cstate,
1997 struct intel_plane_state *pristate,
1998 struct intel_plane_state *sprstate,
1999 struct intel_plane_state *curstate,
2000 struct intel_wm_level *result)
2001{
2002 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2003 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2004 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2005
2006
2007 if (level > 0) {
2008 pri_latency *= 5;
2009 spr_latency *= 5;
2010 cur_latency *= 5;
2011 }
2012
2013 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2014 pri_latency, level);
2015 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2016 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2017 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2018 result->enable = true;
2019}
2020
2021static uint32_t
2022hsw_compute_linetime_wm(struct drm_device *dev,
2023 struct intel_crtc_state *cstate)
2024{
2025 struct drm_i915_private *dev_priv = dev->dev_private;
2026 const struct drm_display_mode *adjusted_mode =
2027 &cstate->base.adjusted_mode;
2028 u32 linetime, ips_linetime;
2029
2030 if (!cstate->base.active)
2031 return 0;
2032 if (WARN_ON(adjusted_mode->crtc_clock == 0))
2033 return 0;
2034 if (WARN_ON(dev_priv->cdclk_freq == 0))
2035 return 0;
2036
2037
2038
2039
2040 linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2041 adjusted_mode->crtc_clock);
2042 ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2043 dev_priv->cdclk_freq);
2044
2045 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2046 PIPE_WM_LINETIME_TIME(linetime);
2047}
2048
2049static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
2050{
2051 struct drm_i915_private *dev_priv = dev->dev_private;
2052
2053 if (IS_GEN9(dev)) {
2054 uint32_t val;
2055 int ret, i;
2056 int level, max_level = ilk_wm_max_level(dev);
2057
2058
2059 val = 0;
2060 mutex_lock(&dev_priv->rps.hw_lock);
2061 ret = sandybridge_pcode_read(dev_priv,
2062 GEN9_PCODE_READ_MEM_LATENCY,
2063 &val);
2064 mutex_unlock(&dev_priv->rps.hw_lock);
2065
2066 if (ret) {
2067 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2068 return;
2069 }
2070
2071 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2072 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2073 GEN9_MEM_LATENCY_LEVEL_MASK;
2074 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2075 GEN9_MEM_LATENCY_LEVEL_MASK;
2076 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2077 GEN9_MEM_LATENCY_LEVEL_MASK;
2078
2079
2080 val = 1;
2081 mutex_lock(&dev_priv->rps.hw_lock);
2082 ret = sandybridge_pcode_read(dev_priv,
2083 GEN9_PCODE_READ_MEM_LATENCY,
2084 &val);
2085 mutex_unlock(&dev_priv->rps.hw_lock);
2086 if (ret) {
2087 DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2088 return;
2089 }
2090
2091 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2092 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2093 GEN9_MEM_LATENCY_LEVEL_MASK;
2094 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2095 GEN9_MEM_LATENCY_LEVEL_MASK;
2096 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2097 GEN9_MEM_LATENCY_LEVEL_MASK;
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116 wm[0] += 2;
2117 for (level = 1; level <= max_level; level++)
2118 if (wm[level] != 0)
2119 wm[level] += 2;
2120 else {
2121 for (i = level + 1; i <= max_level; i++)
2122 wm[i] = 0;
2123
2124 break;
2125 }
2126 } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2127 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2128
2129 wm[0] = (sskpd >> 56) & 0xFF;
2130 if (wm[0] == 0)
2131 wm[0] = sskpd & 0xF;
2132 wm[1] = (sskpd >> 4) & 0xFF;
2133 wm[2] = (sskpd >> 12) & 0xFF;
2134 wm[3] = (sskpd >> 20) & 0x1FF;
2135 wm[4] = (sskpd >> 32) & 0x1FF;
2136 } else if (INTEL_INFO(dev)->gen >= 6) {
2137 uint32_t sskpd = I915_READ(MCH_SSKPD);
2138
2139 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2140 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2141 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2142 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2143 } else if (INTEL_INFO(dev)->gen >= 5) {
2144 uint32_t mltr = I915_READ(MLTR_ILK);
2145
2146
2147 wm[0] = 7;
2148 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2149 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2150 }
2151}
2152
2153static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2154{
2155
2156 if (INTEL_INFO(dev)->gen == 5)
2157 wm[0] = 13;
2158}
2159
2160static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2161{
2162
2163 if (INTEL_INFO(dev)->gen == 5)
2164 wm[0] = 13;
2165
2166
2167 if (IS_IVYBRIDGE(dev))
2168 wm[3] *= 2;
2169}
2170
2171int ilk_wm_max_level(const struct drm_device *dev)
2172{
2173
2174 if (INTEL_INFO(dev)->gen >= 9)
2175 return 7;
2176 else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2177 return 4;
2178 else if (INTEL_INFO(dev)->gen >= 6)
2179 return 3;
2180 else
2181 return 2;
2182}
2183
2184static void intel_print_wm_latency(struct drm_device *dev,
2185 const char *name,
2186 const uint16_t wm[8])
2187{
2188 int level, max_level = ilk_wm_max_level(dev);
2189
2190 for (level = 0; level <= max_level; level++) {
2191 unsigned int latency = wm[level];
2192
2193 if (latency == 0) {
2194 DRM_ERROR("%s WM%d latency not provided\n",
2195 name, level);
2196 continue;
2197 }
2198
2199
2200
2201
2202
2203 if (IS_GEN9(dev))
2204 latency *= 10;
2205 else if (level > 0)
2206 latency *= 5;
2207
2208 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2209 name, level, wm[level],
2210 latency / 10, latency % 10);
2211 }
2212}
2213
2214static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2215 uint16_t wm[5], uint16_t min)
2216{
2217 int level, max_level = ilk_wm_max_level(dev_priv->dev);
2218
2219 if (wm[0] >= min)
2220 return false;
2221
2222 wm[0] = max(wm[0], min);
2223 for (level = 1; level <= max_level; level++)
2224 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2225
2226 return true;
2227}
2228
2229static void snb_wm_latency_quirk(struct drm_device *dev)
2230{
2231 struct drm_i915_private *dev_priv = dev->dev_private;
2232 bool changed;
2233
2234
2235
2236
2237
2238 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2239 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2240 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2241
2242 if (!changed)
2243 return;
2244
2245 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2246 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2247 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2248 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2249}
2250
2251static void ilk_setup_wm_latency(struct drm_device *dev)
2252{
2253 struct drm_i915_private *dev_priv = dev->dev_private;
2254
2255 intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2256
2257 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2258 sizeof(dev_priv->wm.pri_latency));
2259 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2260 sizeof(dev_priv->wm.pri_latency));
2261
2262 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2263 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
2264
2265 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2266 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2267 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2268
2269 if (IS_GEN6(dev))
2270 snb_wm_latency_quirk(dev);
2271}
2272
2273static void skl_setup_wm_latency(struct drm_device *dev)
2274{
2275 struct drm_i915_private *dev_priv = dev->dev_private;
2276
2277 intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
2278 intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
2279}
2280
2281
2282static int ilk_compute_pipe_wm(struct intel_crtc *intel_crtc,
2283 struct drm_atomic_state *state)
2284{
2285 struct intel_pipe_wm *pipe_wm;
2286 struct drm_device *dev = intel_crtc->base.dev;
2287 const struct drm_i915_private *dev_priv = dev->dev_private;
2288 struct intel_crtc_state *cstate = NULL;
2289 struct intel_plane *intel_plane;
2290 struct drm_plane_state *ps;
2291 struct intel_plane_state *pristate = NULL;
2292 struct intel_plane_state *sprstate = NULL;
2293 struct intel_plane_state *curstate = NULL;
2294 int level, max_level = ilk_wm_max_level(dev);
2295
2296 struct intel_wm_config config = {
2297 .num_pipes_active = 1,
2298 };
2299 struct ilk_wm_maximums max;
2300
2301 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
2302 if (IS_ERR(cstate))
2303 return PTR_ERR(cstate);
2304
2305 pipe_wm = &cstate->wm.optimal.ilk;
2306 memset(pipe_wm, 0, sizeof(*pipe_wm));
2307
2308 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
2309 ps = drm_atomic_get_plane_state(state,
2310 &intel_plane->base);
2311 if (IS_ERR(ps))
2312 return PTR_ERR(ps);
2313
2314 if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2315 pristate = to_intel_plane_state(ps);
2316 else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2317 sprstate = to_intel_plane_state(ps);
2318 else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
2319 curstate = to_intel_plane_state(ps);
2320 }
2321
2322 config.sprites_enabled = sprstate->visible;
2323 config.sprites_scaled = sprstate->visible &&
2324 (drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 ||
2325 drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
2326
2327 pipe_wm->pipe_enabled = cstate->base.active;
2328 pipe_wm->sprites_enabled = config.sprites_enabled;
2329 pipe_wm->sprites_scaled = config.sprites_scaled;
2330
2331
2332 if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
2333 max_level = 1;
2334
2335
2336 if (config.sprites_scaled)
2337 max_level = 0;
2338
2339 ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
2340 pristate, sprstate, curstate, &pipe_wm->wm[0]);
2341
2342 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2343 pipe_wm->linetime = hsw_compute_linetime_wm(dev, cstate);
2344
2345
2346 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2347
2348
2349 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
2350 return -EINVAL;
2351
2352 ilk_compute_wm_reg_maximums(dev, 1, &max);
2353
2354 for (level = 1; level <= max_level; level++) {
2355 struct intel_wm_level wm = {};
2356
2357 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
2358 pristate, sprstate, curstate, &wm);
2359
2360
2361
2362
2363
2364
2365 if (!ilk_validate_wm_level(level, &max, &wm))
2366 break;
2367
2368 pipe_wm->wm[level] = wm;
2369 }
2370
2371 return 0;
2372}
2373
2374
2375
2376
2377static void ilk_merge_wm_level(struct drm_device *dev,
2378 int level,
2379 struct intel_wm_level *ret_wm)
2380{
2381 const struct intel_crtc *intel_crtc;
2382
2383 ret_wm->enable = true;
2384
2385 for_each_intel_crtc(dev, intel_crtc) {
2386 const struct intel_crtc_state *cstate =
2387 to_intel_crtc_state(intel_crtc->base.state);
2388 const struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
2389 const struct intel_wm_level *wm = &active->wm[level];
2390
2391 if (!active->pipe_enabled)
2392 continue;
2393
2394
2395
2396
2397
2398
2399 if (!wm->enable)
2400 ret_wm->enable = false;
2401
2402 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2403 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2404 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2405 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2406 }
2407}
2408
2409
2410
2411
2412static void ilk_wm_merge(struct drm_device *dev,
2413 const struct intel_wm_config *config,
2414 const struct ilk_wm_maximums *max,
2415 struct intel_pipe_wm *merged)
2416{
2417 struct drm_i915_private *dev_priv = dev->dev_private;
2418 int level, max_level = ilk_wm_max_level(dev);
2419 int last_enabled_level = max_level;
2420
2421
2422 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2423 config->num_pipes_active > 1)
2424 return;
2425
2426
2427 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
2428
2429
2430 for (level = 1; level <= max_level; level++) {
2431 struct intel_wm_level *wm = &merged->wm[level];
2432
2433 ilk_merge_wm_level(dev, level, wm);
2434
2435 if (level > last_enabled_level)
2436 wm->enable = false;
2437 else if (!ilk_validate_wm_level(level, max, wm))
2438
2439 last_enabled_level = level - 1;
2440
2441
2442
2443
2444
2445 if (wm->fbc_val > max->fbc) {
2446 if (wm->enable)
2447 merged->fbc_wm_enabled = false;
2448 wm->fbc_val = 0;
2449 }
2450 }
2451
2452
2453
2454
2455
2456
2457
2458 if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
2459 intel_fbc_is_active(dev_priv)) {
2460 for (level = 2; level <= max_level; level++) {
2461 struct intel_wm_level *wm = &merged->wm[level];
2462
2463 wm->enable = false;
2464 }
2465 }
2466}
2467
2468static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2469{
2470
2471 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2472}
2473
2474
2475static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2476{
2477 struct drm_i915_private *dev_priv = dev->dev_private;
2478
2479 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2480 return 2 * level;
2481 else
2482 return dev_priv->wm.pri_latency[level];
2483}
2484
2485static void ilk_compute_wm_results(struct drm_device *dev,
2486 const struct intel_pipe_wm *merged,
2487 enum intel_ddb_partitioning partitioning,
2488 struct ilk_wm_values *results)
2489{
2490 struct intel_crtc *intel_crtc;
2491 int level, wm_lp;
2492
2493 results->enable_fbc_wm = merged->fbc_wm_enabled;
2494 results->partitioning = partitioning;
2495
2496
2497 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2498 const struct intel_wm_level *r;
2499
2500 level = ilk_wm_lp_to_level(wm_lp, merged);
2501
2502 r = &merged->wm[level];
2503
2504
2505
2506
2507
2508 results->wm_lp[wm_lp - 1] =
2509 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2510 (r->pri_val << WM1_LP_SR_SHIFT) |
2511 r->cur_val;
2512
2513 if (r->enable)
2514 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2515
2516 if (INTEL_INFO(dev)->gen >= 8)
2517 results->wm_lp[wm_lp - 1] |=
2518 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2519 else
2520 results->wm_lp[wm_lp - 1] |=
2521 r->fbc_val << WM1_LP_FBC_SHIFT;
2522
2523
2524
2525
2526
2527 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2528 WARN_ON(wm_lp != 1);
2529 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2530 } else
2531 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2532 }
2533
2534
2535 for_each_intel_crtc(dev, intel_crtc) {
2536 const struct intel_crtc_state *cstate =
2537 to_intel_crtc_state(intel_crtc->base.state);
2538 enum pipe pipe = intel_crtc->pipe;
2539 const struct intel_wm_level *r = &cstate->wm.optimal.ilk.wm[0];
2540
2541 if (WARN_ON(!r->enable))
2542 continue;
2543
2544 results->wm_linetime[pipe] = cstate->wm.optimal.ilk.linetime;
2545
2546 results->wm_pipe[pipe] =
2547 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2548 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2549 r->cur_val;
2550 }
2551}
2552
2553
2554
2555static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2556 struct intel_pipe_wm *r1,
2557 struct intel_pipe_wm *r2)
2558{
2559 int level, max_level = ilk_wm_max_level(dev);
2560 int level1 = 0, level2 = 0;
2561
2562 for (level = 1; level <= max_level; level++) {
2563 if (r1->wm[level].enable)
2564 level1 = level;
2565 if (r2->wm[level].enable)
2566 level2 = level;
2567 }
2568
2569 if (level1 == level2) {
2570 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2571 return r2;
2572 else
2573 return r1;
2574 } else if (level1 > level2) {
2575 return r1;
2576 } else {
2577 return r2;
2578 }
2579}
2580
2581
2582#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2583#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2584#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2585#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2586#define WM_DIRTY_FBC (1 << 24)
2587#define WM_DIRTY_DDB (1 << 25)
2588
2589static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
2590 const struct ilk_wm_values *old,
2591 const struct ilk_wm_values *new)
2592{
2593 unsigned int dirty = 0;
2594 enum pipe pipe;
2595 int wm_lp;
2596
2597 for_each_pipe(dev_priv, pipe) {
2598 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2599 dirty |= WM_DIRTY_LINETIME(pipe);
2600
2601 dirty |= WM_DIRTY_LP_ALL;
2602 }
2603
2604 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2605 dirty |= WM_DIRTY_PIPE(pipe);
2606
2607 dirty |= WM_DIRTY_LP_ALL;
2608 }
2609 }
2610
2611 if (old->enable_fbc_wm != new->enable_fbc_wm) {
2612 dirty |= WM_DIRTY_FBC;
2613
2614 dirty |= WM_DIRTY_LP_ALL;
2615 }
2616
2617 if (old->partitioning != new->partitioning) {
2618 dirty |= WM_DIRTY_DDB;
2619
2620 dirty |= WM_DIRTY_LP_ALL;
2621 }
2622
2623
2624 if (dirty & WM_DIRTY_LP_ALL)
2625 return dirty;
2626
2627
2628 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2629 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2630 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2631 break;
2632 }
2633
2634
2635 for (; wm_lp <= 3; wm_lp++)
2636 dirty |= WM_DIRTY_LP(wm_lp);
2637
2638 return dirty;
2639}
2640
2641static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2642 unsigned int dirty)
2643{
2644 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2645 bool changed = false;
2646
2647 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2648 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2649 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2650 changed = true;
2651 }
2652 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2653 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2654 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2655 changed = true;
2656 }
2657 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2658 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2659 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2660 changed = true;
2661 }
2662
2663
2664
2665
2666
2667
2668 return changed;
2669}
2670
2671
2672
2673
2674
2675static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2676 struct ilk_wm_values *results)
2677{
2678 struct drm_device *dev = dev_priv->dev;
2679 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2680 unsigned int dirty;
2681 uint32_t val;
2682
2683 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
2684 if (!dirty)
2685 return;
2686
2687 _ilk_disable_lp_wm(dev_priv, dirty);
2688
2689 if (dirty & WM_DIRTY_PIPE(PIPE_A))
2690 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2691 if (dirty & WM_DIRTY_PIPE(PIPE_B))
2692 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
2693 if (dirty & WM_DIRTY_PIPE(PIPE_C))
2694 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2695
2696 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
2697 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
2698 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
2699 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
2700 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
2701 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2702
2703 if (dirty & WM_DIRTY_DDB) {
2704 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2705 val = I915_READ(WM_MISC);
2706 if (results->partitioning == INTEL_DDB_PART_1_2)
2707 val &= ~WM_MISC_DATA_PARTITION_5_6;
2708 else
2709 val |= WM_MISC_DATA_PARTITION_5_6;
2710 I915_WRITE(WM_MISC, val);
2711 } else {
2712 val = I915_READ(DISP_ARB_CTL2);
2713 if (results->partitioning == INTEL_DDB_PART_1_2)
2714 val &= ~DISP_DATA_PARTITION_5_6;
2715 else
2716 val |= DISP_DATA_PARTITION_5_6;
2717 I915_WRITE(DISP_ARB_CTL2, val);
2718 }
2719 }
2720
2721 if (dirty & WM_DIRTY_FBC) {
2722 val = I915_READ(DISP_ARB_CTL);
2723 if (results->enable_fbc_wm)
2724 val &= ~DISP_FBC_WM_DIS;
2725 else
2726 val |= DISP_FBC_WM_DIS;
2727 I915_WRITE(DISP_ARB_CTL, val);
2728 }
2729
2730 if (dirty & WM_DIRTY_LP(1) &&
2731 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2732 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2733
2734 if (INTEL_INFO(dev)->gen >= 7) {
2735 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2736 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2737 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2738 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2739 }
2740
2741 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
2742 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
2743 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
2744 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
2745 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
2746 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
2747
2748 dev_priv->wm.hw = *results;
2749}
2750
2751static bool ilk_disable_lp_wm(struct drm_device *dev)
2752{
2753 struct drm_i915_private *dev_priv = dev->dev_private;
2754
2755 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2756}
2757
2758
2759
2760
2761
2762
2763#define SKL_DDB_SIZE 896
2764#define BXT_DDB_SIZE 512
2765
2766
2767
2768
2769
2770
2771
2772static int
2773skl_wm_plane_id(const struct intel_plane *plane)
2774{
2775 switch (plane->base.type) {
2776 case DRM_PLANE_TYPE_PRIMARY:
2777 return 0;
2778 case DRM_PLANE_TYPE_CURSOR:
2779 return PLANE_CURSOR;
2780 case DRM_PLANE_TYPE_OVERLAY:
2781 return plane->plane + 1;
2782 default:
2783 MISSING_CASE(plane->base.type);
2784 return plane->plane;
2785 }
2786}
2787
2788static void
2789skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2790 const struct intel_crtc_state *cstate,
2791 const struct intel_wm_config *config,
2792 struct skl_ddb_entry *alloc )
2793{
2794 struct drm_crtc *for_crtc = cstate->base.crtc;
2795 struct drm_crtc *crtc;
2796 unsigned int pipe_size, ddb_size;
2797 int nth_active_pipe;
2798
2799 if (!cstate->base.active) {
2800 alloc->start = 0;
2801 alloc->end = 0;
2802 return;
2803 }
2804
2805 if (IS_BROXTON(dev))
2806 ddb_size = BXT_DDB_SIZE;
2807 else
2808 ddb_size = SKL_DDB_SIZE;
2809
2810 ddb_size -= 4;
2811
2812 nth_active_pipe = 0;
2813 for_each_crtc(dev, crtc) {
2814 if (!to_intel_crtc(crtc)->active)
2815 continue;
2816
2817 if (crtc == for_crtc)
2818 break;
2819
2820 nth_active_pipe++;
2821 }
2822
2823 pipe_size = ddb_size / config->num_pipes_active;
2824 alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
2825 alloc->end = alloc->start + pipe_size;
2826}
2827
2828static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
2829{
2830 if (config->num_pipes_active == 1)
2831 return 32;
2832
2833 return 8;
2834}
2835
2836static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
2837{
2838 entry->start = reg & 0x3ff;
2839 entry->end = (reg >> 16) & 0x3ff;
2840 if (entry->end)
2841 entry->end += 1;
2842}
2843
2844void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
2845 struct skl_ddb_allocation *ddb )
2846{
2847 enum pipe pipe;
2848 int plane;
2849 u32 val;
2850
2851 memset(ddb, 0, sizeof(*ddb));
2852
2853 for_each_pipe(dev_priv, pipe) {
2854 enum intel_display_power_domain power_domain;
2855
2856 power_domain = POWER_DOMAIN_PIPE(pipe);
2857 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
2858 continue;
2859
2860 for_each_plane(dev_priv, pipe, plane) {
2861 val = I915_READ(PLANE_BUF_CFG(pipe, plane));
2862 skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
2863 val);
2864 }
2865
2866 val = I915_READ(CUR_BUF_CFG(pipe));
2867 skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
2868 val);
2869
2870 intel_display_power_put(dev_priv, power_domain);
2871 }
2872}
2873
2874static unsigned int
2875skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
2876 const struct drm_plane_state *pstate,
2877 int y)
2878{
2879 struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
2880 struct drm_framebuffer *fb = pstate->fb;
2881 uint32_t width = 0, height = 0;
2882
2883 width = drm_rect_width(&intel_pstate->src) >> 16;
2884 height = drm_rect_height(&intel_pstate->src) >> 16;
2885
2886 if (intel_rotation_90_or_270(pstate->rotation))
2887 swap(width, height);
2888
2889
2890 if (fb->pixel_format == DRM_FORMAT_NV12) {
2891 if (y)
2892 return width * height *
2893 drm_format_plane_cpp(fb->pixel_format, 0);
2894 else
2895 return (width / 2) * (height / 2) *
2896 drm_format_plane_cpp(fb->pixel_format, 1);
2897 }
2898
2899
2900 return width * height * drm_format_plane_cpp(fb->pixel_format, 0);
2901}
2902
2903
2904
2905
2906
2907
2908static unsigned int
2909skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate)
2910{
2911 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
2912 struct drm_device *dev = intel_crtc->base.dev;
2913 const struct intel_plane *intel_plane;
2914 unsigned int total_data_rate = 0;
2915
2916 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
2917 const struct drm_plane_state *pstate = intel_plane->base.state;
2918
2919 if (pstate->fb == NULL)
2920 continue;
2921
2922 if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
2923 continue;
2924
2925
2926 total_data_rate += skl_plane_relative_data_rate(cstate,
2927 pstate,
2928 0);
2929
2930 if (pstate->fb->pixel_format == DRM_FORMAT_NV12)
2931
2932 total_data_rate += skl_plane_relative_data_rate(cstate,
2933 pstate,
2934 1);
2935 }
2936
2937 return total_data_rate;
2938}
2939
2940static void
2941skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
2942 struct skl_ddb_allocation *ddb )
2943{
2944 struct drm_crtc *crtc = cstate->base.crtc;
2945 struct drm_device *dev = crtc->dev;
2946 struct drm_i915_private *dev_priv = to_i915(dev);
2947 struct intel_wm_config *config = &dev_priv->wm.config;
2948 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2949 struct intel_plane *intel_plane;
2950 enum pipe pipe = intel_crtc->pipe;
2951 struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
2952 uint16_t alloc_size, start, cursor_blocks;
2953 uint16_t minimum[I915_MAX_PLANES];
2954 uint16_t y_minimum[I915_MAX_PLANES];
2955 unsigned int total_data_rate;
2956
2957 skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc);
2958 alloc_size = skl_ddb_entry_size(alloc);
2959 if (alloc_size == 0) {
2960 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
2961 memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
2962 sizeof(ddb->plane[pipe][PLANE_CURSOR]));
2963 return;
2964 }
2965
2966 cursor_blocks = skl_cursor_allocation(config);
2967 ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
2968 ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
2969
2970 alloc_size -= cursor_blocks;
2971 alloc->end -= cursor_blocks;
2972
2973
2974 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
2975 struct drm_plane *plane = &intel_plane->base;
2976 struct drm_framebuffer *fb = plane->state->fb;
2977 int id = skl_wm_plane_id(intel_plane);
2978
2979 if (!to_intel_plane_state(plane->state)->visible)
2980 continue;
2981
2982 if (plane->type == DRM_PLANE_TYPE_CURSOR)
2983 continue;
2984
2985 minimum[id] = 8;
2986 alloc_size -= minimum[id];
2987 y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0;
2988 alloc_size -= y_minimum[id];
2989 }
2990
2991
2992
2993
2994
2995
2996
2997 total_data_rate = skl_get_total_relative_data_rate(cstate);
2998
2999 start = alloc->start;
3000 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
3001 struct drm_plane *plane = &intel_plane->base;
3002 struct drm_plane_state *pstate = intel_plane->base.state;
3003 unsigned int data_rate, y_data_rate;
3004 uint16_t plane_blocks, y_plane_blocks = 0;
3005 int id = skl_wm_plane_id(intel_plane);
3006
3007 if (!to_intel_plane_state(pstate)->visible)
3008 continue;
3009 if (plane->type == DRM_PLANE_TYPE_CURSOR)
3010 continue;
3011
3012 data_rate = skl_plane_relative_data_rate(cstate, pstate, 0);
3013
3014
3015
3016
3017
3018
3019 plane_blocks = minimum[id];
3020 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
3021 total_data_rate);
3022
3023 ddb->plane[pipe][id].start = start;
3024 ddb->plane[pipe][id].end = start + plane_blocks;
3025
3026 start += plane_blocks;
3027
3028
3029
3030
3031 if (pstate->fb->pixel_format == DRM_FORMAT_NV12) {
3032 y_data_rate = skl_plane_relative_data_rate(cstate,
3033 pstate,
3034 1);
3035 y_plane_blocks = y_minimum[id];
3036 y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
3037 total_data_rate);
3038
3039 ddb->y_plane[pipe][id].start = start;
3040 ddb->y_plane[pipe][id].end = start + y_plane_blocks;
3041
3042 start += y_plane_blocks;
3043 }
3044
3045 }
3046
3047}
3048
3049static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
3050{
3051
3052 return config->base.adjusted_mode.crtc_clock;
3053}
3054
3055
3056
3057
3058
3059
3060
3061static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latency)
3062{
3063 uint32_t wm_intermediate_val, ret;
3064
3065 if (latency == 0)
3066 return UINT_MAX;
3067
3068 wm_intermediate_val = latency * pixel_rate * cpp / 512;
3069 ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
3070
3071 return ret;
3072}
3073
3074static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
3075 uint32_t horiz_pixels, uint8_t cpp,
3076 uint64_t tiling, uint32_t latency)
3077{
3078 uint32_t ret;
3079 uint32_t plane_bytes_per_line, plane_blocks_per_line;
3080 uint32_t wm_intermediate_val;
3081
3082 if (latency == 0)
3083 return UINT_MAX;
3084
3085 plane_bytes_per_line = horiz_pixels * cpp;
3086
3087 if (tiling == I915_FORMAT_MOD_Y_TILED ||
3088 tiling == I915_FORMAT_MOD_Yf_TILED) {
3089 plane_bytes_per_line *= 4;
3090 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
3091 plane_blocks_per_line /= 4;
3092 } else {
3093 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
3094 }
3095
3096 wm_intermediate_val = latency * pixel_rate;
3097 ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
3098 plane_blocks_per_line;
3099
3100 return ret;
3101}
3102
3103static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
3104 const struct intel_crtc *intel_crtc)
3105{
3106 struct drm_device *dev = intel_crtc->base.dev;
3107 struct drm_i915_private *dev_priv = dev->dev_private;
3108 const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
3109
3110
3111
3112
3113
3114 if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe)))
3115 return true;
3116
3117 return false;
3118}
3119
3120static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3121 struct intel_crtc_state *cstate,
3122 struct intel_plane *intel_plane,
3123 uint16_t ddb_allocation,
3124 int level,
3125 uint16_t *out_blocks,
3126 uint8_t *out_lines )
3127{
3128 struct drm_plane *plane = &intel_plane->base;
3129 struct drm_framebuffer *fb = plane->state->fb;
3130 struct intel_plane_state *intel_pstate =
3131 to_intel_plane_state(plane->state);
3132 uint32_t latency = dev_priv->wm.skl_latency[level];
3133 uint32_t method1, method2;
3134 uint32_t plane_bytes_per_line, plane_blocks_per_line;
3135 uint32_t res_blocks, res_lines;
3136 uint32_t selected_result;
3137 uint8_t cpp;
3138 uint32_t width = 0, height = 0;
3139
3140 if (latency == 0 || !cstate->base.active || !intel_pstate->visible)
3141 return false;
3142
3143 width = drm_rect_width(&intel_pstate->src) >> 16;
3144 height = drm_rect_height(&intel_pstate->src) >> 16;
3145
3146 if (intel_rotation_90_or_270(plane->state->rotation))
3147 swap(width, height);
3148
3149 cpp = drm_format_plane_cpp(fb->pixel_format, 0);
3150 method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate),
3151 cpp, latency);
3152 method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate),
3153 cstate->base.adjusted_mode.crtc_htotal,
3154 width,
3155 cpp,
3156 fb->modifier[0],
3157 latency);
3158
3159 plane_bytes_per_line = width * cpp;
3160 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
3161
3162 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
3163 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
3164 uint32_t min_scanlines = 4;
3165 uint32_t y_tile_minimum;
3166 if (intel_rotation_90_or_270(plane->state->rotation)) {
3167 int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
3168 drm_format_plane_cpp(fb->pixel_format, 1) :
3169 drm_format_plane_cpp(fb->pixel_format, 0);
3170
3171 switch (cpp) {
3172 case 1:
3173 min_scanlines = 16;
3174 break;
3175 case 2:
3176 min_scanlines = 8;
3177 break;
3178 case 8:
3179 WARN(1, "Unsupported pixel depth for rotation");
3180 }
3181 }
3182 y_tile_minimum = plane_blocks_per_line * min_scanlines;
3183 selected_result = max(method2, y_tile_minimum);
3184 } else {
3185 if ((ddb_allocation / plane_blocks_per_line) >= 1)
3186 selected_result = min(method1, method2);
3187 else
3188 selected_result = method1;
3189 }
3190
3191 res_blocks = selected_result + 1;
3192 res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
3193
3194 if (level >= 1 && level <= 7) {
3195 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
3196 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)
3197 res_lines += 4;
3198 else
3199 res_blocks++;
3200 }
3201
3202 if (res_blocks >= ddb_allocation || res_lines > 31)
3203 return false;
3204
3205 *out_blocks = res_blocks;
3206 *out_lines = res_lines;
3207
3208 return true;
3209}
3210
3211static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
3212 struct skl_ddb_allocation *ddb,
3213 struct intel_crtc_state *cstate,
3214 int level,
3215 struct skl_wm_level *result)
3216{
3217 struct drm_device *dev = dev_priv->dev;
3218 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3219 struct intel_plane *intel_plane;
3220 uint16_t ddb_blocks;
3221 enum pipe pipe = intel_crtc->pipe;
3222
3223 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
3224 int i = skl_wm_plane_id(intel_plane);
3225
3226 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
3227
3228 result->plane_en[i] = skl_compute_plane_wm(dev_priv,
3229 cstate,
3230 intel_plane,
3231 ddb_blocks,
3232 level,
3233 &result->plane_res_b[i],
3234 &result->plane_res_l[i]);
3235 }
3236}
3237
3238static uint32_t
3239skl_compute_linetime_wm(struct intel_crtc_state *cstate)
3240{
3241 if (!cstate->base.active)
3242 return 0;
3243
3244 if (WARN_ON(skl_pipe_pixel_rate(cstate) == 0))
3245 return 0;
3246
3247 return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000,
3248 skl_pipe_pixel_rate(cstate));
3249}
3250
3251static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
3252 struct skl_wm_level *trans_wm )
3253{
3254 struct drm_crtc *crtc = cstate->base.crtc;
3255 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3256 struct intel_plane *intel_plane;
3257
3258 if (!cstate->base.active)
3259 return;
3260
3261
3262 for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) {
3263 int i = skl_wm_plane_id(intel_plane);
3264
3265 trans_wm->plane_en[i] = false;
3266 }
3267}
3268
3269static void skl_compute_pipe_wm(struct intel_crtc_state *cstate,
3270 struct skl_ddb_allocation *ddb,
3271 struct skl_pipe_wm *pipe_wm)
3272{
3273 struct drm_device *dev = cstate->base.crtc->dev;
3274 const struct drm_i915_private *dev_priv = dev->dev_private;
3275 int level, max_level = ilk_wm_max_level(dev);
3276
3277 for (level = 0; level <= max_level; level++) {
3278 skl_compute_wm_level(dev_priv, ddb, cstate,
3279 level, &pipe_wm->wm[level]);
3280 }
3281 pipe_wm->linetime = skl_compute_linetime_wm(cstate);
3282
3283 skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
3284}
3285
3286static void skl_compute_wm_results(struct drm_device *dev,
3287 struct skl_pipe_wm *p_wm,
3288 struct skl_wm_values *r,
3289 struct intel_crtc *intel_crtc)
3290{
3291 int level, max_level = ilk_wm_max_level(dev);
3292 enum pipe pipe = intel_crtc->pipe;
3293 uint32_t temp;
3294 int i;
3295
3296 for (level = 0; level <= max_level; level++) {
3297 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3298 temp = 0;
3299
3300 temp |= p_wm->wm[level].plane_res_l[i] <<
3301 PLANE_WM_LINES_SHIFT;
3302 temp |= p_wm->wm[level].plane_res_b[i];
3303 if (p_wm->wm[level].plane_en[i])
3304 temp |= PLANE_WM_EN;
3305
3306 r->plane[pipe][i][level] = temp;
3307 }
3308
3309 temp = 0;
3310
3311 temp |= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
3312 temp |= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
3313
3314 if (p_wm->wm[level].plane_en[PLANE_CURSOR])
3315 temp |= PLANE_WM_EN;
3316
3317 r->plane[pipe][PLANE_CURSOR][level] = temp;
3318
3319 }
3320
3321
3322 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3323 temp = 0;
3324 temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
3325 temp |= p_wm->trans_wm.plane_res_b[i];
3326 if (p_wm->trans_wm.plane_en[i])
3327 temp |= PLANE_WM_EN;
3328
3329 r->plane_trans[pipe][i] = temp;
3330 }
3331
3332 temp = 0;
3333 temp |= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
3334 temp |= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
3335 if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
3336 temp |= PLANE_WM_EN;
3337
3338 r->plane_trans[pipe][PLANE_CURSOR] = temp;
3339
3340 r->wm_linetime[pipe] = p_wm->linetime;
3341}
3342
3343static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
3344 i915_reg_t reg,
3345 const struct skl_ddb_entry *entry)
3346{
3347 if (entry->end)
3348 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
3349 else
3350 I915_WRITE(reg, 0);
3351}
3352
3353static void skl_write_wm_values(struct drm_i915_private *dev_priv,
3354 const struct skl_wm_values *new)
3355{
3356 struct drm_device *dev = dev_priv->dev;
3357 struct intel_crtc *crtc;
3358
3359 for_each_intel_crtc(dev, crtc) {
3360 int i, level, max_level = ilk_wm_max_level(dev);
3361 enum pipe pipe = crtc->pipe;
3362
3363 if (!new->dirty[pipe])
3364 continue;
3365
3366 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
3367
3368 for (level = 0; level <= max_level; level++) {
3369 for (i = 0; i < intel_num_planes(crtc); i++)
3370 I915_WRITE(PLANE_WM(pipe, i, level),
3371 new->plane[pipe][i][level]);
3372 I915_WRITE(CUR_WM(pipe, level),
3373 new->plane[pipe][PLANE_CURSOR][level]);
3374 }
3375 for (i = 0; i < intel_num_planes(crtc); i++)
3376 I915_WRITE(PLANE_WM_TRANS(pipe, i),
3377 new->plane_trans[pipe][i]);
3378 I915_WRITE(CUR_WM_TRANS(pipe),
3379 new->plane_trans[pipe][PLANE_CURSOR]);
3380
3381 for (i = 0; i < intel_num_planes(crtc); i++) {
3382 skl_ddb_entry_write(dev_priv,
3383 PLANE_BUF_CFG(pipe, i),
3384 &new->ddb.plane[pipe][i]);
3385 skl_ddb_entry_write(dev_priv,
3386 PLANE_NV12_BUF_CFG(pipe, i),
3387 &new->ddb.y_plane[pipe][i]);
3388 }
3389
3390 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
3391 &new->ddb.plane[pipe][PLANE_CURSOR]);
3392 }
3393}
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419static void
3420skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
3421{
3422 int plane;
3423
3424 DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
3425
3426 for_each_plane(dev_priv, pipe, plane) {
3427 I915_WRITE(PLANE_SURF(pipe, plane),
3428 I915_READ(PLANE_SURF(pipe, plane)));
3429 }
3430 I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
3431}
3432
3433static bool
3434skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
3435 const struct skl_ddb_allocation *new,
3436 enum pipe pipe)
3437{
3438 uint16_t old_size, new_size;
3439
3440 old_size = skl_ddb_entry_size(&old->pipe[pipe]);
3441 new_size = skl_ddb_entry_size(&new->pipe[pipe]);
3442
3443 return old_size != new_size &&
3444 new->pipe[pipe].start >= old->pipe[pipe].start &&
3445 new->pipe[pipe].end <= old->pipe[pipe].end;
3446}
3447
3448static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
3449 struct skl_wm_values *new_values)
3450{
3451 struct drm_device *dev = dev_priv->dev;
3452 struct skl_ddb_allocation *cur_ddb, *new_ddb;
3453 bool reallocated[I915_MAX_PIPES] = {};
3454 struct intel_crtc *crtc;
3455 enum pipe pipe;
3456
3457 new_ddb = &new_values->ddb;
3458 cur_ddb = &dev_priv->wm.skl_hw.ddb;
3459
3460
3461
3462
3463
3464
3465
3466
3467 for_each_intel_crtc(dev, crtc) {
3468 if (!crtc->active)
3469 continue;
3470
3471 pipe = crtc->pipe;
3472
3473 if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
3474 continue;
3475
3476 skl_wm_flush_pipe(dev_priv, pipe, 1);
3477 intel_wait_for_vblank(dev, pipe);
3478
3479 reallocated[pipe] = true;
3480 }
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490 for_each_intel_crtc(dev, crtc) {
3491 if (!crtc->active)
3492 continue;
3493
3494 pipe = crtc->pipe;
3495
3496 if (reallocated[pipe])
3497 continue;
3498
3499 if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
3500 skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
3501 skl_wm_flush_pipe(dev_priv, pipe, 2);
3502 intel_wait_for_vblank(dev, pipe);
3503 reallocated[pipe] = true;
3504 }
3505 }
3506
3507
3508
3509
3510
3511
3512
3513 for_each_intel_crtc(dev, crtc) {
3514 if (!crtc->active)
3515 continue;
3516
3517 pipe = crtc->pipe;
3518
3519
3520
3521
3522
3523 if (reallocated[pipe])
3524 continue;
3525
3526 skl_wm_flush_pipe(dev_priv, pipe, 3);
3527 }
3528}
3529
3530static bool skl_update_pipe_wm(struct drm_crtc *crtc,
3531 struct skl_ddb_allocation *ddb,
3532 struct skl_pipe_wm *pipe_wm )
3533{
3534 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3535 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3536
3537 skl_allocate_pipe_ddb(cstate, ddb);
3538 skl_compute_pipe_wm(cstate, ddb, pipe_wm);
3539
3540 if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
3541 return false;
3542
3543 intel_crtc->wm.active.skl = *pipe_wm;
3544
3545 return true;
3546}
3547
3548static void skl_update_other_pipe_wm(struct drm_device *dev,
3549 struct drm_crtc *crtc,
3550 struct skl_wm_values *r)
3551{
3552 struct intel_crtc *intel_crtc;
3553 struct intel_crtc *this_crtc = to_intel_crtc(crtc);
3554
3555
3556
3557
3558
3559
3560
3561 if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
3562 return;
3563
3564
3565
3566
3567
3568 for_each_intel_crtc(dev, intel_crtc) {
3569 struct skl_pipe_wm pipe_wm = {};
3570 bool wm_changed;
3571
3572 if (this_crtc->pipe == intel_crtc->pipe)
3573 continue;
3574
3575 if (!intel_crtc->active)
3576 continue;
3577
3578 wm_changed = skl_update_pipe_wm(&intel_crtc->base,
3579 &r->ddb, &pipe_wm);
3580
3581
3582
3583
3584
3585
3586 WARN_ON(!wm_changed);
3587
3588 skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc);
3589 r->dirty[intel_crtc->pipe] = true;
3590 }
3591}
3592
3593static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
3594{
3595 watermarks->wm_linetime[pipe] = 0;
3596 memset(watermarks->plane[pipe], 0,
3597 sizeof(uint32_t) * 8 * I915_MAX_PLANES);
3598 memset(watermarks->plane_trans[pipe],
3599 0, sizeof(uint32_t) * I915_MAX_PLANES);
3600 watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
3601
3602
3603 memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
3604 memset(&watermarks->ddb.plane[pipe], 0,
3605 sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
3606 memset(&watermarks->ddb.y_plane[pipe], 0,
3607 sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
3608 memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
3609 sizeof(struct skl_ddb_entry));
3610
3611}
3612
3613static void skl_update_wm(struct drm_crtc *crtc)
3614{
3615 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3616 struct drm_device *dev = crtc->dev;
3617 struct drm_i915_private *dev_priv = dev->dev_private;
3618 struct skl_wm_values *results = &dev_priv->wm.skl_results;
3619 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3620 struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl;
3621
3622
3623
3624 memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
3625
3626 skl_clear_wm(results, intel_crtc->pipe);
3627
3628 if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm))
3629 return;
3630
3631 skl_compute_wm_results(dev, pipe_wm, results, intel_crtc);
3632 results->dirty[intel_crtc->pipe] = true;
3633
3634 skl_update_other_pipe_wm(dev, crtc, results);
3635 skl_write_wm_values(dev_priv, results);
3636 skl_flush_wm_values(dev_priv, results);
3637
3638
3639 dev_priv->wm.skl_hw = *results;
3640}
3641
3642static void ilk_compute_wm_config(struct drm_device *dev,
3643 struct intel_wm_config *config)
3644{
3645 struct intel_crtc *crtc;
3646
3647
3648 for_each_intel_crtc(dev, crtc) {
3649 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3650
3651 if (!wm->pipe_enabled)
3652 continue;
3653
3654 config->sprites_enabled |= wm->sprites_enabled;
3655 config->sprites_scaled |= wm->sprites_scaled;
3656 config->num_pipes_active++;
3657 }
3658}
3659
3660static void ilk_program_watermarks(struct intel_crtc_state *cstate)
3661{
3662 struct drm_crtc *crtc = cstate->base.crtc;
3663 struct drm_device *dev = crtc->dev;
3664 struct drm_i915_private *dev_priv = to_i915(dev);
3665 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3666 struct ilk_wm_maximums max;
3667 struct intel_wm_config config = {};
3668 struct ilk_wm_values results = {};
3669 enum intel_ddb_partitioning partitioning;
3670
3671 ilk_compute_wm_config(dev, &config);
3672
3673 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
3674 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
3675
3676
3677 if (INTEL_INFO(dev)->gen >= 7 &&
3678 config.num_pipes_active == 1 && config.sprites_enabled) {
3679 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
3680 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
3681
3682 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
3683 } else {
3684 best_lp_wm = &lp_wm_1_2;
3685 }
3686
3687 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3688 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3689
3690 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
3691
3692 ilk_write_wm_values(dev_priv, &results);
3693}
3694
3695static void ilk_update_wm(struct drm_crtc *crtc)
3696{
3697 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3698 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3699
3700 WARN_ON(cstate->base.active != intel_crtc->active);
3701
3702
3703
3704
3705
3706
3707
3708
3709 if (cstate->disable_lp_wm) {
3710 ilk_disable_lp_wm(crtc->dev);
3711 intel_wait_for_vblank(crtc->dev, intel_crtc->pipe);
3712 }
3713
3714 intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk;
3715
3716 ilk_program_watermarks(cstate);
3717}
3718
3719static void skl_pipe_wm_active_state(uint32_t val,
3720 struct skl_pipe_wm *active,
3721 bool is_transwm,
3722 bool is_cursor,
3723 int i,
3724 int level)
3725{
3726 bool is_enabled = (val & PLANE_WM_EN) != 0;
3727
3728 if (!is_transwm) {
3729 if (!is_cursor) {
3730 active->wm[level].plane_en[i] = is_enabled;
3731 active->wm[level].plane_res_b[i] =
3732 val & PLANE_WM_BLOCKS_MASK;
3733 active->wm[level].plane_res_l[i] =
3734 (val >> PLANE_WM_LINES_SHIFT) &
3735 PLANE_WM_LINES_MASK;
3736 } else {
3737 active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
3738 active->wm[level].plane_res_b[PLANE_CURSOR] =
3739 val & PLANE_WM_BLOCKS_MASK;
3740 active->wm[level].plane_res_l[PLANE_CURSOR] =
3741 (val >> PLANE_WM_LINES_SHIFT) &
3742 PLANE_WM_LINES_MASK;
3743 }
3744 } else {
3745 if (!is_cursor) {
3746 active->trans_wm.plane_en[i] = is_enabled;
3747 active->trans_wm.plane_res_b[i] =
3748 val & PLANE_WM_BLOCKS_MASK;
3749 active->trans_wm.plane_res_l[i] =
3750 (val >> PLANE_WM_LINES_SHIFT) &
3751 PLANE_WM_LINES_MASK;
3752 } else {
3753 active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
3754 active->trans_wm.plane_res_b[PLANE_CURSOR] =
3755 val & PLANE_WM_BLOCKS_MASK;
3756 active->trans_wm.plane_res_l[PLANE_CURSOR] =
3757 (val >> PLANE_WM_LINES_SHIFT) &
3758 PLANE_WM_LINES_MASK;
3759 }
3760 }
3761}
3762
3763static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3764{
3765 struct drm_device *dev = crtc->dev;
3766 struct drm_i915_private *dev_priv = dev->dev_private;
3767 struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
3768 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3769 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3770 struct skl_pipe_wm *active = &cstate->wm.optimal.skl;
3771 enum pipe pipe = intel_crtc->pipe;
3772 int level, i, max_level;
3773 uint32_t temp;
3774
3775 max_level = ilk_wm_max_level(dev);
3776
3777 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3778
3779 for (level = 0; level <= max_level; level++) {
3780 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3781 hw->plane[pipe][i][level] =
3782 I915_READ(PLANE_WM(pipe, i, level));
3783 hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
3784 }
3785
3786 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3787 hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
3788 hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
3789
3790 if (!intel_crtc->active)
3791 return;
3792
3793 hw->dirty[pipe] = true;
3794
3795 active->linetime = hw->wm_linetime[pipe];
3796
3797 for (level = 0; level <= max_level; level++) {
3798 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3799 temp = hw->plane[pipe][i][level];
3800 skl_pipe_wm_active_state(temp, active, false,
3801 false, i, level);
3802 }
3803 temp = hw->plane[pipe][PLANE_CURSOR][level];
3804 skl_pipe_wm_active_state(temp, active, false, true, i, level);
3805 }
3806
3807 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3808 temp = hw->plane_trans[pipe][i];
3809 skl_pipe_wm_active_state(temp, active, true, false, i, 0);
3810 }
3811
3812 temp = hw->plane_trans[pipe][PLANE_CURSOR];
3813 skl_pipe_wm_active_state(temp, active, true, true, i, 0);
3814
3815 intel_crtc->wm.active.skl = *active;
3816}
3817
3818void skl_wm_get_hw_state(struct drm_device *dev)
3819{
3820 struct drm_i915_private *dev_priv = dev->dev_private;
3821 struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
3822 struct drm_crtc *crtc;
3823
3824 skl_ddb_get_hw_state(dev_priv, ddb);
3825 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3826 skl_pipe_wm_get_hw_state(crtc);
3827}
3828
3829static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3830{
3831 struct drm_device *dev = crtc->dev;
3832 struct drm_i915_private *dev_priv = dev->dev_private;
3833 struct ilk_wm_values *hw = &dev_priv->wm.hw;
3834 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3835 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3836 struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
3837 enum pipe pipe = intel_crtc->pipe;
3838 static const i915_reg_t wm0_pipe_reg[] = {
3839 [PIPE_A] = WM0_PIPEA_ILK,
3840 [PIPE_B] = WM0_PIPEB_ILK,
3841 [PIPE_C] = WM0_PIPEC_IVB,
3842 };
3843
3844 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
3845 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3846 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3847
3848 active->pipe_enabled = intel_crtc->active;
3849
3850 if (active->pipe_enabled) {
3851 u32 tmp = hw->wm_pipe[pipe];
3852
3853
3854
3855
3856
3857
3858
3859 active->wm[0].enable = true;
3860 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
3861 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
3862 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
3863 active->linetime = hw->wm_linetime[pipe];
3864 } else {
3865 int level, max_level = ilk_wm_max_level(dev);
3866
3867
3868
3869
3870
3871
3872 for (level = 0; level <= max_level; level++)
3873 active->wm[level].enable = true;
3874 }
3875
3876 intel_crtc->wm.active.ilk = *active;
3877}
3878
3879#define _FW_WM(value, plane) \
3880 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3881#define _FW_WM_VLV(value, plane) \
3882 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3883
3884static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3885 struct vlv_wm_values *wm)
3886{
3887 enum pipe pipe;
3888 uint32_t tmp;
3889
3890 for_each_pipe(dev_priv, pipe) {
3891 tmp = I915_READ(VLV_DDL(pipe));
3892
3893 wm->ddl[pipe].primary =
3894 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3895 wm->ddl[pipe].cursor =
3896 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3897 wm->ddl[pipe].sprite[0] =
3898 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3899 wm->ddl[pipe].sprite[1] =
3900 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3901 }
3902
3903 tmp = I915_READ(DSPFW1);
3904 wm->sr.plane = _FW_WM(tmp, SR);
3905 wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
3906 wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
3907 wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
3908
3909 tmp = I915_READ(DSPFW2);
3910 wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
3911 wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
3912 wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
3913
3914 tmp = I915_READ(DSPFW3);
3915 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3916
3917 if (IS_CHERRYVIEW(dev_priv)) {
3918 tmp = I915_READ(DSPFW7_CHV);
3919 wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
3920 wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
3921
3922 tmp = I915_READ(DSPFW8_CHV);
3923 wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
3924 wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
3925
3926 tmp = I915_READ(DSPFW9_CHV);
3927 wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
3928 wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
3929
3930 tmp = I915_READ(DSPHOWM);
3931 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3932 wm->pipe[PIPE_C].sprite[1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3933 wm->pipe[PIPE_C].sprite[0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3934 wm->pipe[PIPE_C].primary |= _FW_WM(tmp, PLANEC_HI) << 8;
3935 wm->pipe[PIPE_B].sprite[1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3936 wm->pipe[PIPE_B].sprite[0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3937 wm->pipe[PIPE_B].primary |= _FW_WM(tmp, PLANEB_HI) << 8;
3938 wm->pipe[PIPE_A].sprite[1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3939 wm->pipe[PIPE_A].sprite[0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3940 wm->pipe[PIPE_A].primary |= _FW_WM(tmp, PLANEA_HI) << 8;
3941 } else {
3942 tmp = I915_READ(DSPFW7);
3943 wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
3944 wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
3945
3946 tmp = I915_READ(DSPHOWM);
3947 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3948 wm->pipe[PIPE_B].sprite[1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3949 wm->pipe[PIPE_B].sprite[0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3950 wm->pipe[PIPE_B].primary |= _FW_WM(tmp, PLANEB_HI) << 8;
3951 wm->pipe[PIPE_A].sprite[1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3952 wm->pipe[PIPE_A].sprite[0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3953 wm->pipe[PIPE_A].primary |= _FW_WM(tmp, PLANEA_HI) << 8;
3954 }
3955}
3956
3957#undef _FW_WM
3958#undef _FW_WM_VLV
3959
3960void vlv_wm_get_hw_state(struct drm_device *dev)
3961{
3962 struct drm_i915_private *dev_priv = to_i915(dev);
3963 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
3964 struct intel_plane *plane;
3965 enum pipe pipe;
3966 u32 val;
3967
3968 vlv_read_wm_values(dev_priv, wm);
3969
3970 for_each_intel_plane(dev, plane) {
3971 switch (plane->base.type) {
3972 int sprite;
3973 case DRM_PLANE_TYPE_CURSOR:
3974 plane->wm.fifo_size = 63;
3975 break;
3976 case DRM_PLANE_TYPE_PRIMARY:
3977 plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
3978 break;
3979 case DRM_PLANE_TYPE_OVERLAY:
3980 sprite = plane->plane;
3981 plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
3982 break;
3983 }
3984 }
3985
3986 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3987 wm->level = VLV_WM_LEVEL_PM2;
3988
3989 if (IS_CHERRYVIEW(dev_priv)) {
3990 mutex_lock(&dev_priv->rps.hw_lock);
3991
3992 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
3993 if (val & DSP_MAXFIFO_PM5_ENABLE)
3994 wm->level = VLV_WM_LEVEL_PM5;
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
4006 val |= FORCE_DDR_FREQ_REQ_ACK;
4007 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
4008
4009 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
4010 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
4011 DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
4012 "assuming DDR DVFS is disabled\n");
4013 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
4014 } else {
4015 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
4016 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
4017 wm->level = VLV_WM_LEVEL_DDR_DVFS;
4018 }
4019
4020 mutex_unlock(&dev_priv->rps.hw_lock);
4021 }
4022
4023 for_each_pipe(dev_priv, pipe)
4024 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
4025 pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
4026 wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
4027
4028 DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4029 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4030}
4031
4032void ilk_wm_get_hw_state(struct drm_device *dev)
4033{
4034 struct drm_i915_private *dev_priv = dev->dev_private;
4035 struct ilk_wm_values *hw = &dev_priv->wm.hw;
4036 struct drm_crtc *crtc;
4037
4038 for_each_crtc(dev, crtc)
4039 ilk_pipe_wm_get_hw_state(crtc);
4040
4041 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
4042 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
4043 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
4044
4045 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
4046 if (INTEL_INFO(dev)->gen >= 7) {
4047 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
4048 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
4049 }
4050
4051 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4052 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4053 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4054 else if (IS_IVYBRIDGE(dev))
4055 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4056 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4057
4058 hw->enable_fbc_wm =
4059 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4060}
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094void intel_update_watermarks(struct drm_crtc *crtc)
4095{
4096 struct drm_i915_private *dev_priv = crtc->dev->dev_private;
4097
4098 if (dev_priv->display.update_wm)
4099 dev_priv->display.update_wm(crtc);
4100}
4101
4102
4103
4104
4105DEFINE_SPINLOCK(mchdev_lock);
4106
4107
4108
4109static struct drm_i915_private *i915_mch_dev;
4110
4111bool ironlake_set_drps(struct drm_device *dev, u8 val)
4112{
4113 struct drm_i915_private *dev_priv = dev->dev_private;
4114 u16 rgvswctl;
4115
4116 assert_spin_locked(&mchdev_lock);
4117
4118 rgvswctl = I915_READ16(MEMSWCTL);
4119 if (rgvswctl & MEMCTL_CMD_STS) {
4120 DRM_DEBUG("gpu busy, RCS change rejected\n");
4121 return false;
4122 }
4123
4124 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
4125 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
4126 I915_WRITE16(MEMSWCTL, rgvswctl);
4127 POSTING_READ16(MEMSWCTL);
4128
4129 rgvswctl |= MEMCTL_CMD_STS;
4130 I915_WRITE16(MEMSWCTL, rgvswctl);
4131
4132 return true;
4133}
4134
4135static void ironlake_enable_drps(struct drm_device *dev)
4136{
4137 struct drm_i915_private *dev_priv = dev->dev_private;
4138 u32 rgvmodectl;
4139 u8 fmax, fmin, fstart, vstart;
4140
4141 spin_lock_irq(&mchdev_lock);
4142
4143 rgvmodectl = I915_READ(MEMMODECTL);
4144
4145
4146 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
4147 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
4148
4149
4150 I915_WRITE(RCUPEI, 100000);
4151 I915_WRITE(RCDNEI, 100000);
4152
4153
4154 I915_WRITE(RCBMAXAVG, 90000);
4155 I915_WRITE(RCBMINAVG, 80000);
4156
4157 I915_WRITE(MEMIHYST, 1);
4158
4159
4160 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
4161 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
4162 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
4163 MEMMODE_FSTART_SHIFT;
4164
4165 vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
4166 PXVFREQ_PX_SHIFT;
4167
4168 dev_priv->ips.fmax = fmax;
4169 dev_priv->ips.fstart = fstart;
4170
4171 dev_priv->ips.max_delay = fstart;
4172 dev_priv->ips.min_delay = fmin;
4173 dev_priv->ips.cur_delay = fstart;
4174
4175 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
4176 fmax, fmin, fstart);
4177
4178 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
4179
4180
4181
4182
4183
4184 I915_WRITE(VIDSTART, vstart);
4185 POSTING_READ(VIDSTART);
4186
4187 rgvmodectl |= MEMMODE_SWMODE_EN;
4188 I915_WRITE(MEMMODECTL, rgvmodectl);
4189
4190 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
4191 DRM_ERROR("stuck trying to change perf mode\n");
4192 mdelay(1);
4193
4194 ironlake_set_drps(dev, fstart);
4195
4196 dev_priv->ips.last_count1 = I915_READ(DMIEC) +
4197 I915_READ(DDREC) + I915_READ(CSIEC);
4198 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
4199 dev_priv->ips.last_count2 = I915_READ(GFXEC);
4200 dev_priv->ips.last_time2 = ktime_get_raw_ns();
4201
4202 spin_unlock_irq(&mchdev_lock);
4203}
4204
4205static void ironlake_disable_drps(struct drm_device *dev)
4206{
4207 struct drm_i915_private *dev_priv = dev->dev_private;
4208 u16 rgvswctl;
4209
4210 spin_lock_irq(&mchdev_lock);
4211
4212 rgvswctl = I915_READ16(MEMSWCTL);
4213
4214
4215 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
4216 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
4217 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
4218 I915_WRITE(DEIIR, DE_PCU_EVENT);
4219 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
4220
4221
4222 ironlake_set_drps(dev, dev_priv->ips.fstart);
4223 mdelay(1);
4224 rgvswctl |= MEMCTL_CMD_STS;
4225 I915_WRITE(MEMSWCTL, rgvswctl);
4226 mdelay(1);
4227
4228 spin_unlock_irq(&mchdev_lock);
4229}
4230
4231
4232
4233
4234
4235
4236static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
4237{
4238 u32 limits;
4239
4240
4241
4242
4243
4244
4245
4246 if (IS_GEN9(dev_priv->dev)) {
4247 limits = (dev_priv->rps.max_freq_softlimit) << 23;
4248 if (val <= dev_priv->rps.min_freq_softlimit)
4249 limits |= (dev_priv->rps.min_freq_softlimit) << 14;
4250 } else {
4251 limits = dev_priv->rps.max_freq_softlimit << 24;
4252 if (val <= dev_priv->rps.min_freq_softlimit)
4253 limits |= dev_priv->rps.min_freq_softlimit << 16;
4254 }
4255
4256 return limits;
4257}
4258
4259static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
4260{
4261 int new_power;
4262 u32 threshold_up = 0, threshold_down = 0;
4263 u32 ei_up = 0, ei_down = 0;
4264
4265 new_power = dev_priv->rps.power;
4266 switch (dev_priv->rps.power) {
4267 case LOW_POWER:
4268 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4269 new_power = BETWEEN;
4270 break;
4271
4272 case BETWEEN:
4273 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4274 new_power = LOW_POWER;
4275 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4276 new_power = HIGH_POWER;
4277 break;
4278
4279 case HIGH_POWER:
4280 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4281 new_power = BETWEEN;
4282 break;
4283 }
4284
4285 if (val <= dev_priv->rps.min_freq_softlimit)
4286 new_power = LOW_POWER;
4287 if (val >= dev_priv->rps.max_freq_softlimit)
4288 new_power = HIGH_POWER;
4289 if (new_power == dev_priv->rps.power)
4290 return;
4291
4292
4293 switch (new_power) {
4294 case LOW_POWER:
4295
4296 ei_up = 16000;
4297 threshold_up = 95;
4298
4299
4300 ei_down = 32000;
4301 threshold_down = 85;
4302 break;
4303
4304 case BETWEEN:
4305
4306 ei_up = 13000;
4307 threshold_up = 90;
4308
4309
4310 ei_down = 32000;
4311 threshold_down = 75;
4312 break;
4313
4314 case HIGH_POWER:
4315
4316 ei_up = 10000;
4317 threshold_up = 85;
4318
4319
4320 ei_down = 32000;
4321 threshold_down = 60;
4322 break;
4323 }
4324
4325 I915_WRITE(GEN6_RP_UP_EI,
4326 GT_INTERVAL_FROM_US(dev_priv, ei_up));
4327 I915_WRITE(GEN6_RP_UP_THRESHOLD,
4328 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
4329
4330 I915_WRITE(GEN6_RP_DOWN_EI,
4331 GT_INTERVAL_FROM_US(dev_priv, ei_down));
4332 I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
4333 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
4334
4335 I915_WRITE(GEN6_RP_CONTROL,
4336 GEN6_RP_MEDIA_TURBO |
4337 GEN6_RP_MEDIA_HW_NORMAL_MODE |
4338 GEN6_RP_MEDIA_IS_GFX |
4339 GEN6_RP_ENABLE |
4340 GEN6_RP_UP_BUSY_AVG |
4341 GEN6_RP_DOWN_IDLE_AVG);
4342
4343 dev_priv->rps.power = new_power;
4344 dev_priv->rps.up_threshold = threshold_up;
4345 dev_priv->rps.down_threshold = threshold_down;
4346 dev_priv->rps.last_adj = 0;
4347}
4348
4349static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
4350{
4351 u32 mask = 0;
4352
4353 if (val > dev_priv->rps.min_freq_softlimit)
4354 mask |= GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
4355 if (val < dev_priv->rps.max_freq_softlimit)
4356 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
4357
4358 mask &= dev_priv->pm_rps_events;
4359
4360 return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
4361}
4362
4363
4364
4365
4366static void gen6_set_rps(struct drm_device *dev, u8 val)
4367{
4368 struct drm_i915_private *dev_priv = dev->dev_private;
4369
4370
4371 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
4372 return;
4373
4374 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4375 WARN_ON(val > dev_priv->rps.max_freq);
4376 WARN_ON(val < dev_priv->rps.min_freq);
4377
4378
4379
4380
4381 if (val != dev_priv->rps.cur_freq) {
4382 gen6_set_rps_thresholds(dev_priv, val);
4383
4384 if (IS_GEN9(dev))
4385 I915_WRITE(GEN6_RPNSWREQ,
4386 GEN9_FREQUENCY(val));
4387 else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4388 I915_WRITE(GEN6_RPNSWREQ,
4389 HSW_FREQUENCY(val));
4390 else
4391 I915_WRITE(GEN6_RPNSWREQ,
4392 GEN6_FREQUENCY(val) |
4393 GEN6_OFFSET(0) |
4394 GEN6_AGGRESSIVE_TURBO);
4395 }
4396
4397
4398
4399
4400 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
4401 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
4402
4403 POSTING_READ(GEN6_RPNSWREQ);
4404
4405 dev_priv->rps.cur_freq = val;
4406 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
4407}
4408
4409static void valleyview_set_rps(struct drm_device *dev, u8 val)
4410{
4411 struct drm_i915_private *dev_priv = dev->dev_private;
4412
4413 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4414 WARN_ON(val > dev_priv->rps.max_freq);
4415 WARN_ON(val < dev_priv->rps.min_freq);
4416
4417 if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
4418 "Odd GPU freq value\n"))
4419 val &= ~1;
4420
4421 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
4422
4423 if (val != dev_priv->rps.cur_freq) {
4424 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
4425 if (!IS_CHERRYVIEW(dev_priv))
4426 gen6_set_rps_thresholds(dev_priv, val);
4427 }
4428
4429 dev_priv->rps.cur_freq = val;
4430 trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
4431}
4432
4433
4434
4435
4436
4437
4438
4439
4440static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
4441{
4442 u32 val = dev_priv->rps.idle_freq;
4443
4444 if (dev_priv->rps.cur_freq <= val)
4445 return;
4446
4447
4448
4449 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
4450 valleyview_set_rps(dev_priv->dev, val);
4451 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
4452}
4453
4454void gen6_rps_busy(struct drm_i915_private *dev_priv)
4455{
4456 mutex_lock(&dev_priv->rps.hw_lock);
4457 if (dev_priv->rps.enabled) {
4458 if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED))
4459 gen6_rps_reset_ei(dev_priv);
4460 I915_WRITE(GEN6_PMINTRMSK,
4461 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4462 }
4463 mutex_unlock(&dev_priv->rps.hw_lock);
4464}
4465
4466void gen6_rps_idle(struct drm_i915_private *dev_priv)
4467{
4468 struct drm_device *dev = dev_priv->dev;
4469
4470 mutex_lock(&dev_priv->rps.hw_lock);
4471 if (dev_priv->rps.enabled) {
4472 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
4473 vlv_set_rps_idle(dev_priv);
4474 else
4475 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4476 dev_priv->rps.last_adj = 0;
4477 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4478 }
4479 mutex_unlock(&dev_priv->rps.hw_lock);
4480
4481 spin_lock(&dev_priv->rps.client_lock);
4482 while (!list_empty(&dev_priv->rps.clients))
4483 list_del_init(dev_priv->rps.clients.next);
4484 spin_unlock(&dev_priv->rps.client_lock);
4485}
4486
4487void gen6_rps_boost(struct drm_i915_private *dev_priv,
4488 struct intel_rps_client *rps,
4489 unsigned long submitted)
4490{
4491
4492
4493
4494 if (!(dev_priv->mm.busy &&
4495 dev_priv->rps.enabled &&
4496 dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
4497 return;
4498
4499
4500
4501
4502 if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
4503 rps = NULL;
4504
4505 spin_lock(&dev_priv->rps.client_lock);
4506 if (rps == NULL || list_empty(&rps->link)) {
4507 spin_lock_irq(&dev_priv->irq_lock);
4508 if (dev_priv->rps.interrupts_enabled) {
4509 dev_priv->rps.client_boost = true;
4510 queue_work(dev_priv->wq, &dev_priv->rps.work);
4511 }
4512 spin_unlock_irq(&dev_priv->irq_lock);
4513
4514 if (rps != NULL) {
4515 list_add(&rps->link, &dev_priv->rps.clients);
4516 rps->boosts++;
4517 } else
4518 dev_priv->rps.boosts++;
4519 }
4520 spin_unlock(&dev_priv->rps.client_lock);
4521}
4522
4523void intel_set_rps(struct drm_device *dev, u8 val)
4524{
4525 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
4526 valleyview_set_rps(dev, val);
4527 else
4528 gen6_set_rps(dev, val);
4529}
4530
4531static void gen9_disable_rps(struct drm_device *dev)
4532{
4533 struct drm_i915_private *dev_priv = dev->dev_private;
4534
4535 I915_WRITE(GEN6_RC_CONTROL, 0);
4536 I915_WRITE(GEN9_PG_ENABLE, 0);
4537}
4538
4539static void gen6_disable_rps(struct drm_device *dev)
4540{
4541 struct drm_i915_private *dev_priv = dev->dev_private;
4542
4543 I915_WRITE(GEN6_RC_CONTROL, 0);
4544 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
4545}
4546
4547static void cherryview_disable_rps(struct drm_device *dev)
4548{
4549 struct drm_i915_private *dev_priv = dev->dev_private;
4550
4551 I915_WRITE(GEN6_RC_CONTROL, 0);
4552}
4553
4554static void valleyview_disable_rps(struct drm_device *dev)
4555{
4556 struct drm_i915_private *dev_priv = dev->dev_private;
4557
4558
4559
4560 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4561
4562 I915_WRITE(GEN6_RC_CONTROL, 0);
4563
4564 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4565}
4566
4567static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
4568{
4569 if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
4570 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
4571 mode = GEN6_RC_CTL_RC6_ENABLE;
4572 else
4573 mode = 0;
4574 }
4575 if (HAS_RC6p(dev))
4576 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
4577 onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
4578 onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
4579 onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
4580
4581 else
4582 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
4583 onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
4584}
4585
4586static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
4587{
4588 struct drm_i915_private *dev_priv = dev->dev_private;
4589 bool enable_rc6 = true;
4590 unsigned long rc6_ctx_base;
4591
4592 if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
4593 DRM_DEBUG_KMS("RC6 Base location not set properly.\n");
4594 enable_rc6 = false;
4595 }
4596
4597
4598
4599
4600
4601 rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
4602 if (!((rc6_ctx_base >= dev_priv->gtt.stolen_reserved_base) &&
4603 (rc6_ctx_base + PAGE_SIZE <= dev_priv->gtt.stolen_reserved_base +
4604 dev_priv->gtt.stolen_reserved_size))) {
4605 DRM_DEBUG_KMS("RC6 Base address not as expected.\n");
4606 enable_rc6 = false;
4607 }
4608
4609 if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
4610 ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
4611 ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
4612 ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
4613 DRM_DEBUG_KMS("Engine Idle wait time not set properly.\n");
4614 enable_rc6 = false;
4615 }
4616
4617 if (!(I915_READ(GEN6_RC_CONTROL) & (GEN6_RC_CTL_RC6_ENABLE |
4618 GEN6_RC_CTL_HW_ENABLE)) &&
4619 ((I915_READ(GEN6_RC_CONTROL) & GEN6_RC_CTL_HW_ENABLE) ||
4620 !(I915_READ(GEN6_RC_STATE) & RC6_STATE))) {
4621 DRM_DEBUG_KMS("HW/SW RC6 is not enabled by BIOS.\n");
4622 enable_rc6 = false;
4623 }
4624
4625 return enable_rc6;
4626}
4627
4628int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
4629{
4630
4631 if (INTEL_INFO(dev)->gen < 6)
4632 return 0;
4633
4634 if (!enable_rc6)
4635 return 0;
4636
4637 if (IS_BROXTON(dev) && !bxt_check_bios_rc6_setup(dev)) {
4638 DRM_INFO("RC6 disabled by BIOS\n");
4639 return 0;
4640 }
4641
4642
4643 if (enable_rc6 >= 0) {
4644 int mask;
4645
4646 if (HAS_RC6p(dev))
4647 mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
4648 INTEL_RC6pp_ENABLE;
4649 else
4650 mask = INTEL_RC6_ENABLE;
4651
4652 if ((enable_rc6 & mask) != enable_rc6)
4653 DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
4654 enable_rc6 & mask, enable_rc6, mask);
4655
4656 return enable_rc6 & mask;
4657 }
4658
4659 if (IS_IVYBRIDGE(dev))
4660 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
4661
4662 return INTEL_RC6_ENABLE;
4663}
4664
4665int intel_enable_rc6(const struct drm_device *dev)
4666{
4667 return i915.enable_rc6;
4668}
4669
4670static void gen6_init_rps_frequencies(struct drm_device *dev)
4671{
4672 struct drm_i915_private *dev_priv = dev->dev_private;
4673 uint32_t rp_state_cap;
4674 u32 ddcc_status = 0;
4675 int ret;
4676
4677
4678 dev_priv->rps.cur_freq = 0;
4679
4680 if (IS_BROXTON(dev)) {
4681 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
4682 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
4683 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
4684 dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
4685 } else {
4686 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
4687 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
4688 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
4689 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
4690 }
4691
4692
4693 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4694
4695 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
4696 if (IS_HASWELL(dev) || IS_BROADWELL(dev) ||
4697 IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
4698 ret = sandybridge_pcode_read(dev_priv,
4699 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
4700 &ddcc_status);
4701 if (0 == ret)
4702 dev_priv->rps.efficient_freq =
4703 clamp_t(u8,
4704 ((ddcc_status >> 8) & 0xff),
4705 dev_priv->rps.min_freq,
4706 dev_priv->rps.max_freq);
4707 }
4708
4709 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
4710
4711
4712 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
4713 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
4714 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
4715 dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
4716 dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
4717 }
4718
4719 dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
4720
4721
4722 if (dev_priv->rps.max_freq_softlimit == 0)
4723 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4724
4725 if (dev_priv->rps.min_freq_softlimit == 0) {
4726 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4727 dev_priv->rps.min_freq_softlimit =
4728 max_t(int, dev_priv->rps.efficient_freq,
4729 intel_freq_opcode(dev_priv, 450));
4730 else
4731 dev_priv->rps.min_freq_softlimit =
4732 dev_priv->rps.min_freq;
4733 }
4734}
4735
4736
4737static void gen9_enable_rps(struct drm_device *dev)
4738{
4739 struct drm_i915_private *dev_priv = dev->dev_private;
4740
4741 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4742
4743 gen6_init_rps_frequencies(dev);
4744
4745
4746 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
4747 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4748 return;
4749 }
4750
4751
4752 I915_WRITE(GEN6_RC_VIDEO_FREQ,
4753 GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
4754
4755
4756 I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
4757 GT_INTERVAL_FROM_US(dev_priv, 1000000));
4758
4759 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
4760
4761
4762
4763
4764 dev_priv->rps.power = HIGH_POWER;
4765 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4766
4767 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4768}
4769
4770static void gen9_enable_rc6(struct drm_device *dev)
4771{
4772 struct drm_i915_private *dev_priv = dev->dev_private;
4773 struct intel_engine_cs *ring;
4774 uint32_t rc6_mask = 0;
4775 int unused;
4776
4777
4778 I915_WRITE(GEN6_RC_STATE, 0);
4779
4780
4781
4782 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4783
4784
4785 I915_WRITE(GEN6_RC_CONTROL, 0);
4786
4787
4788
4789
4790 if (IS_SKYLAKE(dev))
4791 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
4792 else
4793 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
4794 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4795 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4796 for_each_ring(ring, dev_priv, unused)
4797 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4798
4799 if (HAS_GUC_UCODE(dev))
4800 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
4801
4802 I915_WRITE(GEN6_RC_SLEEP, 0);
4803
4804
4805 I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
4806 I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
4807
4808
4809 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4810 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4811 DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
4812
4813 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
4814 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
4815 I915_WRITE(GEN6_RC6_THRESHOLD, 625);
4816 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4817 GEN7_RC_CTL_TO_MODE |
4818 rc6_mask);
4819 } else {
4820 I915_WRITE(GEN6_RC6_THRESHOLD, 37500);
4821 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4822 GEN6_RC_CTL_EI_MODE(1) |
4823 rc6_mask);
4824 }
4825
4826
4827
4828
4829
4830 if (NEEDS_WaRsDisableCoarsePowerGating(dev))
4831 I915_WRITE(GEN9_PG_ENABLE, 0);
4832 else
4833 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4834 (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
4835
4836 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4837
4838}
4839
4840static void gen8_enable_rps(struct drm_device *dev)
4841{
4842 struct drm_i915_private *dev_priv = dev->dev_private;
4843 struct intel_engine_cs *ring;
4844 uint32_t rc6_mask = 0;
4845 int unused;
4846
4847
4848 I915_WRITE(GEN6_RC_STATE, 0);
4849
4850
4851
4852 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4853
4854
4855 I915_WRITE(GEN6_RC_CONTROL, 0);
4856
4857
4858 gen6_init_rps_frequencies(dev);
4859
4860
4861 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4862 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4863 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4864 for_each_ring(ring, dev_priv, unused)
4865 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4866 I915_WRITE(GEN6_RC_SLEEP, 0);
4867 if (IS_BROADWELL(dev))
4868 I915_WRITE(GEN6_RC6_THRESHOLD, 625);
4869 else
4870 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
4871
4872
4873 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4874 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4875 intel_print_rc6_info(dev, rc6_mask);
4876 if (IS_BROADWELL(dev))
4877 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4878 GEN7_RC_CTL_TO_MODE |
4879 rc6_mask);
4880 else
4881 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4882 GEN6_RC_CTL_EI_MODE(1) |
4883 rc6_mask);
4884
4885
4886 I915_WRITE(GEN6_RPNSWREQ,
4887 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4888 I915_WRITE(GEN6_RC_VIDEO_FREQ,
4889 HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4890
4891 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128);
4892
4893
4894 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
4895 dev_priv->rps.max_freq_softlimit << 24 |
4896 dev_priv->rps.min_freq_softlimit << 16);
4897
4898 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128);
4899 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128);
4900 I915_WRITE(GEN6_RP_UP_EI, 66000);
4901 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4902
4903 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4904
4905
4906 I915_WRITE(GEN6_RP_CONTROL,
4907 GEN6_RP_MEDIA_TURBO |
4908 GEN6_RP_MEDIA_HW_NORMAL_MODE |
4909 GEN6_RP_MEDIA_IS_GFX |
4910 GEN6_RP_ENABLE |
4911 GEN6_RP_UP_BUSY_AVG |
4912 GEN6_RP_DOWN_IDLE_AVG);
4913
4914
4915
4916 dev_priv->rps.power = HIGH_POWER;
4917 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4918
4919 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4920}
4921
4922static void gen6_enable_rps(struct drm_device *dev)
4923{
4924 struct drm_i915_private *dev_priv = dev->dev_private;
4925 struct intel_engine_cs *ring;
4926 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
4927 u32 gtfifodbg;
4928 int rc6_mode;
4929 int i, ret;
4930
4931 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4932
4933
4934
4935
4936
4937
4938
4939 I915_WRITE(GEN6_RC_STATE, 0);
4940
4941
4942 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4943 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
4944 I915_WRITE(GTFIFODBG, gtfifodbg);
4945 }
4946
4947 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4948
4949
4950 gen6_init_rps_frequencies(dev);
4951
4952
4953 I915_WRITE(GEN6_RC_CONTROL, 0);
4954
4955 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
4956 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
4957 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
4958 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4959 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4960
4961 for_each_ring(ring, dev_priv, i)
4962 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4963
4964 I915_WRITE(GEN6_RC_SLEEP, 0);
4965 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4966 if (IS_IVYBRIDGE(dev))
4967 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
4968 else
4969 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
4970 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
4971 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000);
4972
4973
4974 rc6_mode = intel_enable_rc6(dev_priv->dev);
4975 if (rc6_mode & INTEL_RC6_ENABLE)
4976 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
4977
4978
4979 if (!IS_HASWELL(dev)) {
4980 if (rc6_mode & INTEL_RC6p_ENABLE)
4981 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
4982
4983 if (rc6_mode & INTEL_RC6pp_ENABLE)
4984 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
4985 }
4986
4987 intel_print_rc6_info(dev, rc6_mask);
4988
4989 I915_WRITE(GEN6_RC_CONTROL,
4990 rc6_mask |
4991 GEN6_RC_CTL_EI_MODE(1) |
4992 GEN6_RC_CTL_HW_ENABLE);
4993
4994
4995 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
4996 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4997
4998 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
4999 if (ret)
5000 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
5001
5002 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
5003 if (!ret && (pcu_mbox & (1<<31))) {
5004 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5005 (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
5006 (pcu_mbox & 0xff) * 50);
5007 dev_priv->rps.max_freq = pcu_mbox & 0xff;
5008 }
5009
5010 dev_priv->rps.power = HIGH_POWER;
5011 gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
5012
5013 rc6vids = 0;
5014 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
5015 if (IS_GEN6(dev) && ret) {
5016 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
5017 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
5018 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
5019 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
5020 rc6vids &= 0xffff00;
5021 rc6vids |= GEN6_ENCODE_RC6_VID(450);
5022 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
5023 if (ret)
5024 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
5025 }
5026
5027 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5028}
5029
5030static void __gen6_update_ring_freq(struct drm_device *dev)
5031{
5032 struct drm_i915_private *dev_priv = dev->dev_private;
5033 int min_freq = 15;
5034 unsigned int gpu_freq;
5035 unsigned int max_ia_freq, min_ring_freq;
5036 unsigned int max_gpu_freq, min_gpu_freq;
5037 int scaling_factor = 180;
5038 struct cpufreq_policy *policy;
5039
5040 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5041
5042 policy = cpufreq_cpu_get(0);
5043 if (policy) {
5044 max_ia_freq = policy->cpuinfo.max_freq;
5045 cpufreq_cpu_put(policy);
5046 } else {
5047
5048
5049
5050
5051 max_ia_freq = tsc_khz;
5052 }
5053
5054
5055 max_ia_freq /= 1000;
5056
5057 min_ring_freq = I915_READ(DCLK) & 0xf;
5058
5059 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
5060
5061 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
5062
5063 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
5064 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
5065 } else {
5066 min_gpu_freq = dev_priv->rps.min_freq;
5067 max_gpu_freq = dev_priv->rps.max_freq;
5068 }
5069
5070
5071
5072
5073
5074
5075 for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
5076 int diff = max_gpu_freq - gpu_freq;
5077 unsigned int ia_freq = 0, ring_freq = 0;
5078
5079 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
5080
5081
5082
5083
5084 ring_freq = gpu_freq;
5085 } else if (INTEL_INFO(dev)->gen >= 8) {
5086
5087 ring_freq = max(min_ring_freq, gpu_freq);
5088 } else if (IS_HASWELL(dev)) {
5089 ring_freq = mult_frac(gpu_freq, 5, 4);
5090 ring_freq = max(min_ring_freq, ring_freq);
5091
5092 } else {
5093
5094
5095
5096
5097
5098
5099
5100 if (gpu_freq < min_freq)
5101 ia_freq = 800;
5102 else
5103 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
5104 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
5105 }
5106
5107 sandybridge_pcode_write(dev_priv,
5108 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
5109 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
5110 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
5111 gpu_freq);
5112 }
5113}
5114
5115void gen6_update_ring_freq(struct drm_device *dev)
5116{
5117 struct drm_i915_private *dev_priv = dev->dev_private;
5118
5119 if (!HAS_CORE_RING_FREQ(dev))
5120 return;
5121
5122 mutex_lock(&dev_priv->rps.hw_lock);
5123 __gen6_update_ring_freq(dev);
5124 mutex_unlock(&dev_priv->rps.hw_lock);
5125}
5126
5127static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
5128{
5129 struct drm_device *dev = dev_priv->dev;
5130 u32 val, rp0;
5131
5132 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5133
5134 switch (INTEL_INFO(dev)->eu_total) {
5135 case 8:
5136
5137 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
5138 break;
5139 case 12:
5140
5141 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
5142 break;
5143 case 16:
5144
5145 default:
5146
5147 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
5148 break;
5149 }
5150
5151 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
5152
5153 return rp0;
5154}
5155
5156static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
5157{
5158 u32 val, rpe;
5159
5160 val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
5161 rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
5162
5163 return rpe;
5164}
5165
5166static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
5167{
5168 u32 val, rp1;
5169
5170 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5171 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5172
5173 return rp1;
5174}
5175
5176static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
5177{
5178 u32 val, rp1;
5179
5180 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
5181
5182 rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
5183
5184 return rp1;
5185}
5186
5187static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
5188{
5189 u32 val, rp0;
5190
5191 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
5192
5193 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
5194
5195 rp0 = min_t(u32, rp0, 0xea);
5196
5197 return rp0;
5198}
5199
5200static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
5201{
5202 u32 val, rpe;
5203
5204 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
5205 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
5206 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
5207 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
5208
5209 return rpe;
5210}
5211
5212static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
5213{
5214 u32 val;
5215
5216 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
5217
5218
5219
5220
5221
5222
5223
5224 return max_t(u32, val, 0xc0);
5225}
5226
5227
5228static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
5229{
5230 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
5231
5232 WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
5233 dev_priv->vlv_pctx->stolen->start);
5234}
5235
5236
5237
5238static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
5239{
5240 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
5241
5242 WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
5243}
5244
5245static void cherryview_setup_pctx(struct drm_device *dev)
5246{
5247 struct drm_i915_private *dev_priv = dev->dev_private;
5248 unsigned long pctx_paddr, paddr;
5249 struct i915_gtt *gtt = &dev_priv->gtt;
5250 u32 pcbr;
5251 int pctx_size = 32*1024;
5252
5253 pcbr = I915_READ(VLV_PCBR);
5254 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5255 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5256 paddr = (dev_priv->mm.stolen_base +
5257 (gtt->stolen_size - pctx_size));
5258
5259 pctx_paddr = (paddr & (~4095));
5260 I915_WRITE(VLV_PCBR, pctx_paddr);
5261 }
5262
5263 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5264}
5265
5266static void valleyview_setup_pctx(struct drm_device *dev)
5267{
5268 struct drm_i915_private *dev_priv = dev->dev_private;
5269 struct drm_i915_gem_object *pctx;
5270 unsigned long pctx_paddr;
5271 u32 pcbr;
5272 int pctx_size = 24*1024;
5273
5274 mutex_lock(&dev->struct_mutex);
5275
5276 pcbr = I915_READ(VLV_PCBR);
5277 if (pcbr) {
5278
5279 int pcbr_offset;
5280
5281 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
5282 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
5283 pcbr_offset,
5284 I915_GTT_OFFSET_NONE,
5285 pctx_size);
5286 goto out;
5287 }
5288
5289 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299 pctx = i915_gem_object_create_stolen(dev, pctx_size);
5300 if (!pctx) {
5301 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
5302 goto out;
5303 }
5304
5305 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
5306 I915_WRITE(VLV_PCBR, pctx_paddr);
5307
5308out:
5309 DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5310 dev_priv->vlv_pctx = pctx;
5311 mutex_unlock(&dev->struct_mutex);
5312}
5313
5314static void valleyview_cleanup_pctx(struct drm_device *dev)
5315{
5316 struct drm_i915_private *dev_priv = dev->dev_private;
5317
5318 if (WARN_ON(!dev_priv->vlv_pctx))
5319 return;
5320
5321 drm_gem_object_unreference_unlocked(&dev_priv->vlv_pctx->base);
5322 dev_priv->vlv_pctx = NULL;
5323}
5324
5325static void valleyview_init_gt_powersave(struct drm_device *dev)
5326{
5327 struct drm_i915_private *dev_priv = dev->dev_private;
5328 u32 val;
5329
5330 valleyview_setup_pctx(dev);
5331
5332 mutex_lock(&dev_priv->rps.hw_lock);
5333
5334 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5335 switch ((val >> 6) & 3) {
5336 case 0:
5337 case 1:
5338 dev_priv->mem_freq = 800;
5339 break;
5340 case 2:
5341 dev_priv->mem_freq = 1066;
5342 break;
5343 case 3:
5344 dev_priv->mem_freq = 1333;
5345 break;
5346 }
5347 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
5348
5349 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
5350 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
5351 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
5352 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5353 dev_priv->rps.max_freq);
5354
5355 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
5356 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
5357 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5358 dev_priv->rps.efficient_freq);
5359
5360 dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
5361 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
5362 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5363 dev_priv->rps.rp1_freq);
5364
5365 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
5366 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
5367 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5368 dev_priv->rps.min_freq);
5369
5370 dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
5371
5372
5373 if (dev_priv->rps.max_freq_softlimit == 0)
5374 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
5375
5376 if (dev_priv->rps.min_freq_softlimit == 0)
5377 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
5378
5379 mutex_unlock(&dev_priv->rps.hw_lock);
5380}
5381
5382static void cherryview_init_gt_powersave(struct drm_device *dev)
5383{
5384 struct drm_i915_private *dev_priv = dev->dev_private;
5385 u32 val;
5386
5387 cherryview_setup_pctx(dev);
5388
5389 mutex_lock(&dev_priv->rps.hw_lock);
5390
5391 mutex_lock(&dev_priv->sb_lock);
5392 val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
5393 mutex_unlock(&dev_priv->sb_lock);
5394
5395 switch ((val >> 2) & 0x7) {
5396 case 3:
5397 dev_priv->mem_freq = 2000;
5398 break;
5399 default:
5400 dev_priv->mem_freq = 1600;
5401 break;
5402 }
5403 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
5404
5405 dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
5406 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
5407 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
5408 intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5409 dev_priv->rps.max_freq);
5410
5411 dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
5412 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
5413 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5414 dev_priv->rps.efficient_freq);
5415
5416 dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
5417 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
5418 intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5419 dev_priv->rps.rp1_freq);
5420
5421
5422 dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5423 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
5424 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5425 dev_priv->rps.min_freq);
5426
5427 WARN_ONCE((dev_priv->rps.max_freq |
5428 dev_priv->rps.efficient_freq |
5429 dev_priv->rps.rp1_freq |
5430 dev_priv->rps.min_freq) & 1,
5431 "Odd GPU freq values\n");
5432
5433 dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
5434
5435
5436 if (dev_priv->rps.max_freq_softlimit == 0)
5437 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
5438
5439 if (dev_priv->rps.min_freq_softlimit == 0)
5440 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
5441
5442 mutex_unlock(&dev_priv->rps.hw_lock);
5443}
5444
5445static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
5446{
5447 valleyview_cleanup_pctx(dev);
5448}
5449
5450static void cherryview_enable_rps(struct drm_device *dev)
5451{
5452 struct drm_i915_private *dev_priv = dev->dev_private;
5453 struct intel_engine_cs *ring;
5454 u32 gtfifodbg, val, rc6_mode = 0, pcbr;
5455 int i;
5456
5457 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5458
5459 gtfifodbg = I915_READ(GTFIFODBG);
5460 if (gtfifodbg) {
5461 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
5462 gtfifodbg);
5463 I915_WRITE(GTFIFODBG, gtfifodbg);
5464 }
5465
5466 cherryview_check_pctx(dev_priv);
5467
5468
5469
5470 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5471
5472
5473 I915_WRITE(GEN6_RC_CONTROL, 0);
5474
5475
5476 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
5477 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5478 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5479
5480 for_each_ring(ring, dev_priv, i)
5481 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
5482 I915_WRITE(GEN6_RC_SLEEP, 0);
5483
5484
5485 I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5486
5487
5488 I915_WRITE(VLV_COUNTER_CONTROL,
5489 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
5490 VLV_MEDIA_RC6_COUNT_EN |
5491 VLV_RENDER_RC6_COUNT_EN));
5492
5493
5494 pcbr = I915_READ(VLV_PCBR);
5495
5496
5497 if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
5498 (pcbr >> VLV_PCBR_ADDR_SHIFT))
5499 rc6_mode = GEN7_RC_CTL_TO_MODE;
5500
5501 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
5502
5503
5504 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5505 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
5506 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
5507 I915_WRITE(GEN6_RP_UP_EI, 66000);
5508 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5509
5510 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5511
5512
5513 I915_WRITE(GEN6_RP_CONTROL,
5514 GEN6_RP_MEDIA_HW_NORMAL_MODE |
5515 GEN6_RP_MEDIA_IS_GFX |
5516 GEN6_RP_ENABLE |
5517 GEN6_RP_UP_BUSY_AVG |
5518 GEN6_RP_DOWN_IDLE_AVG);
5519
5520
5521 val = VLV_OVERRIDE_EN |
5522 VLV_SOC_TDP_EN |
5523 CHV_BIAS_CPU_50_SOC_50;
5524 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
5525
5526 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5527
5528
5529 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5530
5531 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5532 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5533
5534 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
5535 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5536 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5537 dev_priv->rps.cur_freq);
5538
5539 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5540 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5541 dev_priv->rps.efficient_freq);
5542
5543 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
5544
5545 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5546}
5547
5548static void valleyview_enable_rps(struct drm_device *dev)
5549{
5550 struct drm_i915_private *dev_priv = dev->dev_private;
5551 struct intel_engine_cs *ring;
5552 u32 gtfifodbg, val, rc6_mode = 0;
5553 int i;
5554
5555 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5556
5557 valleyview_check_pctx(dev_priv);
5558
5559 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
5560 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
5561 gtfifodbg);
5562 I915_WRITE(GTFIFODBG, gtfifodbg);
5563 }
5564
5565
5566 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5567
5568
5569 I915_WRITE(GEN6_RC_CONTROL, 0);
5570
5571 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5572 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
5573 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
5574 I915_WRITE(GEN6_RP_UP_EI, 66000);
5575 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5576
5577 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5578
5579 I915_WRITE(GEN6_RP_CONTROL,
5580 GEN6_RP_MEDIA_TURBO |
5581 GEN6_RP_MEDIA_HW_NORMAL_MODE |
5582 GEN6_RP_MEDIA_IS_GFX |
5583 GEN6_RP_ENABLE |
5584 GEN6_RP_UP_BUSY_AVG |
5585 GEN6_RP_DOWN_IDLE_CONT);
5586
5587 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
5588 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5589 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5590
5591 for_each_ring(ring, dev_priv, i)
5592 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
5593
5594 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
5595
5596
5597 I915_WRITE(VLV_COUNTER_CONTROL,
5598 _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN |
5599 VLV_RENDER_RC0_COUNT_EN |
5600 VLV_MEDIA_RC6_COUNT_EN |
5601 VLV_RENDER_RC6_COUNT_EN));
5602
5603 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
5604 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
5605
5606 intel_print_rc6_info(dev, rc6_mode);
5607
5608 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
5609
5610
5611 val = VLV_OVERRIDE_EN |
5612 VLV_SOC_TDP_EN |
5613 VLV_BIAS_CPU_125_SOC_875;
5614 vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
5615
5616 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5617
5618
5619 WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5620
5621 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5622 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5623
5624 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
5625 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5626 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5627 dev_priv->rps.cur_freq);
5628
5629 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5630 intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5631 dev_priv->rps.efficient_freq);
5632
5633 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
5634
5635 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5636}
5637
5638static unsigned long intel_pxfreq(u32 vidfreq)
5639{
5640 unsigned long freq;
5641 int div = (vidfreq & 0x3f0000) >> 16;
5642 int post = (vidfreq & 0x3000) >> 12;
5643 int pre = (vidfreq & 0x7);
5644
5645 if (!pre)
5646 return 0;
5647
5648 freq = ((div * 133333) / ((1<<post) * pre));
5649
5650 return freq;
5651}
5652
5653static const struct cparams {
5654 u16 i;
5655 u16 t;
5656 u16 m;
5657 u16 c;
5658} cparams[] = {
5659 { 1, 1333, 301, 28664 },
5660 { 1, 1066, 294, 24460 },
5661 { 1, 800, 294, 25192 },
5662 { 0, 1333, 276, 27605 },
5663 { 0, 1066, 276, 27605 },
5664 { 0, 800, 231, 23784 },
5665};
5666
5667static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
5668{
5669 u64 total_count, diff, ret;
5670 u32 count1, count2, count3, m = 0, c = 0;
5671 unsigned long now = jiffies_to_msecs(jiffies), diff1;
5672 int i;
5673
5674 assert_spin_locked(&mchdev_lock);
5675
5676 diff1 = now - dev_priv->ips.last_time1;
5677
5678
5679
5680
5681
5682
5683 if (diff1 <= 10)
5684 return dev_priv->ips.chipset_power;
5685
5686 count1 = I915_READ(DMIEC);
5687 count2 = I915_READ(DDREC);
5688 count3 = I915_READ(CSIEC);
5689
5690 total_count = count1 + count2 + count3;
5691
5692
5693 if (total_count < dev_priv->ips.last_count1) {
5694 diff = ~0UL - dev_priv->ips.last_count1;
5695 diff += total_count;
5696 } else {
5697 diff = total_count - dev_priv->ips.last_count1;
5698 }
5699
5700 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
5701 if (cparams[i].i == dev_priv->ips.c_m &&
5702 cparams[i].t == dev_priv->ips.r_t) {
5703 m = cparams[i].m;
5704 c = cparams[i].c;
5705 break;
5706 }
5707 }
5708
5709 diff = div_u64(diff, diff1);
5710 ret = ((m * diff) + c);
5711 ret = div_u64(ret, 10);
5712
5713 dev_priv->ips.last_count1 = total_count;
5714 dev_priv->ips.last_time1 = now;
5715
5716 dev_priv->ips.chipset_power = ret;
5717
5718 return ret;
5719}
5720
5721unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
5722{
5723 struct drm_device *dev = dev_priv->dev;
5724 unsigned long val;
5725
5726 if (INTEL_INFO(dev)->gen != 5)
5727 return 0;
5728
5729 spin_lock_irq(&mchdev_lock);
5730
5731 val = __i915_chipset_val(dev_priv);
5732
5733 spin_unlock_irq(&mchdev_lock);
5734
5735 return val;
5736}
5737
5738unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
5739{
5740 unsigned long m, x, b;
5741 u32 tsfs;
5742
5743 tsfs = I915_READ(TSFS);
5744
5745 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
5746 x = I915_READ8(TR1);
5747
5748 b = tsfs & TSFS_INTR_MASK;
5749
5750 return ((m * x) / 127) - b;
5751}
5752
5753static int _pxvid_to_vd(u8 pxvid)
5754{
5755 if (pxvid == 0)
5756 return 0;
5757
5758 if (pxvid >= 8 && pxvid < 31)
5759 pxvid = 31;
5760
5761 return (pxvid + 2) * 125;
5762}
5763
5764static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
5765{
5766 struct drm_device *dev = dev_priv->dev;
5767 const int vd = _pxvid_to_vd(pxvid);
5768 const int vm = vd - 1125;
5769
5770 if (INTEL_INFO(dev)->is_mobile)
5771 return vm > 0 ? vm : 0;
5772
5773 return vd;
5774}
5775
5776static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
5777{
5778 u64 now, diff, diffms;
5779 u32 count;
5780
5781 assert_spin_locked(&mchdev_lock);
5782
5783 now = ktime_get_raw_ns();
5784 diffms = now - dev_priv->ips.last_time2;
5785 do_div(diffms, NSEC_PER_MSEC);
5786
5787
5788 if (!diffms)
5789 return;
5790
5791 count = I915_READ(GFXEC);
5792
5793 if (count < dev_priv->ips.last_count2) {
5794 diff = ~0UL - dev_priv->ips.last_count2;
5795 diff += count;
5796 } else {
5797 diff = count - dev_priv->ips.last_count2;
5798 }
5799
5800 dev_priv->ips.last_count2 = count;
5801 dev_priv->ips.last_time2 = now;
5802
5803
5804 diff = diff * 1181;
5805 diff = div_u64(diff, diffms * 10);
5806 dev_priv->ips.gfx_power = diff;
5807}
5808
5809void i915_update_gfx_val(struct drm_i915_private *dev_priv)
5810{
5811 struct drm_device *dev = dev_priv->dev;
5812
5813 if (INTEL_INFO(dev)->gen != 5)
5814 return;
5815
5816 spin_lock_irq(&mchdev_lock);
5817
5818 __i915_update_gfx_val(dev_priv);
5819
5820 spin_unlock_irq(&mchdev_lock);
5821}
5822
5823static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
5824{
5825 unsigned long t, corr, state1, corr2, state2;
5826 u32 pxvid, ext_v;
5827
5828 assert_spin_locked(&mchdev_lock);
5829
5830 pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
5831 pxvid = (pxvid >> 24) & 0x7f;
5832 ext_v = pvid_to_extvid(dev_priv, pxvid);
5833
5834 state1 = ext_v;
5835
5836 t = i915_mch_val(dev_priv);
5837
5838
5839
5840
5841 if (t > 80)
5842 corr = ((t * 2349) + 135940);
5843 else if (t >= 50)
5844 corr = ((t * 964) + 29317);
5845 else
5846 corr = ((t * 301) + 1004);
5847
5848 corr = corr * ((150142 * state1) / 10000 - 78642);
5849 corr /= 100000;
5850 corr2 = (corr * dev_priv->ips.corr);
5851
5852 state2 = (corr2 * state1) / 10000;
5853 state2 /= 100;
5854
5855 __i915_update_gfx_val(dev_priv);
5856
5857 return dev_priv->ips.gfx_power + state2;
5858}
5859
5860unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
5861{
5862 struct drm_device *dev = dev_priv->dev;
5863 unsigned long val;
5864
5865 if (INTEL_INFO(dev)->gen != 5)
5866 return 0;
5867
5868 spin_lock_irq(&mchdev_lock);
5869
5870 val = __i915_gfx_val(dev_priv);
5871
5872 spin_unlock_irq(&mchdev_lock);
5873
5874 return val;
5875}
5876
5877
5878
5879
5880
5881
5882
5883unsigned long i915_read_mch_val(void)
5884{
5885 struct drm_i915_private *dev_priv;
5886 unsigned long chipset_val, graphics_val, ret = 0;
5887
5888 spin_lock_irq(&mchdev_lock);
5889 if (!i915_mch_dev)
5890 goto out_unlock;
5891 dev_priv = i915_mch_dev;
5892
5893 chipset_val = __i915_chipset_val(dev_priv);
5894 graphics_val = __i915_gfx_val(dev_priv);
5895
5896 ret = chipset_val + graphics_val;
5897
5898out_unlock:
5899 spin_unlock_irq(&mchdev_lock);
5900
5901 return ret;
5902}
5903EXPORT_SYMBOL_GPL(i915_read_mch_val);
5904
5905
5906
5907
5908
5909
5910bool i915_gpu_raise(void)
5911{
5912 struct drm_i915_private *dev_priv;
5913 bool ret = true;
5914
5915 spin_lock_irq(&mchdev_lock);
5916 if (!i915_mch_dev) {
5917 ret = false;
5918 goto out_unlock;
5919 }
5920 dev_priv = i915_mch_dev;
5921
5922 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
5923 dev_priv->ips.max_delay--;
5924
5925out_unlock:
5926 spin_unlock_irq(&mchdev_lock);
5927
5928 return ret;
5929}
5930EXPORT_SYMBOL_GPL(i915_gpu_raise);
5931
5932
5933
5934
5935
5936
5937
5938bool i915_gpu_lower(void)
5939{
5940 struct drm_i915_private *dev_priv;
5941 bool ret = true;
5942
5943 spin_lock_irq(&mchdev_lock);
5944 if (!i915_mch_dev) {
5945 ret = false;
5946 goto out_unlock;
5947 }
5948 dev_priv = i915_mch_dev;
5949
5950 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
5951 dev_priv->ips.max_delay++;
5952
5953out_unlock:
5954 spin_unlock_irq(&mchdev_lock);
5955
5956 return ret;
5957}
5958EXPORT_SYMBOL_GPL(i915_gpu_lower);
5959
5960
5961
5962
5963
5964
5965bool i915_gpu_busy(void)
5966{
5967 struct drm_i915_private *dev_priv;
5968 struct intel_engine_cs *ring;
5969 bool ret = false;
5970 int i;
5971
5972 spin_lock_irq(&mchdev_lock);
5973 if (!i915_mch_dev)
5974 goto out_unlock;
5975 dev_priv = i915_mch_dev;
5976
5977 for_each_ring(ring, dev_priv, i)
5978 ret |= !list_empty(&ring->request_list);
5979
5980out_unlock:
5981 spin_unlock_irq(&mchdev_lock);
5982
5983 return ret;
5984}
5985EXPORT_SYMBOL_GPL(i915_gpu_busy);
5986
5987
5988
5989
5990
5991
5992
5993bool i915_gpu_turbo_disable(void)
5994{
5995 struct drm_i915_private *dev_priv;
5996 bool ret = true;
5997
5998 spin_lock_irq(&mchdev_lock);
5999 if (!i915_mch_dev) {
6000 ret = false;
6001 goto out_unlock;
6002 }
6003 dev_priv = i915_mch_dev;
6004
6005 dev_priv->ips.max_delay = dev_priv->ips.fstart;
6006
6007 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
6008 ret = false;
6009
6010out_unlock:
6011 spin_unlock_irq(&mchdev_lock);
6012
6013 return ret;
6014}
6015EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025static void
6026ips_ping_for_i915_load(void)
6027{
6028 void (*link)(void);
6029
6030 link = symbol_get(ips_link_to_i915_driver);
6031 if (link) {
6032 link();
6033 symbol_put(ips_link_to_i915_driver);
6034 }
6035}
6036
6037void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
6038{
6039
6040
6041 spin_lock_irq(&mchdev_lock);
6042 i915_mch_dev = dev_priv;
6043 spin_unlock_irq(&mchdev_lock);
6044
6045 ips_ping_for_i915_load();
6046}
6047
6048void intel_gpu_ips_teardown(void)
6049{
6050 spin_lock_irq(&mchdev_lock);
6051 i915_mch_dev = NULL;
6052 spin_unlock_irq(&mchdev_lock);
6053}
6054
6055static void intel_init_emon(struct drm_device *dev)
6056{
6057 struct drm_i915_private *dev_priv = dev->dev_private;
6058 u32 lcfuse;
6059 u8 pxw[16];
6060 int i;
6061
6062
6063 I915_WRITE(ECR, 0);
6064 POSTING_READ(ECR);
6065
6066
6067 I915_WRITE(SDEW, 0x15040d00);
6068 I915_WRITE(CSIEW0, 0x007f0000);
6069 I915_WRITE(CSIEW1, 0x1e220004);
6070 I915_WRITE(CSIEW2, 0x04000004);
6071
6072 for (i = 0; i < 5; i++)
6073 I915_WRITE(PEW(i), 0);
6074 for (i = 0; i < 3; i++)
6075 I915_WRITE(DEW(i), 0);
6076
6077
6078 for (i = 0; i < 16; i++) {
6079 u32 pxvidfreq = I915_READ(PXVFREQ(i));
6080 unsigned long freq = intel_pxfreq(pxvidfreq);
6081 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
6082 PXVFREQ_PX_SHIFT;
6083 unsigned long val;
6084
6085 val = vid * vid;
6086 val *= (freq / 1000);
6087 val *= 255;
6088 val /= (127*127*900);
6089 if (val > 0xff)
6090 DRM_ERROR("bad pxval: %ld\n", val);
6091 pxw[i] = val;
6092 }
6093
6094 pxw[14] = 0;
6095 pxw[15] = 0;
6096
6097 for (i = 0; i < 4; i++) {
6098 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
6099 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
6100 I915_WRITE(PXW(i), val);
6101 }
6102
6103
6104 I915_WRITE(OGW0, 0);
6105 I915_WRITE(OGW1, 0);
6106 I915_WRITE(EG0, 0x00007f00);
6107 I915_WRITE(EG1, 0x0000000e);
6108 I915_WRITE(EG2, 0x000e0000);
6109 I915_WRITE(EG3, 0x68000300);
6110 I915_WRITE(EG4, 0x42000000);
6111 I915_WRITE(EG5, 0x00140031);
6112 I915_WRITE(EG6, 0);
6113 I915_WRITE(EG7, 0);
6114
6115 for (i = 0; i < 8; i++)
6116 I915_WRITE(PXWL(i), 0);
6117
6118
6119 I915_WRITE(ECR, 0x80000019);
6120
6121 lcfuse = I915_READ(LCFUSE02);
6122
6123 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
6124}
6125
6126void intel_init_gt_powersave(struct drm_device *dev)
6127{
6128 struct drm_i915_private *dev_priv = dev->dev_private;
6129
6130
6131
6132
6133
6134 if (!i915.enable_rc6) {
6135 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
6136 intel_runtime_pm_get(dev_priv);
6137 }
6138
6139 if (IS_CHERRYVIEW(dev))
6140 cherryview_init_gt_powersave(dev);
6141 else if (IS_VALLEYVIEW(dev))
6142 valleyview_init_gt_powersave(dev);
6143}
6144
6145void intel_cleanup_gt_powersave(struct drm_device *dev)
6146{
6147 struct drm_i915_private *dev_priv = dev->dev_private;
6148
6149 if (IS_CHERRYVIEW(dev))
6150 return;
6151 else if (IS_VALLEYVIEW(dev))
6152 valleyview_cleanup_gt_powersave(dev);
6153
6154 if (!i915.enable_rc6)
6155 intel_runtime_pm_put(dev_priv);
6156}
6157
6158static void gen6_suspend_rps(struct drm_device *dev)
6159{
6160 struct drm_i915_private *dev_priv = dev->dev_private;
6161
6162 flush_delayed_work(&dev_priv->rps.delayed_resume_work);
6163
6164 gen6_disable_rps_interrupts(dev);
6165}
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175void intel_suspend_gt_powersave(struct drm_device *dev)
6176{
6177 struct drm_i915_private *dev_priv = dev->dev_private;
6178
6179 if (INTEL_INFO(dev)->gen < 6)
6180 return;
6181
6182 gen6_suspend_rps(dev);
6183
6184
6185 gen6_rps_idle(dev_priv);
6186}
6187
6188void intel_disable_gt_powersave(struct drm_device *dev)
6189{
6190 struct drm_i915_private *dev_priv = dev->dev_private;
6191
6192 if (IS_IRONLAKE_M(dev)) {
6193 ironlake_disable_drps(dev);
6194 } else if (INTEL_INFO(dev)->gen >= 6) {
6195 intel_suspend_gt_powersave(dev);
6196
6197 mutex_lock(&dev_priv->rps.hw_lock);
6198 if (INTEL_INFO(dev)->gen >= 9)
6199 gen9_disable_rps(dev);
6200 else if (IS_CHERRYVIEW(dev))
6201 cherryview_disable_rps(dev);
6202 else if (IS_VALLEYVIEW(dev))
6203 valleyview_disable_rps(dev);
6204 else
6205 gen6_disable_rps(dev);
6206
6207 dev_priv->rps.enabled = false;
6208 mutex_unlock(&dev_priv->rps.hw_lock);
6209 }
6210}
6211
6212static void intel_gen6_powersave_work(struct work_struct *work)
6213{
6214 struct drm_i915_private *dev_priv =
6215 container_of(work, struct drm_i915_private,
6216 rps.delayed_resume_work.work);
6217 struct drm_device *dev = dev_priv->dev;
6218
6219 mutex_lock(&dev_priv->rps.hw_lock);
6220
6221 gen6_reset_rps_interrupts(dev);
6222
6223 if (IS_CHERRYVIEW(dev)) {
6224 cherryview_enable_rps(dev);
6225 } else if (IS_VALLEYVIEW(dev)) {
6226 valleyview_enable_rps(dev);
6227 } else if (INTEL_INFO(dev)->gen >= 9) {
6228 gen9_enable_rc6(dev);
6229 gen9_enable_rps(dev);
6230 if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
6231 __gen6_update_ring_freq(dev);
6232 } else if (IS_BROADWELL(dev)) {
6233 gen8_enable_rps(dev);
6234 __gen6_update_ring_freq(dev);
6235 } else {
6236 gen6_enable_rps(dev);
6237 __gen6_update_ring_freq(dev);
6238 }
6239
6240 WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
6241 WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
6242
6243 WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
6244 WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
6245
6246 dev_priv->rps.enabled = true;
6247
6248 gen6_enable_rps_interrupts(dev);
6249
6250 mutex_unlock(&dev_priv->rps.hw_lock);
6251
6252 intel_runtime_pm_put(dev_priv);
6253}
6254
6255void intel_enable_gt_powersave(struct drm_device *dev)
6256{
6257 struct drm_i915_private *dev_priv = dev->dev_private;
6258
6259
6260 if (intel_vgpu_active(dev))
6261 return;
6262
6263 if (IS_IRONLAKE_M(dev)) {
6264 ironlake_enable_drps(dev);
6265 mutex_lock(&dev->struct_mutex);
6266 intel_init_emon(dev);
6267 mutex_unlock(&dev->struct_mutex);
6268 } else if (INTEL_INFO(dev)->gen >= 6) {
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
6282 round_jiffies_up_relative(HZ)))
6283 intel_runtime_pm_get_noresume(dev_priv);
6284 }
6285}
6286
6287void intel_reset_gt_powersave(struct drm_device *dev)
6288{
6289 struct drm_i915_private *dev_priv = dev->dev_private;
6290
6291 if (INTEL_INFO(dev)->gen < 6)
6292 return;
6293
6294 gen6_suspend_rps(dev);
6295 dev_priv->rps.enabled = false;
6296}
6297
6298static void ibx_init_clock_gating(struct drm_device *dev)
6299{
6300 struct drm_i915_private *dev_priv = dev->dev_private;
6301
6302
6303
6304
6305
6306
6307 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6308}
6309
6310static void g4x_disable_trickle_feed(struct drm_device *dev)
6311{
6312 struct drm_i915_private *dev_priv = dev->dev_private;
6313 enum pipe pipe;
6314
6315 for_each_pipe(dev_priv, pipe) {
6316 I915_WRITE(DSPCNTR(pipe),
6317 I915_READ(DSPCNTR(pipe)) |
6318 DISPPLANE_TRICKLE_FEED_DISABLE);
6319
6320 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6321 POSTING_READ(DSPSURF(pipe));
6322 }
6323}
6324
6325static void ilk_init_lp_watermarks(struct drm_device *dev)
6326{
6327 struct drm_i915_private *dev_priv = dev->dev_private;
6328
6329 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6330 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6331 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6332
6333
6334
6335
6336
6337}
6338
6339static void ironlake_init_clock_gating(struct drm_device *dev)
6340{
6341 struct drm_i915_private *dev_priv = dev->dev_private;
6342 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6343
6344
6345
6346
6347
6348 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6349 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6350 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6351
6352 I915_WRITE(PCH_3DCGDIS0,
6353 MARIUNIT_CLOCK_GATE_DISABLE |
6354 SVSMUNIT_CLOCK_GATE_DISABLE);
6355 I915_WRITE(PCH_3DCGDIS1,
6356 VFMUNIT_CLOCK_GATE_DISABLE);
6357
6358
6359
6360
6361
6362
6363
6364
6365 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6366 (I915_READ(ILK_DISPLAY_CHICKEN2) |
6367 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6368 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6369 I915_WRITE(DISP_ARB_CTL,
6370 (I915_READ(DISP_ARB_CTL) |
6371 DISP_FBC_WM_DIS));
6372
6373 ilk_init_lp_watermarks(dev);
6374
6375
6376
6377
6378
6379
6380
6381
6382 if (IS_IRONLAKE_M(dev)) {
6383
6384 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6385 I915_READ(ILK_DISPLAY_CHICKEN1) |
6386 ILK_FBCQ_DIS);
6387 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6388 I915_READ(ILK_DISPLAY_CHICKEN2) |
6389 ILK_DPARB_GATE);
6390 }
6391
6392 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6393
6394 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6395 I915_READ(ILK_DISPLAY_CHICKEN2) |
6396 ILK_ELPIN_409_SELECT);
6397 I915_WRITE(_3D_CHICKEN2,
6398 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
6399 _3D_CHICKEN2_WM_READ_PIPELINED);
6400
6401
6402 I915_WRITE(CACHE_MODE_0,
6403 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6404
6405
6406 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6407
6408 g4x_disable_trickle_feed(dev);
6409
6410 ibx_init_clock_gating(dev);
6411}
6412
6413static void cpt_init_clock_gating(struct drm_device *dev)
6414{
6415 struct drm_i915_private *dev_priv = dev->dev_private;
6416 int pipe;
6417 uint32_t val;
6418
6419
6420
6421
6422
6423
6424 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6425 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6426 PCH_CPUNIT_CLOCK_GATE_DISABLE);
6427 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6428 DPLS_EDP_PPS_FIX_DIS);
6429
6430
6431
6432 for_each_pipe(dev_priv, pipe) {
6433 val = I915_READ(TRANS_CHICKEN2(pipe));
6434 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6435 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6436 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6437 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6438 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
6439 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6440 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6441 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6442 }
6443
6444 for_each_pipe(dev_priv, pipe) {
6445 I915_WRITE(TRANS_CHICKEN1(pipe),
6446 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6447 }
6448}
6449
6450static void gen6_check_mch_setup(struct drm_device *dev)
6451{
6452 struct drm_i915_private *dev_priv = dev->dev_private;
6453 uint32_t tmp;
6454
6455 tmp = I915_READ(MCH_SSKPD);
6456 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6457 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6458 tmp);
6459}
6460
6461static void gen6_init_clock_gating(struct drm_device *dev)
6462{
6463 struct drm_i915_private *dev_priv = dev->dev_private;
6464 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6465
6466 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6467
6468 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6469 I915_READ(ILK_DISPLAY_CHICKEN2) |
6470 ILK_ELPIN_409_SELECT);
6471
6472
6473 I915_WRITE(_3D_CHICKEN,
6474 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
6475
6476
6477 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487 I915_WRITE(GEN6_GT_MODE,
6488 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6489
6490 ilk_init_lp_watermarks(dev);
6491
6492 I915_WRITE(CACHE_MODE_0,
6493 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6494
6495 I915_WRITE(GEN6_UCGCTL1,
6496 I915_READ(GEN6_UCGCTL1) |
6497 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6498 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513 I915_WRITE(GEN6_UCGCTL2,
6514 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
6515 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
6516
6517
6518 I915_WRITE(_3D_CHICKEN3,
6519 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6520
6521
6522
6523
6524
6525
6526 I915_WRITE(_3D_CHICKEN3,
6527 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6541 I915_READ(ILK_DISPLAY_CHICKEN1) |
6542 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
6543 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6544 I915_READ(ILK_DISPLAY_CHICKEN2) |
6545 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
6546 I915_WRITE(ILK_DSPCLK_GATE_D,
6547 I915_READ(ILK_DSPCLK_GATE_D) |
6548 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
6549 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6550
6551 g4x_disable_trickle_feed(dev);
6552
6553 cpt_init_clock_gating(dev);
6554
6555 gen6_check_mch_setup(dev);
6556}
6557
6558static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6559{
6560 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
6561
6562
6563
6564
6565
6566
6567
6568 reg &= ~GEN7_FF_SCHED_MASK;
6569 reg |= GEN7_FF_TS_SCHED_HW;
6570 reg |= GEN7_FF_VS_SCHED_HW;
6571 reg |= GEN7_FF_DS_SCHED_HW;
6572
6573 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
6574}
6575
6576static void lpt_init_clock_gating(struct drm_device *dev)
6577{
6578 struct drm_i915_private *dev_priv = dev->dev_private;
6579
6580
6581
6582
6583
6584 if (HAS_PCH_LPT_LP(dev))
6585 I915_WRITE(SOUTH_DSPCLK_GATE_D,
6586 I915_READ(SOUTH_DSPCLK_GATE_D) |
6587 PCH_LP_PARTITION_LEVEL_DISABLE);
6588
6589
6590 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
6591 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
6592 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6593}
6594
6595static void lpt_suspend_hw(struct drm_device *dev)
6596{
6597 struct drm_i915_private *dev_priv = dev->dev_private;
6598
6599 if (HAS_PCH_LPT_LP(dev)) {
6600 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
6601
6602 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
6603 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
6604 }
6605}
6606
6607static void broadwell_init_clock_gating(struct drm_device *dev)
6608{
6609 struct drm_i915_private *dev_priv = dev->dev_private;
6610 enum pipe pipe;
6611 uint32_t misccpctl;
6612
6613 ilk_init_lp_watermarks(dev);
6614
6615
6616 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6617
6618
6619 I915_WRITE(CHICKEN_PAR1_1,
6620 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
6621
6622
6623 for_each_pipe(dev_priv, pipe) {
6624 I915_WRITE(CHICKEN_PIPESL_1(pipe),
6625 I915_READ(CHICKEN_PIPESL_1(pipe)) |
6626 BDW_DPRS_MASK_VBLANK_SRD);
6627 }
6628
6629
6630
6631 I915_WRITE(GEN7_FF_THREAD_MODE,
6632 I915_READ(GEN7_FF_THREAD_MODE) &
6633 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6634
6635 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6636 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6637
6638
6639 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6640 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6641
6642
6643
6644
6645
6646 misccpctl = I915_READ(GEN7_MISCCPCTL);
6647 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
6648 I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6649
6650
6651
6652
6653 POSTING_READ(GEN8_L3SQCREG1);
6654 udelay(1);
6655 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
6656
6657
6658
6659
6660
6661
6662 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
6663
6664 lpt_init_clock_gating(dev);
6665}
6666
6667static void haswell_init_clock_gating(struct drm_device *dev)
6668{
6669 struct drm_i915_private *dev_priv = dev->dev_private;
6670
6671 ilk_init_lp_watermarks(dev);
6672
6673
6674 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
6675 I915_WRITE(HSW_ROW_CHICKEN3,
6676 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
6677
6678
6679 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6680 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6681 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6682
6683
6684 I915_WRITE(GEN7_FF_THREAD_MODE,
6685 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
6686
6687
6688 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6689
6690
6691 I915_WRITE(CACHE_MODE_0_GEN7,
6692 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6693
6694
6695 I915_WRITE(CACHE_MODE_1,
6696 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706 I915_WRITE(GEN7_GT_MODE,
6707 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6708
6709
6710 I915_WRITE(HALF_SLICE_CHICKEN3,
6711 _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
6712
6713
6714 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6715
6716
6717 I915_WRITE(CHICKEN_PAR1_1,
6718 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
6719
6720 lpt_init_clock_gating(dev);
6721}
6722
6723static void ivybridge_init_clock_gating(struct drm_device *dev)
6724{
6725 struct drm_i915_private *dev_priv = dev->dev_private;
6726 uint32_t snpcr;
6727
6728 ilk_init_lp_watermarks(dev);
6729
6730 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6731
6732
6733 I915_WRITE(_3D_CHICKEN3,
6734 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6735
6736
6737 I915_WRITE(IVB_CHICKEN3,
6738 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6739 CHICKEN3_DGMG_DONE_FIX_DISABLE);
6740
6741
6742 if (IS_IVB_GT1(dev))
6743 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6744 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6745
6746
6747 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6748
6749
6750 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
6751 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
6752
6753
6754 I915_WRITE(GEN7_L3CNTLREG1,
6755 GEN7_WA_FOR_GEN7_L3_CONTROL);
6756 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6757 GEN7_WA_L3_CHICKEN_MODE);
6758 if (IS_IVB_GT1(dev))
6759 I915_WRITE(GEN7_ROW_CHICKEN2,
6760 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6761 else {
6762
6763 I915_WRITE(GEN7_ROW_CHICKEN2,
6764 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6765 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
6766 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6767 }
6768
6769
6770 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6771 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6772
6773
6774
6775
6776
6777 I915_WRITE(GEN6_UCGCTL2,
6778 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6779
6780
6781 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6782 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6783 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6784
6785 g4x_disable_trickle_feed(dev);
6786
6787 gen7_setup_fixed_func_scheduler(dev_priv);
6788
6789 if (0) {
6790
6791 I915_WRITE(CACHE_MODE_0_GEN7,
6792 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6793 }
6794
6795
6796 I915_WRITE(CACHE_MODE_1,
6797 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807 I915_WRITE(GEN7_GT_MODE,
6808 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6809
6810 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
6811 snpcr &= ~GEN6_MBC_SNPCR_MASK;
6812 snpcr |= GEN6_MBC_SNPCR_MED;
6813 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
6814
6815 if (!HAS_PCH_NOP(dev))
6816 cpt_init_clock_gating(dev);
6817
6818 gen6_check_mch_setup(dev);
6819}
6820
6821static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
6822{
6823 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6824
6825
6826
6827
6828 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6829 I915_WRITE(CBR1_VLV, 0);
6830}
6831
6832static void valleyview_init_clock_gating(struct drm_device *dev)
6833{
6834 struct drm_i915_private *dev_priv = dev->dev_private;
6835
6836 vlv_init_display_clock_gating(dev_priv);
6837
6838
6839 I915_WRITE(_3D_CHICKEN3,
6840 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6841
6842
6843 I915_WRITE(IVB_CHICKEN3,
6844 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6845 CHICKEN3_DGMG_DONE_FIX_DISABLE);
6846
6847
6848
6849 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6850 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
6851 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6852
6853
6854 I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6855
6856
6857 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6858 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6859
6860
6861 I915_WRITE(GEN7_ROW_CHICKEN2,
6862 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6863
6864
6865 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6866 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6867 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6868
6869 gen7_setup_fixed_func_scheduler(dev_priv);
6870
6871
6872
6873
6874
6875 I915_WRITE(GEN6_UCGCTL2,
6876 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6877
6878
6879
6880
6881 I915_WRITE(GEN7_UCGCTL4,
6882 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
6883
6884
6885
6886
6887
6888 I915_WRITE(CACHE_MODE_1,
6889 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899 I915_WRITE(GEN7_GT_MODE,
6900 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6901
6902
6903
6904
6905
6906 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
6907
6908
6909
6910
6911
6912
6913 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6914}
6915
6916static void cherryview_init_clock_gating(struct drm_device *dev)
6917{
6918 struct drm_i915_private *dev_priv = dev->dev_private;
6919
6920 vlv_init_display_clock_gating(dev_priv);
6921
6922
6923
6924 I915_WRITE(GEN7_FF_THREAD_MODE,
6925 I915_READ(GEN7_FF_THREAD_MODE) &
6926 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6927
6928
6929 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6930 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6931
6932
6933 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6934 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6935
6936
6937 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6938 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6939
6940
6941
6942
6943
6944 I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
6945}
6946
6947static void g4x_init_clock_gating(struct drm_device *dev)
6948{
6949 struct drm_i915_private *dev_priv = dev->dev_private;
6950 uint32_t dspclk_gate;
6951
6952 I915_WRITE(RENCLK_GATE_D1, 0);
6953 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
6954 GS_UNIT_CLOCK_GATE_DISABLE |
6955 CL_UNIT_CLOCK_GATE_DISABLE);
6956 I915_WRITE(RAMCLK_GATE_D, 0);
6957 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
6958 OVRUNIT_CLOCK_GATE_DISABLE |
6959 OVCUNIT_CLOCK_GATE_DISABLE;
6960 if (IS_GM45(dev))
6961 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
6962 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
6963
6964
6965 I915_WRITE(CACHE_MODE_0,
6966 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6967
6968
6969 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6970
6971 g4x_disable_trickle_feed(dev);
6972}
6973
6974static void crestline_init_clock_gating(struct drm_device *dev)
6975{
6976 struct drm_i915_private *dev_priv = dev->dev_private;
6977
6978 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
6979 I915_WRITE(RENCLK_GATE_D2, 0);
6980 I915_WRITE(DSPCLK_GATE_D, 0);
6981 I915_WRITE(RAMCLK_GATE_D, 0);
6982 I915_WRITE16(DEUC, 0);
6983 I915_WRITE(MI_ARB_STATE,
6984 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6985
6986
6987 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6988}
6989
6990static void broadwater_init_clock_gating(struct drm_device *dev)
6991{
6992 struct drm_i915_private *dev_priv = dev->dev_private;
6993
6994 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
6995 I965_RCC_CLOCK_GATE_DISABLE |
6996 I965_RCPB_CLOCK_GATE_DISABLE |
6997 I965_ISC_CLOCK_GATE_DISABLE |
6998 I965_FBC_CLOCK_GATE_DISABLE);
6999 I915_WRITE(RENCLK_GATE_D2, 0);
7000 I915_WRITE(MI_ARB_STATE,
7001 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7002
7003
7004 I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7005}
7006
7007static void gen3_init_clock_gating(struct drm_device *dev)
7008{
7009 struct drm_i915_private *dev_priv = dev->dev_private;
7010 u32 dstate = I915_READ(D_STATE);
7011
7012 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7013 DSTATE_DOT_CLOCK_GATING;
7014 I915_WRITE(D_STATE, dstate);
7015
7016 if (IS_PINEVIEW(dev))
7017 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7018
7019
7020 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7021
7022
7023 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7024
7025
7026 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7027
7028 I915_WRITE(MI_ARB_STATE,
7029 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7030}
7031
7032static void i85x_init_clock_gating(struct drm_device *dev)
7033{
7034 struct drm_i915_private *dev_priv = dev->dev_private;
7035
7036 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7037
7038
7039 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7040 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7041
7042 I915_WRITE(MEM_MODE,
7043 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7044}
7045
7046static void i830_init_clock_gating(struct drm_device *dev)
7047{
7048 struct drm_i915_private *dev_priv = dev->dev_private;
7049
7050 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
7051
7052 I915_WRITE(MEM_MODE,
7053 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7054 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7055}
7056
7057void intel_init_clock_gating(struct drm_device *dev)
7058{
7059 struct drm_i915_private *dev_priv = dev->dev_private;
7060
7061 if (dev_priv->display.init_clock_gating)
7062 dev_priv->display.init_clock_gating(dev);
7063}
7064
7065void intel_suspend_hw(struct drm_device *dev)
7066{
7067 if (HAS_PCH_LPT(dev))
7068 lpt_suspend_hw(dev);
7069}
7070
7071
7072void intel_init_pm(struct drm_device *dev)
7073{
7074 struct drm_i915_private *dev_priv = dev->dev_private;
7075
7076 intel_fbc_init(dev_priv);
7077
7078
7079 if (IS_PINEVIEW(dev))
7080 i915_pineview_get_mem_freq(dev);
7081 else if (IS_GEN5(dev))
7082 i915_ironlake_get_mem_freq(dev);
7083
7084
7085 if (INTEL_INFO(dev)->gen >= 9) {
7086 skl_setup_wm_latency(dev);
7087
7088 if (IS_BROXTON(dev))
7089 dev_priv->display.init_clock_gating =
7090 bxt_init_clock_gating;
7091 dev_priv->display.update_wm = skl_update_wm;
7092 } else if (HAS_PCH_SPLIT(dev)) {
7093 ilk_setup_wm_latency(dev);
7094
7095 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
7096 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7097 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
7098 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7099 dev_priv->display.update_wm = ilk_update_wm;
7100 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7101 dev_priv->display.program_watermarks = ilk_program_watermarks;
7102 } else {
7103 DRM_DEBUG_KMS("Failed to read display plane latency. "
7104 "Disable CxSR\n");
7105 }
7106
7107 if (IS_GEN5(dev))
7108 dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
7109 else if (IS_GEN6(dev))
7110 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7111 else if (IS_IVYBRIDGE(dev))
7112 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
7113 else if (IS_HASWELL(dev))
7114 dev_priv->display.init_clock_gating = haswell_init_clock_gating;
7115 else if (INTEL_INFO(dev)->gen == 8)
7116 dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
7117 } else if (IS_CHERRYVIEW(dev)) {
7118 vlv_setup_wm_latency(dev);
7119
7120 dev_priv->display.update_wm = vlv_update_wm;
7121 dev_priv->display.init_clock_gating =
7122 cherryview_init_clock_gating;
7123 } else if (IS_VALLEYVIEW(dev)) {
7124 vlv_setup_wm_latency(dev);
7125
7126 dev_priv->display.update_wm = vlv_update_wm;
7127 dev_priv->display.init_clock_gating =
7128 valleyview_init_clock_gating;
7129 } else if (IS_PINEVIEW(dev)) {
7130 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
7131 dev_priv->is_ddr3,
7132 dev_priv->fsb_freq,
7133 dev_priv->mem_freq)) {
7134 DRM_INFO("failed to find known CxSR latency "
7135 "(found ddr%s fsb freq %d, mem freq %d), "
7136 "disabling CxSR\n",
7137 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7138 dev_priv->fsb_freq, dev_priv->mem_freq);
7139
7140 intel_set_memory_cxsr(dev_priv, false);
7141 dev_priv->display.update_wm = NULL;
7142 } else
7143 dev_priv->display.update_wm = pineview_update_wm;
7144 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7145 } else if (IS_G4X(dev)) {
7146 dev_priv->display.update_wm = g4x_update_wm;
7147 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7148 } else if (IS_GEN4(dev)) {
7149 dev_priv->display.update_wm = i965_update_wm;
7150 if (IS_CRESTLINE(dev))
7151 dev_priv->display.init_clock_gating = crestline_init_clock_gating;
7152 else if (IS_BROADWATER(dev))
7153 dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
7154 } else if (IS_GEN3(dev)) {
7155 dev_priv->display.update_wm = i9xx_update_wm;
7156 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7157 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7158 } else if (IS_GEN2(dev)) {
7159 if (INTEL_INFO(dev)->num_pipes == 1) {
7160 dev_priv->display.update_wm = i845_update_wm;
7161 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7162 } else {
7163 dev_priv->display.update_wm = i9xx_update_wm;
7164 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7165 }
7166
7167 if (IS_I85X(dev) || IS_I865G(dev))
7168 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7169 else
7170 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7171 } else {
7172 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
7173 }
7174}
7175
7176int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
7177{
7178 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
7179
7180 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
7181 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
7182 return -EAGAIN;
7183 }
7184
7185 I915_WRITE(GEN6_PCODE_DATA, *val);
7186 I915_WRITE(GEN6_PCODE_DATA1, 0);
7187 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
7188
7189 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7190 500)) {
7191 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
7192 return -ETIMEDOUT;
7193 }
7194
7195 *val = I915_READ(GEN6_PCODE_DATA);
7196 I915_WRITE(GEN6_PCODE_DATA, 0);
7197
7198 return 0;
7199}
7200
7201int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
7202{
7203 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
7204
7205 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
7206 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
7207 return -EAGAIN;
7208 }
7209
7210 I915_WRITE(GEN6_PCODE_DATA, val);
7211 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
7212
7213 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
7214 500)) {
7215 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
7216 return -ETIMEDOUT;
7217 }
7218
7219 I915_WRITE(GEN6_PCODE_DATA, 0);
7220
7221 return 0;
7222}
7223
7224static int vlv_gpu_freq_div(unsigned int czclk_freq)
7225{
7226 switch (czclk_freq) {
7227 case 200:
7228 return 10;
7229 case 267:
7230 return 12;
7231 case 320:
7232 case 333:
7233 return 16;
7234 case 400:
7235 return 20;
7236 default:
7237 return -1;
7238 }
7239}
7240
7241static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
7242{
7243 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
7244
7245 div = vlv_gpu_freq_div(czclk_freq);
7246 if (div < 0)
7247 return div;
7248
7249 return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
7250}
7251
7252static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
7253{
7254 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
7255
7256 mul = vlv_gpu_freq_div(czclk_freq);
7257 if (mul < 0)
7258 return mul;
7259
7260 return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
7261}
7262
7263static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
7264{
7265 int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
7266
7267 div = vlv_gpu_freq_div(czclk_freq);
7268 if (div < 0)
7269 return div;
7270 div /= 2;
7271
7272 return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
7273}
7274
7275static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
7276{
7277 int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
7278
7279 mul = vlv_gpu_freq_div(czclk_freq);
7280 if (mul < 0)
7281 return mul;
7282 mul /= 2;
7283
7284
7285 return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
7286}
7287
7288int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
7289{
7290 if (IS_GEN9(dev_priv->dev))
7291 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
7292 GEN9_FREQ_SCALER);
7293 else if (IS_CHERRYVIEW(dev_priv->dev))
7294 return chv_gpu_freq(dev_priv, val);
7295 else if (IS_VALLEYVIEW(dev_priv->dev))
7296 return byt_gpu_freq(dev_priv, val);
7297 else
7298 return val * GT_FREQUENCY_MULTIPLIER;
7299}
7300
7301int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
7302{
7303 if (IS_GEN9(dev_priv->dev))
7304 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
7305 GT_FREQUENCY_MULTIPLIER);
7306 else if (IS_CHERRYVIEW(dev_priv->dev))
7307 return chv_freq_opcode(dev_priv, val);
7308 else if (IS_VALLEYVIEW(dev_priv->dev))
7309 return byt_freq_opcode(dev_priv, val);
7310 else
7311 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
7312}
7313
7314struct request_boost {
7315 struct work_struct work;
7316 struct drm_i915_gem_request *req;
7317};
7318
7319static void __intel_rps_boost_work(struct work_struct *work)
7320{
7321 struct request_boost *boost = container_of(work, struct request_boost, work);
7322 struct drm_i915_gem_request *req = boost->req;
7323
7324 if (!i915_gem_request_completed(req, true))
7325 gen6_rps_boost(to_i915(req->ring->dev), NULL,
7326 req->emitted_jiffies);
7327
7328 i915_gem_request_unreference__unlocked(req);
7329 kfree(boost);
7330}
7331
7332void intel_queue_rps_boost_for_request(struct drm_device *dev,
7333 struct drm_i915_gem_request *req)
7334{
7335 struct request_boost *boost;
7336
7337 if (req == NULL || INTEL_INFO(dev)->gen < 6)
7338 return;
7339
7340 if (i915_gem_request_completed(req, true))
7341 return;
7342
7343 boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
7344 if (boost == NULL)
7345 return;
7346
7347 i915_gem_request_reference(req);
7348 boost->req = req;
7349
7350 INIT_WORK(&boost->work, __intel_rps_boost_work);
7351 queue_work(to_i915(dev)->wq, &boost->work);
7352}
7353
7354void intel_pm_setup(struct drm_device *dev)
7355{
7356 struct drm_i915_private *dev_priv = dev->dev_private;
7357
7358 mutex_init(&dev_priv->rps.hw_lock);
7359 spin_lock_init(&dev_priv->rps.client_lock);
7360
7361 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
7362 intel_gen6_powersave_work);
7363 INIT_LIST_HEAD(&dev_priv->rps.clients);
7364 INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
7365 INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
7366
7367 dev_priv->pm.suspended = false;
7368 atomic_set(&dev_priv->pm.wakeref_count, 0);
7369 atomic_set(&dev_priv->pm.atomic_seq, 0);
7370}
7371