1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/module.h>
29#include <linux/pm_runtime.h>
30
31#include <drm/drm_atomic_helper.h>
32#include <drm/drm_fourcc.h>
33#include <drm/drm_plane_helper.h>
34
35#include "display/intel_atomic.h"
36#include "display/intel_atomic_plane.h"
37#include "display/intel_bw.h"
38#include "display/intel_de.h"
39#include "display/intel_display_types.h"
40#include "display/intel_fbc.h"
41#include "display/intel_sprite.h"
42#include "display/skl_universal_plane.h"
43
44#include "gt/intel_llc.h"
45
46#include "i915_drv.h"
47#include "i915_fixed.h"
48#include "i915_irq.h"
49#include "i915_trace.h"
50#include "intel_pm.h"
51#include "intel_sideband.h"
52#include "../../../platform/x86/intel_ips.h"
53
54
55struct skl_wm_params {
56 bool x_tiled, y_tiled;
57 bool rc_surface;
58 bool is_planar;
59 u32 width;
60 u8 cpp;
61 u32 plane_pixel_rate;
62 u32 y_min_scanlines;
63 u32 plane_bytes_per_line;
64 uint_fixed_16_16_t plane_blocks_per_line;
65 uint_fixed_16_16_t y_tile_minimum;
66 u32 linetime_us;
67 u32 dbuf_block_size;
68};
69
70
71struct intel_wm_config {
72 unsigned int num_pipes_active;
73 bool sprites_enabled;
74 bool sprites_scaled;
75};
76
77static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
78{
79 enum pipe pipe;
80
81 if (HAS_LLC(dev_priv)) {
82
83
84
85
86
87
88
89 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
90 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
91 SKL_DE_COMPRESSED_HASH_MODE);
92 }
93
94 for_each_pipe(dev_priv, pipe) {
95
96
97
98
99 if (!IS_GEMINILAKE(dev_priv) && intel_vtd_active())
100 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
101 SKL_PLANE1_STRETCH_MAX_MASK, SKL_PLANE1_STRETCH_MAX_X1);
102 }
103
104
105 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
106 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
107
108
109 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
110 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
111
112
113
114
115
116 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
117 DISP_FBC_MEMORY_WAKE);
118}
119
120static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
121{
122 gen9_init_clock_gating(dev_priv);
123
124
125 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
126 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
127
128
129
130
131
132 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
133 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
134
135
136
137
138
139 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
140 PWM1_GATING_DIS | PWM2_GATING_DIS);
141
142
143
144
145
146
147
148 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
149
150
151
152
153
154 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
155 DISP_FBC_WM_DIS);
156
157
158
159
160
161 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
162 ILK_DPFC_DISABLE_DUMMY0);
163}
164
165static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
166{
167 gen9_init_clock_gating(dev_priv);
168
169
170
171
172
173
174 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
175 PWM1_GATING_DIS | PWM2_GATING_DIS);
176}
177
178static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
179{
180 u32 tmp;
181
182 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
183
184 switch (tmp & CLKCFG_FSB_MASK) {
185 case CLKCFG_FSB_533:
186 dev_priv->fsb_freq = 533;
187 break;
188 case CLKCFG_FSB_800:
189 dev_priv->fsb_freq = 800;
190 break;
191 case CLKCFG_FSB_667:
192 dev_priv->fsb_freq = 667;
193 break;
194 case CLKCFG_FSB_400:
195 dev_priv->fsb_freq = 400;
196 break;
197 }
198
199 switch (tmp & CLKCFG_MEM_MASK) {
200 case CLKCFG_MEM_533:
201 dev_priv->mem_freq = 533;
202 break;
203 case CLKCFG_MEM_667:
204 dev_priv->mem_freq = 667;
205 break;
206 case CLKCFG_MEM_800:
207 dev_priv->mem_freq = 800;
208 break;
209 }
210
211
212 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
213 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
214}
215
216static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
217{
218 u16 ddrpll, csipll;
219
220 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
221 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
222
223 switch (ddrpll & 0xff) {
224 case 0xc:
225 dev_priv->mem_freq = 800;
226 break;
227 case 0x10:
228 dev_priv->mem_freq = 1066;
229 break;
230 case 0x14:
231 dev_priv->mem_freq = 1333;
232 break;
233 case 0x18:
234 dev_priv->mem_freq = 1600;
235 break;
236 default:
237 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
238 ddrpll & 0xff);
239 dev_priv->mem_freq = 0;
240 break;
241 }
242
243 switch (csipll & 0x3ff) {
244 case 0x00c:
245 dev_priv->fsb_freq = 3200;
246 break;
247 case 0x00e:
248 dev_priv->fsb_freq = 3733;
249 break;
250 case 0x010:
251 dev_priv->fsb_freq = 4266;
252 break;
253 case 0x012:
254 dev_priv->fsb_freq = 4800;
255 break;
256 case 0x014:
257 dev_priv->fsb_freq = 5333;
258 break;
259 case 0x016:
260 dev_priv->fsb_freq = 5866;
261 break;
262 case 0x018:
263 dev_priv->fsb_freq = 6400;
264 break;
265 default:
266 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
267 csipll & 0x3ff);
268 dev_priv->fsb_freq = 0;
269 break;
270 }
271}
272
273static const struct cxsr_latency cxsr_latency_table[] = {
274 {1, 0, 800, 400, 3382, 33382, 3983, 33983},
275 {1, 0, 800, 667, 3354, 33354, 3807, 33807},
276 {1, 0, 800, 800, 3347, 33347, 3763, 33763},
277 {1, 1, 800, 667, 6420, 36420, 6873, 36873},
278 {1, 1, 800, 800, 5902, 35902, 6318, 36318},
279
280 {1, 0, 667, 400, 3400, 33400, 4021, 34021},
281 {1, 0, 667, 667, 3372, 33372, 3845, 33845},
282 {1, 0, 667, 800, 3386, 33386, 3822, 33822},
283 {1, 1, 667, 667, 6438, 36438, 6911, 36911},
284 {1, 1, 667, 800, 5941, 35941, 6377, 36377},
285
286 {1, 0, 400, 400, 3472, 33472, 4173, 34173},
287 {1, 0, 400, 667, 3443, 33443, 3996, 33996},
288 {1, 0, 400, 800, 3430, 33430, 3946, 33946},
289 {1, 1, 400, 667, 6509, 36509, 7062, 37062},
290 {1, 1, 400, 800, 5985, 35985, 6501, 36501},
291
292 {0, 0, 800, 400, 3438, 33438, 4065, 34065},
293 {0, 0, 800, 667, 3410, 33410, 3889, 33889},
294 {0, 0, 800, 800, 3403, 33403, 3845, 33845},
295 {0, 1, 800, 667, 6476, 36476, 6955, 36955},
296 {0, 1, 800, 800, 5958, 35958, 6400, 36400},
297
298 {0, 0, 667, 400, 3456, 33456, 4103, 34106},
299 {0, 0, 667, 667, 3428, 33428, 3927, 33927},
300 {0, 0, 667, 800, 3443, 33443, 3905, 33905},
301 {0, 1, 667, 667, 6494, 36494, 6993, 36993},
302 {0, 1, 667, 800, 5998, 35998, 6460, 36460},
303
304 {0, 0, 400, 400, 3528, 33528, 4255, 34255},
305 {0, 0, 400, 667, 3500, 33500, 4079, 34079},
306 {0, 0, 400, 800, 3487, 33487, 4029, 34029},
307 {0, 1, 400, 667, 6566, 36566, 7145, 37145},
308 {0, 1, 400, 800, 6042, 36042, 6584, 36584},
309};
310
311static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
312 bool is_ddr3,
313 int fsb,
314 int mem)
315{
316 const struct cxsr_latency *latency;
317 int i;
318
319 if (fsb == 0 || mem == 0)
320 return NULL;
321
322 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
323 latency = &cxsr_latency_table[i];
324 if (is_desktop == latency->is_desktop &&
325 is_ddr3 == latency->is_ddr3 &&
326 fsb == latency->fsb_freq && mem == latency->mem_freq)
327 return latency;
328 }
329
330 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
331
332 return NULL;
333}
334
335static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
336{
337 u32 val;
338
339 vlv_punit_get(dev_priv);
340
341 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
342 if (enable)
343 val &= ~FORCE_DDR_HIGH_FREQ;
344 else
345 val |= FORCE_DDR_HIGH_FREQ;
346 val &= ~FORCE_DDR_LOW_FREQ;
347 val |= FORCE_DDR_FREQ_REQ_ACK;
348 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
349
350 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
351 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
352 drm_err(&dev_priv->drm,
353 "timed out waiting for Punit DDR DVFS request\n");
354
355 vlv_punit_put(dev_priv);
356}
357
358static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
359{
360 u32 val;
361
362 vlv_punit_get(dev_priv);
363
364 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
365 if (enable)
366 val |= DSP_MAXFIFO_PM5_ENABLE;
367 else
368 val &= ~DSP_MAXFIFO_PM5_ENABLE;
369 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
370
371 vlv_punit_put(dev_priv);
372}
373
374#define FW_WM(value, plane) \
375 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
376
377static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
378{
379 bool was_enabled;
380 u32 val;
381
382 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
383 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
384 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
385 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
386 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
387 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
388 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
389 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
390 } else if (IS_PINEVIEW(dev_priv)) {
391 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
392 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
393 if (enable)
394 val |= PINEVIEW_SELF_REFRESH_EN;
395 else
396 val &= ~PINEVIEW_SELF_REFRESH_EN;
397 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
398 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
399 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
400 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
401 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
402 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
403 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
404 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
405 } else if (IS_I915GM(dev_priv)) {
406
407
408
409
410
411 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
412 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
413 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
414 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
415 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
416 } else {
417 return false;
418 }
419
420 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
421
422 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
423 enableddisabled(enable),
424 enableddisabled(was_enabled));
425
426 return was_enabled;
427}
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
467{
468 bool ret;
469
470 mutex_lock(&dev_priv->wm.wm_mutex);
471 ret = _intel_set_memory_cxsr(dev_priv, enable);
472 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
473 dev_priv->wm.vlv.cxsr = enable;
474 else if (IS_G4X(dev_priv))
475 dev_priv->wm.g4x.cxsr = enable;
476 mutex_unlock(&dev_priv->wm.wm_mutex);
477
478 return ret;
479}
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495static const int pessimal_latency_ns = 5000;
496
497#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
498 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
499
500static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
501{
502 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
503 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
504 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
505 enum pipe pipe = crtc->pipe;
506 int sprite0_start, sprite1_start;
507 u32 dsparb, dsparb2, dsparb3;
508
509 switch (pipe) {
510 case PIPE_A:
511 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
512 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
513 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
514 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
515 break;
516 case PIPE_B:
517 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
518 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
519 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
520 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
521 break;
522 case PIPE_C:
523 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
524 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
525 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
526 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
527 break;
528 default:
529 MISSING_CASE(pipe);
530 return;
531 }
532
533 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
534 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
535 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
536 fifo_state->plane[PLANE_CURSOR] = 63;
537}
538
539static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
540 enum i9xx_plane_id i9xx_plane)
541{
542 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
543 int size;
544
545 size = dsparb & 0x7f;
546 if (i9xx_plane == PLANE_B)
547 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
548
549 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
550 dsparb, plane_name(i9xx_plane), size);
551
552 return size;
553}
554
555static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
556 enum i9xx_plane_id i9xx_plane)
557{
558 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
559 int size;
560
561 size = dsparb & 0x1ff;
562 if (i9xx_plane == PLANE_B)
563 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
564 size >>= 1;
565
566 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
567 dsparb, plane_name(i9xx_plane), size);
568
569 return size;
570}
571
572static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
573 enum i9xx_plane_id i9xx_plane)
574{
575 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
576 int size;
577
578 size = dsparb & 0x7f;
579 size >>= 2;
580
581 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
582 dsparb, plane_name(i9xx_plane), size);
583
584 return size;
585}
586
587
588static const struct intel_watermark_params pnv_display_wm = {
589 .fifo_size = PINEVIEW_DISPLAY_FIFO,
590 .max_wm = PINEVIEW_MAX_WM,
591 .default_wm = PINEVIEW_DFT_WM,
592 .guard_size = PINEVIEW_GUARD_WM,
593 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
594};
595
596static const struct intel_watermark_params pnv_display_hplloff_wm = {
597 .fifo_size = PINEVIEW_DISPLAY_FIFO,
598 .max_wm = PINEVIEW_MAX_WM,
599 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
600 .guard_size = PINEVIEW_GUARD_WM,
601 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
602};
603
604static const struct intel_watermark_params pnv_cursor_wm = {
605 .fifo_size = PINEVIEW_CURSOR_FIFO,
606 .max_wm = PINEVIEW_CURSOR_MAX_WM,
607 .default_wm = PINEVIEW_CURSOR_DFT_WM,
608 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
609 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
610};
611
612static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
613 .fifo_size = PINEVIEW_CURSOR_FIFO,
614 .max_wm = PINEVIEW_CURSOR_MAX_WM,
615 .default_wm = PINEVIEW_CURSOR_DFT_WM,
616 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
617 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
618};
619
620static const struct intel_watermark_params i965_cursor_wm_info = {
621 .fifo_size = I965_CURSOR_FIFO,
622 .max_wm = I965_CURSOR_MAX_WM,
623 .default_wm = I965_CURSOR_DFT_WM,
624 .guard_size = 2,
625 .cacheline_size = I915_FIFO_LINE_SIZE,
626};
627
628static const struct intel_watermark_params i945_wm_info = {
629 .fifo_size = I945_FIFO_SIZE,
630 .max_wm = I915_MAX_WM,
631 .default_wm = 1,
632 .guard_size = 2,
633 .cacheline_size = I915_FIFO_LINE_SIZE,
634};
635
636static const struct intel_watermark_params i915_wm_info = {
637 .fifo_size = I915_FIFO_SIZE,
638 .max_wm = I915_MAX_WM,
639 .default_wm = 1,
640 .guard_size = 2,
641 .cacheline_size = I915_FIFO_LINE_SIZE,
642};
643
644static const struct intel_watermark_params i830_a_wm_info = {
645 .fifo_size = I855GM_FIFO_SIZE,
646 .max_wm = I915_MAX_WM,
647 .default_wm = 1,
648 .guard_size = 2,
649 .cacheline_size = I830_FIFO_LINE_SIZE,
650};
651
652static const struct intel_watermark_params i830_bc_wm_info = {
653 .fifo_size = I855GM_FIFO_SIZE,
654 .max_wm = I915_MAX_WM/2,
655 .default_wm = 1,
656 .guard_size = 2,
657 .cacheline_size = I830_FIFO_LINE_SIZE,
658};
659
660static const struct intel_watermark_params i845_wm_info = {
661 .fifo_size = I830_FIFO_SIZE,
662 .max_wm = I915_MAX_WM,
663 .default_wm = 1,
664 .guard_size = 2,
665 .cacheline_size = I830_FIFO_LINE_SIZE,
666};
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701static unsigned int intel_wm_method1(unsigned int pixel_rate,
702 unsigned int cpp,
703 unsigned int latency)
704{
705 u64 ret;
706
707 ret = mul_u32_u32(pixel_rate, cpp * latency);
708 ret = DIV_ROUND_UP_ULL(ret, 10000);
709
710 return ret;
711}
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743static unsigned int intel_wm_method2(unsigned int pixel_rate,
744 unsigned int htotal,
745 unsigned int width,
746 unsigned int cpp,
747 unsigned int latency)
748{
749 unsigned int ret;
750
751
752
753
754
755 if (WARN_ON_ONCE(htotal == 0))
756 htotal = 1;
757
758 ret = (latency * pixel_rate) / (htotal * 10000);
759 ret = (ret + 1) * width * cpp;
760
761 return ret;
762}
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783static unsigned int intel_calculate_wm(int pixel_rate,
784 const struct intel_watermark_params *wm,
785 int fifo_size, int cpp,
786 unsigned int latency_ns)
787{
788 int entries, wm_size;
789
790
791
792
793
794
795
796 entries = intel_wm_method1(pixel_rate, cpp,
797 latency_ns / 100);
798 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
799 wm->guard_size;
800 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
801
802 wm_size = fifo_size - entries;
803 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
804
805
806 if (wm_size > wm->max_wm)
807 wm_size = wm->max_wm;
808 if (wm_size <= 0)
809 wm_size = wm->default_wm;
810
811
812
813
814
815
816
817
818 if (wm_size <= 8)
819 wm_size = 8;
820
821 return wm_size;
822}
823
824static bool is_disabling(int old, int new, int threshold)
825{
826 return old >= threshold && new < threshold;
827}
828
829static bool is_enabling(int old, int new, int threshold)
830{
831 return old < threshold && new >= threshold;
832}
833
834static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
835{
836 return dev_priv->wm.max_level + 1;
837}
838
839static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
840 const struct intel_plane_state *plane_state)
841{
842 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
843
844
845 if (!crtc_state->hw.active)
846 return false;
847
848
849
850
851
852
853
854
855
856 if (plane->id == PLANE_CURSOR)
857 return plane_state->hw.fb != NULL;
858 else
859 return plane_state->uapi.visible;
860}
861
862static bool intel_crtc_active(struct intel_crtc *crtc)
863{
864
865
866
867
868
869
870
871
872
873
874
875
876
877 return crtc->active && crtc->base.primary->state->fb &&
878 crtc->config->hw.adjusted_mode.crtc_clock;
879}
880
881static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
882{
883 struct intel_crtc *crtc, *enabled = NULL;
884
885 for_each_intel_crtc(&dev_priv->drm, crtc) {
886 if (intel_crtc_active(crtc)) {
887 if (enabled)
888 return NULL;
889 enabled = crtc;
890 }
891 }
892
893 return enabled;
894}
895
896static void pnv_update_wm(struct intel_crtc *unused_crtc)
897{
898 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
899 struct intel_crtc *crtc;
900 const struct cxsr_latency *latency;
901 u32 reg;
902 unsigned int wm;
903
904 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
905 dev_priv->is_ddr3,
906 dev_priv->fsb_freq,
907 dev_priv->mem_freq);
908 if (!latency) {
909 drm_dbg_kms(&dev_priv->drm,
910 "Unknown FSB/MEM found, disable CxSR\n");
911 intel_set_memory_cxsr(dev_priv, false);
912 return;
913 }
914
915 crtc = single_enabled_crtc(dev_priv);
916 if (crtc) {
917 const struct drm_display_mode *pipe_mode =
918 &crtc->config->hw.pipe_mode;
919 const struct drm_framebuffer *fb =
920 crtc->base.primary->state->fb;
921 int cpp = fb->format->cpp[0];
922 int clock = pipe_mode->crtc_clock;
923
924
925 wm = intel_calculate_wm(clock, &pnv_display_wm,
926 pnv_display_wm.fifo_size,
927 cpp, latency->display_sr);
928 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
929 reg &= ~DSPFW_SR_MASK;
930 reg |= FW_WM(wm, SR);
931 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
932 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
933
934
935 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
936 pnv_display_wm.fifo_size,
937 4, latency->cursor_sr);
938 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
939 reg &= ~DSPFW_CURSOR_SR_MASK;
940 reg |= FW_WM(wm, CURSOR_SR);
941 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
942
943
944 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
945 pnv_display_hplloff_wm.fifo_size,
946 cpp, latency->display_hpll_disable);
947 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
948 reg &= ~DSPFW_HPLL_SR_MASK;
949 reg |= FW_WM(wm, HPLL_SR);
950 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
951
952
953 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
954 pnv_display_hplloff_wm.fifo_size,
955 4, latency->cursor_hpll_disable);
956 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
957 reg &= ~DSPFW_HPLL_CURSOR_MASK;
958 reg |= FW_WM(wm, HPLL_CURSOR);
959 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
960 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
961
962 intel_set_memory_cxsr(dev_priv, true);
963 } else {
964 intel_set_memory_cxsr(dev_priv, false);
965 }
966}
967
968
969
970
971
972
973
974
975
976
977
978static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
979{
980 int tlb_miss = fifo_size * 64 - width * cpp * 8;
981
982 return max(0, tlb_miss);
983}
984
985static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
986 const struct g4x_wm_values *wm)
987{
988 enum pipe pipe;
989
990 for_each_pipe(dev_priv, pipe)
991 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
992
993 intel_uncore_write(&dev_priv->uncore, DSPFW1,
994 FW_WM(wm->sr.plane, SR) |
995 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
996 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
998 intel_uncore_write(&dev_priv->uncore, DSPFW2,
999 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
1000 FW_WM(wm->sr.fbc, FBC_SR) |
1001 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
1002 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
1003 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1004 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1005 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1006 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1007 FW_WM(wm->sr.cursor, CURSOR_SR) |
1008 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1009 FW_WM(wm->hpll.plane, HPLL_SR));
1010
1011 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1012}
1013
1014#define FW_WM_VLV(value, plane) \
1015 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1016
1017static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1018 const struct vlv_wm_values *wm)
1019{
1020 enum pipe pipe;
1021
1022 for_each_pipe(dev_priv, pipe) {
1023 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1024
1025 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1026 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1027 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1028 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1029 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1030 }
1031
1032
1033
1034
1035
1036
1037 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1039 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1040 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1041 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1042
1043 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1044 FW_WM(wm->sr.plane, SR) |
1045 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1046 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1047 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1048 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1049 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1050 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1051 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1052 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1053 FW_WM(wm->sr.cursor, CURSOR_SR));
1054
1055 if (IS_CHERRYVIEW(dev_priv)) {
1056 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1057 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1058 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1059 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1060 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1061 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1062 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1063 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1064 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1065 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1066 FW_WM(wm->sr.plane >> 9, SR_HI) |
1067 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1068 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1069 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1071 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1072 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1074 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1075 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1076 } else {
1077 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1078 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1079 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1080 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1081 FW_WM(wm->sr.plane >> 9, SR_HI) |
1082 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1083 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1084 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1085 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1086 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1087 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1088 }
1089
1090 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1091}
1092
1093#undef FW_WM_VLV
1094
1095static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1096{
1097
1098 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1099 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1100 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1101
1102 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1103}
1104
1105static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1106{
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121 switch (plane_id) {
1122 case PLANE_CURSOR:
1123 return 63;
1124 case PLANE_PRIMARY:
1125 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1126 case PLANE_SPRITE0:
1127 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1128 default:
1129 MISSING_CASE(plane_id);
1130 return 0;
1131 }
1132}
1133
1134static int g4x_fbc_fifo_size(int level)
1135{
1136 switch (level) {
1137 case G4X_WM_LEVEL_SR:
1138 return 7;
1139 case G4X_WM_LEVEL_HPLL:
1140 return 15;
1141 default:
1142 MISSING_CASE(level);
1143 return 0;
1144 }
1145}
1146
1147static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1148 const struct intel_plane_state *plane_state,
1149 int level)
1150{
1151 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1152 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1153 const struct drm_display_mode *pipe_mode =
1154 &crtc_state->hw.pipe_mode;
1155 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1156 unsigned int clock, htotal, cpp, width, wm;
1157
1158 if (latency == 0)
1159 return USHRT_MAX;
1160
1161 if (!intel_wm_plane_visible(crtc_state, plane_state))
1162 return 0;
1163
1164 cpp = plane_state->hw.fb->format->cpp[0];
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1178 level != G4X_WM_LEVEL_NORMAL)
1179 cpp = max(cpp, 4u);
1180
1181 clock = pipe_mode->crtc_clock;
1182 htotal = pipe_mode->crtc_htotal;
1183
1184 width = drm_rect_width(&plane_state->uapi.dst);
1185
1186 if (plane->id == PLANE_CURSOR) {
1187 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1188 } else if (plane->id == PLANE_PRIMARY &&
1189 level == G4X_WM_LEVEL_NORMAL) {
1190 wm = intel_wm_method1(clock, cpp, latency);
1191 } else {
1192 unsigned int small, large;
1193
1194 small = intel_wm_method1(clock, cpp, latency);
1195 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1196
1197 wm = min(small, large);
1198 }
1199
1200 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1201 width, cpp);
1202
1203 wm = DIV_ROUND_UP(wm, 64) + 2;
1204
1205 return min_t(unsigned int, wm, USHRT_MAX);
1206}
1207
1208static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1209 int level, enum plane_id plane_id, u16 value)
1210{
1211 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1212 bool dirty = false;
1213
1214 for (; level < intel_wm_num_levels(dev_priv); level++) {
1215 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1216
1217 dirty |= raw->plane[plane_id] != value;
1218 raw->plane[plane_id] = value;
1219 }
1220
1221 return dirty;
1222}
1223
1224static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1225 int level, u16 value)
1226{
1227 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1228 bool dirty = false;
1229
1230
1231 level = max(level, G4X_WM_LEVEL_SR);
1232
1233 for (; level < intel_wm_num_levels(dev_priv); level++) {
1234 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1235
1236 dirty |= raw->fbc != value;
1237 raw->fbc = value;
1238 }
1239
1240 return dirty;
1241}
1242
1243static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1244 const struct intel_plane_state *plane_state,
1245 u32 pri_val);
1246
1247static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1248 const struct intel_plane_state *plane_state)
1249{
1250 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1251 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1252 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1253 enum plane_id plane_id = plane->id;
1254 bool dirty = false;
1255 int level;
1256
1257 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1258 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1259 if (plane_id == PLANE_PRIMARY)
1260 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1261 goto out;
1262 }
1263
1264 for (level = 0; level < num_levels; level++) {
1265 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1266 int wm, max_wm;
1267
1268 wm = g4x_compute_wm(crtc_state, plane_state, level);
1269 max_wm = g4x_plane_fifo_size(plane_id, level);
1270
1271 if (wm > max_wm)
1272 break;
1273
1274 dirty |= raw->plane[plane_id] != wm;
1275 raw->plane[plane_id] = wm;
1276
1277 if (plane_id != PLANE_PRIMARY ||
1278 level == G4X_WM_LEVEL_NORMAL)
1279 continue;
1280
1281 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1282 raw->plane[plane_id]);
1283 max_wm = g4x_fbc_fifo_size(level);
1284
1285
1286
1287
1288
1289 if (wm > max_wm)
1290 wm = USHRT_MAX;
1291
1292 dirty |= raw->fbc != wm;
1293 raw->fbc = wm;
1294 }
1295
1296
1297 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1298
1299 if (plane_id == PLANE_PRIMARY)
1300 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1301
1302 out:
1303 if (dirty) {
1304 drm_dbg_kms(&dev_priv->drm,
1305 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1306 plane->base.name,
1307 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1308 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1309 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1310
1311 if (plane_id == PLANE_PRIMARY)
1312 drm_dbg_kms(&dev_priv->drm,
1313 "FBC watermarks: SR=%d, HPLL=%d\n",
1314 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1315 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1316 }
1317
1318 return dirty;
1319}
1320
1321static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1322 enum plane_id plane_id, int level)
1323{
1324 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1325
1326 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1327}
1328
1329static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1330 int level)
1331{
1332 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1333
1334 if (level > dev_priv->wm.max_level)
1335 return false;
1336
1337 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1338 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1339 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1340}
1341
1342
1343static void g4x_invalidate_wms(struct intel_crtc *crtc,
1344 struct g4x_wm_state *wm_state, int level)
1345{
1346 if (level <= G4X_WM_LEVEL_NORMAL) {
1347 enum plane_id plane_id;
1348
1349 for_each_plane_id_on_crtc(crtc, plane_id)
1350 wm_state->wm.plane[plane_id] = USHRT_MAX;
1351 }
1352
1353 if (level <= G4X_WM_LEVEL_SR) {
1354 wm_state->cxsr = false;
1355 wm_state->sr.cursor = USHRT_MAX;
1356 wm_state->sr.plane = USHRT_MAX;
1357 wm_state->sr.fbc = USHRT_MAX;
1358 }
1359
1360 if (level <= G4X_WM_LEVEL_HPLL) {
1361 wm_state->hpll_en = false;
1362 wm_state->hpll.cursor = USHRT_MAX;
1363 wm_state->hpll.plane = USHRT_MAX;
1364 wm_state->hpll.fbc = USHRT_MAX;
1365 }
1366}
1367
1368static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1369 int level)
1370{
1371 if (level < G4X_WM_LEVEL_SR)
1372 return false;
1373
1374 if (level >= G4X_WM_LEVEL_SR &&
1375 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1376 return false;
1377
1378 if (level >= G4X_WM_LEVEL_HPLL &&
1379 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1380 return false;
1381
1382 return true;
1383}
1384
1385static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1386 struct intel_crtc *crtc)
1387{
1388 struct intel_crtc_state *crtc_state =
1389 intel_atomic_get_new_crtc_state(state, crtc);
1390 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1391 int num_active_planes = hweight8(crtc_state->active_planes &
1392 ~BIT(PLANE_CURSOR));
1393 const struct g4x_pipe_wm *raw;
1394 const struct intel_plane_state *old_plane_state;
1395 const struct intel_plane_state *new_plane_state;
1396 struct intel_plane *plane;
1397 enum plane_id plane_id;
1398 int i, level;
1399 unsigned int dirty = 0;
1400
1401 for_each_oldnew_intel_plane_in_state(state, plane,
1402 old_plane_state,
1403 new_plane_state, i) {
1404 if (new_plane_state->hw.crtc != &crtc->base &&
1405 old_plane_state->hw.crtc != &crtc->base)
1406 continue;
1407
1408 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1409 dirty |= BIT(plane->id);
1410 }
1411
1412 if (!dirty)
1413 return 0;
1414
1415 level = G4X_WM_LEVEL_NORMAL;
1416 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1417 goto out;
1418
1419 raw = &crtc_state->wm.g4x.raw[level];
1420 for_each_plane_id_on_crtc(crtc, plane_id)
1421 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1422
1423 level = G4X_WM_LEVEL_SR;
1424 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1425 goto out;
1426
1427 raw = &crtc_state->wm.g4x.raw[level];
1428 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1429 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1430 wm_state->sr.fbc = raw->fbc;
1431
1432 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1433
1434 level = G4X_WM_LEVEL_HPLL;
1435 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1436 goto out;
1437
1438 raw = &crtc_state->wm.g4x.raw[level];
1439 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1440 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1441 wm_state->hpll.fbc = raw->fbc;
1442
1443 wm_state->hpll_en = wm_state->cxsr;
1444
1445 level++;
1446
1447 out:
1448 if (level == G4X_WM_LEVEL_NORMAL)
1449 return -EINVAL;
1450
1451
1452 g4x_invalidate_wms(crtc, wm_state, level);
1453
1454
1455
1456
1457
1458
1459
1460
1461 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1462
1463 return 0;
1464}
1465
1466static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1467 struct intel_crtc *crtc)
1468{
1469 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1470 struct intel_crtc_state *new_crtc_state =
1471 intel_atomic_get_new_crtc_state(state, crtc);
1472 const struct intel_crtc_state *old_crtc_state =
1473 intel_atomic_get_old_crtc_state(state, crtc);
1474 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1475 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1476 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1477 enum plane_id plane_id;
1478
1479 if (!new_crtc_state->hw.active ||
1480 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1481 *intermediate = *optimal;
1482
1483 intermediate->cxsr = false;
1484 intermediate->hpll_en = false;
1485 goto out;
1486 }
1487
1488 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1489 !new_crtc_state->disable_cxsr;
1490 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1491 !new_crtc_state->disable_cxsr;
1492 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1493
1494 for_each_plane_id_on_crtc(crtc, plane_id) {
1495 intermediate->wm.plane[plane_id] =
1496 max(optimal->wm.plane[plane_id],
1497 active->wm.plane[plane_id]);
1498
1499 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1500 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1501 }
1502
1503 intermediate->sr.plane = max(optimal->sr.plane,
1504 active->sr.plane);
1505 intermediate->sr.cursor = max(optimal->sr.cursor,
1506 active->sr.cursor);
1507 intermediate->sr.fbc = max(optimal->sr.fbc,
1508 active->sr.fbc);
1509
1510 intermediate->hpll.plane = max(optimal->hpll.plane,
1511 active->hpll.plane);
1512 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1513 active->hpll.cursor);
1514 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1515 active->hpll.fbc);
1516
1517 drm_WARN_ON(&dev_priv->drm,
1518 (intermediate->sr.plane >
1519 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1520 intermediate->sr.cursor >
1521 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1522 intermediate->cxsr);
1523 drm_WARN_ON(&dev_priv->drm,
1524 (intermediate->sr.plane >
1525 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1526 intermediate->sr.cursor >
1527 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1528 intermediate->hpll_en);
1529
1530 drm_WARN_ON(&dev_priv->drm,
1531 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1532 intermediate->fbc_en && intermediate->cxsr);
1533 drm_WARN_ON(&dev_priv->drm,
1534 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1535 intermediate->fbc_en && intermediate->hpll_en);
1536
1537out:
1538
1539
1540
1541
1542 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1543 new_crtc_state->wm.need_postvbl_update = true;
1544
1545 return 0;
1546}
1547
1548static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1549 struct g4x_wm_values *wm)
1550{
1551 struct intel_crtc *crtc;
1552 int num_active_pipes = 0;
1553
1554 wm->cxsr = true;
1555 wm->hpll_en = true;
1556 wm->fbc_en = true;
1557
1558 for_each_intel_crtc(&dev_priv->drm, crtc) {
1559 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1560
1561 if (!crtc->active)
1562 continue;
1563
1564 if (!wm_state->cxsr)
1565 wm->cxsr = false;
1566 if (!wm_state->hpll_en)
1567 wm->hpll_en = false;
1568 if (!wm_state->fbc_en)
1569 wm->fbc_en = false;
1570
1571 num_active_pipes++;
1572 }
1573
1574 if (num_active_pipes != 1) {
1575 wm->cxsr = false;
1576 wm->hpll_en = false;
1577 wm->fbc_en = false;
1578 }
1579
1580 for_each_intel_crtc(&dev_priv->drm, crtc) {
1581 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1582 enum pipe pipe = crtc->pipe;
1583
1584 wm->pipe[pipe] = wm_state->wm;
1585 if (crtc->active && wm->cxsr)
1586 wm->sr = wm_state->sr;
1587 if (crtc->active && wm->hpll_en)
1588 wm->hpll = wm_state->hpll;
1589 }
1590}
1591
1592static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1593{
1594 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1595 struct g4x_wm_values new_wm = {};
1596
1597 g4x_merge_wm(dev_priv, &new_wm);
1598
1599 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1600 return;
1601
1602 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1603 _intel_set_memory_cxsr(dev_priv, false);
1604
1605 g4x_write_wm_values(dev_priv, &new_wm);
1606
1607 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1608 _intel_set_memory_cxsr(dev_priv, true);
1609
1610 *old_wm = new_wm;
1611}
1612
1613static void g4x_initial_watermarks(struct intel_atomic_state *state,
1614 struct intel_crtc *crtc)
1615{
1616 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1617 const struct intel_crtc_state *crtc_state =
1618 intel_atomic_get_new_crtc_state(state, crtc);
1619
1620 mutex_lock(&dev_priv->wm.wm_mutex);
1621 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1622 g4x_program_watermarks(dev_priv);
1623 mutex_unlock(&dev_priv->wm.wm_mutex);
1624}
1625
1626static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1627 struct intel_crtc *crtc)
1628{
1629 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1630 const struct intel_crtc_state *crtc_state =
1631 intel_atomic_get_new_crtc_state(state, crtc);
1632
1633 if (!crtc_state->wm.need_postvbl_update)
1634 return;
1635
1636 mutex_lock(&dev_priv->wm.wm_mutex);
1637 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1638 g4x_program_watermarks(dev_priv);
1639 mutex_unlock(&dev_priv->wm.wm_mutex);
1640}
1641
1642
1643static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1644 unsigned int htotal,
1645 unsigned int width,
1646 unsigned int cpp,
1647 unsigned int latency)
1648{
1649 unsigned int ret;
1650
1651 ret = intel_wm_method2(pixel_rate, htotal,
1652 width, cpp, latency);
1653 ret = DIV_ROUND_UP(ret, 64);
1654
1655 return ret;
1656}
1657
1658static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1659{
1660
1661 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1662
1663 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1664
1665 if (IS_CHERRYVIEW(dev_priv)) {
1666 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1667 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1668
1669 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1670 }
1671}
1672
1673static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1674 const struct intel_plane_state *plane_state,
1675 int level)
1676{
1677 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1678 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1679 const struct drm_display_mode *pipe_mode =
1680 &crtc_state->hw.pipe_mode;
1681 unsigned int clock, htotal, cpp, width, wm;
1682
1683 if (dev_priv->wm.pri_latency[level] == 0)
1684 return USHRT_MAX;
1685
1686 if (!intel_wm_plane_visible(crtc_state, plane_state))
1687 return 0;
1688
1689 cpp = plane_state->hw.fb->format->cpp[0];
1690 clock = pipe_mode->crtc_clock;
1691 htotal = pipe_mode->crtc_htotal;
1692 width = crtc_state->pipe_src_w;
1693
1694 if (plane->id == PLANE_CURSOR) {
1695
1696
1697
1698
1699
1700
1701 wm = 63;
1702 } else {
1703 wm = vlv_wm_method2(clock, htotal, width, cpp,
1704 dev_priv->wm.pri_latency[level] * 10);
1705 }
1706
1707 return min_t(unsigned int, wm, USHRT_MAX);
1708}
1709
1710static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1711{
1712 return (active_planes & (BIT(PLANE_SPRITE0) |
1713 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1714}
1715
1716static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1717{
1718 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1719 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1720 const struct g4x_pipe_wm *raw =
1721 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1722 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1723 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1724 int num_active_planes = hweight8(active_planes);
1725 const int fifo_size = 511;
1726 int fifo_extra, fifo_left = fifo_size;
1727 int sprite0_fifo_extra = 0;
1728 unsigned int total_rate;
1729 enum plane_id plane_id;
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739 if (vlv_need_sprite0_fifo_workaround(active_planes))
1740 sprite0_fifo_extra = 1;
1741
1742 total_rate = raw->plane[PLANE_PRIMARY] +
1743 raw->plane[PLANE_SPRITE0] +
1744 raw->plane[PLANE_SPRITE1] +
1745 sprite0_fifo_extra;
1746
1747 if (total_rate > fifo_size)
1748 return -EINVAL;
1749
1750 if (total_rate == 0)
1751 total_rate = 1;
1752
1753 for_each_plane_id_on_crtc(crtc, plane_id) {
1754 unsigned int rate;
1755
1756 if ((active_planes & BIT(plane_id)) == 0) {
1757 fifo_state->plane[plane_id] = 0;
1758 continue;
1759 }
1760
1761 rate = raw->plane[plane_id];
1762 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1763 fifo_left -= fifo_state->plane[plane_id];
1764 }
1765
1766 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1767 fifo_left -= sprite0_fifo_extra;
1768
1769 fifo_state->plane[PLANE_CURSOR] = 63;
1770
1771 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1772
1773
1774 for_each_plane_id_on_crtc(crtc, plane_id) {
1775 int plane_extra;
1776
1777 if (fifo_left == 0)
1778 break;
1779
1780 if ((active_planes & BIT(plane_id)) == 0)
1781 continue;
1782
1783 plane_extra = min(fifo_extra, fifo_left);
1784 fifo_state->plane[plane_id] += plane_extra;
1785 fifo_left -= plane_extra;
1786 }
1787
1788 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1789
1790
1791 if (active_planes == 0) {
1792 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1793 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1794 }
1795
1796 return 0;
1797}
1798
1799
1800static void vlv_invalidate_wms(struct intel_crtc *crtc,
1801 struct vlv_wm_state *wm_state, int level)
1802{
1803 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1804
1805 for (; level < intel_wm_num_levels(dev_priv); level++) {
1806 enum plane_id plane_id;
1807
1808 for_each_plane_id_on_crtc(crtc, plane_id)
1809 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1810
1811 wm_state->sr[level].cursor = USHRT_MAX;
1812 wm_state->sr[level].plane = USHRT_MAX;
1813 }
1814}
1815
1816static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1817{
1818 if (wm > fifo_size)
1819 return USHRT_MAX;
1820 else
1821 return fifo_size - wm;
1822}
1823
1824
1825
1826
1827
1828static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1829 int level, enum plane_id plane_id, u16 value)
1830{
1831 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1832 int num_levels = intel_wm_num_levels(dev_priv);
1833 bool dirty = false;
1834
1835 for (; level < num_levels; level++) {
1836 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1837
1838 dirty |= raw->plane[plane_id] != value;
1839 raw->plane[plane_id] = value;
1840 }
1841
1842 return dirty;
1843}
1844
1845static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1846 const struct intel_plane_state *plane_state)
1847{
1848 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1849 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1850 enum plane_id plane_id = plane->id;
1851 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1852 int level;
1853 bool dirty = false;
1854
1855 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1856 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1857 goto out;
1858 }
1859
1860 for (level = 0; level < num_levels; level++) {
1861 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1862 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1863 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1864
1865 if (wm > max_wm)
1866 break;
1867
1868 dirty |= raw->plane[plane_id] != wm;
1869 raw->plane[plane_id] = wm;
1870 }
1871
1872
1873 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1874
1875out:
1876 if (dirty)
1877 drm_dbg_kms(&dev_priv->drm,
1878 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1879 plane->base.name,
1880 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1881 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1882 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1883
1884 return dirty;
1885}
1886
1887static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1888 enum plane_id plane_id, int level)
1889{
1890 const struct g4x_pipe_wm *raw =
1891 &crtc_state->wm.vlv.raw[level];
1892 const struct vlv_fifo_state *fifo_state =
1893 &crtc_state->wm.vlv.fifo_state;
1894
1895 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1896}
1897
1898static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1899{
1900 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1901 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1902 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1903 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1904}
1905
1906static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1907 struct intel_crtc *crtc)
1908{
1909 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1910 struct intel_crtc_state *crtc_state =
1911 intel_atomic_get_new_crtc_state(state, crtc);
1912 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1913 const struct vlv_fifo_state *fifo_state =
1914 &crtc_state->wm.vlv.fifo_state;
1915 int num_active_planes = hweight8(crtc_state->active_planes &
1916 ~BIT(PLANE_CURSOR));
1917 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1918 const struct intel_plane_state *old_plane_state;
1919 const struct intel_plane_state *new_plane_state;
1920 struct intel_plane *plane;
1921 enum plane_id plane_id;
1922 int level, ret, i;
1923 unsigned int dirty = 0;
1924
1925 for_each_oldnew_intel_plane_in_state(state, plane,
1926 old_plane_state,
1927 new_plane_state, i) {
1928 if (new_plane_state->hw.crtc != &crtc->base &&
1929 old_plane_state->hw.crtc != &crtc->base)
1930 continue;
1931
1932 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1933 dirty |= BIT(plane->id);
1934 }
1935
1936
1937
1938
1939
1940
1941
1942 if (needs_modeset)
1943 crtc_state->fifo_changed = true;
1944
1945 if (!dirty)
1946 return 0;
1947
1948
1949 if (dirty & ~BIT(PLANE_CURSOR)) {
1950 const struct intel_crtc_state *old_crtc_state =
1951 intel_atomic_get_old_crtc_state(state, crtc);
1952 const struct vlv_fifo_state *old_fifo_state =
1953 &old_crtc_state->wm.vlv.fifo_state;
1954
1955 ret = vlv_compute_fifo(crtc_state);
1956 if (ret)
1957 return ret;
1958
1959 if (needs_modeset ||
1960 memcmp(old_fifo_state, fifo_state,
1961 sizeof(*fifo_state)) != 0)
1962 crtc_state->fifo_changed = true;
1963 }
1964
1965
1966 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1967
1968
1969
1970
1971
1972 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1973
1974 for (level = 0; level < wm_state->num_levels; level++) {
1975 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1976 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1977
1978 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1979 break;
1980
1981 for_each_plane_id_on_crtc(crtc, plane_id) {
1982 wm_state->wm[level].plane[plane_id] =
1983 vlv_invert_wm_value(raw->plane[plane_id],
1984 fifo_state->plane[plane_id]);
1985 }
1986
1987 wm_state->sr[level].plane =
1988 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1989 raw->plane[PLANE_SPRITE0],
1990 raw->plane[PLANE_SPRITE1]),
1991 sr_fifo_size);
1992
1993 wm_state->sr[level].cursor =
1994 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1995 63);
1996 }
1997
1998 if (level == 0)
1999 return -EINVAL;
2000
2001
2002 wm_state->num_levels = level;
2003
2004
2005 vlv_invalidate_wms(crtc, wm_state, level);
2006
2007 return 0;
2008}
2009
2010#define VLV_FIFO(plane, value) \
2011 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2012
2013static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2014 struct intel_crtc *crtc)
2015{
2016 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2017 struct intel_uncore *uncore = &dev_priv->uncore;
2018 const struct intel_crtc_state *crtc_state =
2019 intel_atomic_get_new_crtc_state(state, crtc);
2020 const struct vlv_fifo_state *fifo_state =
2021 &crtc_state->wm.vlv.fifo_state;
2022 int sprite0_start, sprite1_start, fifo_size;
2023 u32 dsparb, dsparb2, dsparb3;
2024
2025 if (!crtc_state->fifo_changed)
2026 return;
2027
2028 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2029 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2030 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2031
2032 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2033 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2034
2035 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046 spin_lock(&uncore->lock);
2047
2048 switch (crtc->pipe) {
2049 case PIPE_A:
2050 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2051 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2052
2053 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2054 VLV_FIFO(SPRITEB, 0xff));
2055 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2056 VLV_FIFO(SPRITEB, sprite1_start));
2057
2058 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2059 VLV_FIFO(SPRITEB_HI, 0x1));
2060 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2061 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2062
2063 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2064 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2065 break;
2066 case PIPE_B:
2067 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2068 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2069
2070 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2071 VLV_FIFO(SPRITED, 0xff));
2072 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2073 VLV_FIFO(SPRITED, sprite1_start));
2074
2075 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2076 VLV_FIFO(SPRITED_HI, 0xff));
2077 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2078 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2079
2080 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2081 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2082 break;
2083 case PIPE_C:
2084 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2085 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2086
2087 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2088 VLV_FIFO(SPRITEF, 0xff));
2089 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2090 VLV_FIFO(SPRITEF, sprite1_start));
2091
2092 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2093 VLV_FIFO(SPRITEF_HI, 0xff));
2094 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2095 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2096
2097 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2098 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2099 break;
2100 default:
2101 break;
2102 }
2103
2104 intel_uncore_posting_read_fw(uncore, DSPARB);
2105
2106 spin_unlock(&uncore->lock);
2107}
2108
2109#undef VLV_FIFO
2110
2111static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2112 struct intel_crtc *crtc)
2113{
2114 struct intel_crtc_state *new_crtc_state =
2115 intel_atomic_get_new_crtc_state(state, crtc);
2116 const struct intel_crtc_state *old_crtc_state =
2117 intel_atomic_get_old_crtc_state(state, crtc);
2118 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2119 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2120 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2121 int level;
2122
2123 if (!new_crtc_state->hw.active ||
2124 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2125 *intermediate = *optimal;
2126
2127 intermediate->cxsr = false;
2128 goto out;
2129 }
2130
2131 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2132 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2133 !new_crtc_state->disable_cxsr;
2134
2135 for (level = 0; level < intermediate->num_levels; level++) {
2136 enum plane_id plane_id;
2137
2138 for_each_plane_id_on_crtc(crtc, plane_id) {
2139 intermediate->wm[level].plane[plane_id] =
2140 min(optimal->wm[level].plane[plane_id],
2141 active->wm[level].plane[plane_id]);
2142 }
2143
2144 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2145 active->sr[level].plane);
2146 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2147 active->sr[level].cursor);
2148 }
2149
2150 vlv_invalidate_wms(crtc, intermediate, level);
2151
2152out:
2153
2154
2155
2156
2157 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2158 new_crtc_state->wm.need_postvbl_update = true;
2159
2160 return 0;
2161}
2162
2163static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2164 struct vlv_wm_values *wm)
2165{
2166 struct intel_crtc *crtc;
2167 int num_active_pipes = 0;
2168
2169 wm->level = dev_priv->wm.max_level;
2170 wm->cxsr = true;
2171
2172 for_each_intel_crtc(&dev_priv->drm, crtc) {
2173 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2174
2175 if (!crtc->active)
2176 continue;
2177
2178 if (!wm_state->cxsr)
2179 wm->cxsr = false;
2180
2181 num_active_pipes++;
2182 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2183 }
2184
2185 if (num_active_pipes != 1)
2186 wm->cxsr = false;
2187
2188 if (num_active_pipes > 1)
2189 wm->level = VLV_WM_LEVEL_PM2;
2190
2191 for_each_intel_crtc(&dev_priv->drm, crtc) {
2192 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2193 enum pipe pipe = crtc->pipe;
2194
2195 wm->pipe[pipe] = wm_state->wm[wm->level];
2196 if (crtc->active && wm->cxsr)
2197 wm->sr = wm_state->sr[wm->level];
2198
2199 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2200 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2201 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2202 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2203 }
2204}
2205
2206static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2207{
2208 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2209 struct vlv_wm_values new_wm = {};
2210
2211 vlv_merge_wm(dev_priv, &new_wm);
2212
2213 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2214 return;
2215
2216 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2217 chv_set_memory_dvfs(dev_priv, false);
2218
2219 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2220 chv_set_memory_pm5(dev_priv, false);
2221
2222 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2223 _intel_set_memory_cxsr(dev_priv, false);
2224
2225 vlv_write_wm_values(dev_priv, &new_wm);
2226
2227 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2228 _intel_set_memory_cxsr(dev_priv, true);
2229
2230 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2231 chv_set_memory_pm5(dev_priv, true);
2232
2233 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2234 chv_set_memory_dvfs(dev_priv, true);
2235
2236 *old_wm = new_wm;
2237}
2238
2239static void vlv_initial_watermarks(struct intel_atomic_state *state,
2240 struct intel_crtc *crtc)
2241{
2242 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2243 const struct intel_crtc_state *crtc_state =
2244 intel_atomic_get_new_crtc_state(state, crtc);
2245
2246 mutex_lock(&dev_priv->wm.wm_mutex);
2247 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2248 vlv_program_watermarks(dev_priv);
2249 mutex_unlock(&dev_priv->wm.wm_mutex);
2250}
2251
2252static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2253 struct intel_crtc *crtc)
2254{
2255 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2256 const struct intel_crtc_state *crtc_state =
2257 intel_atomic_get_new_crtc_state(state, crtc);
2258
2259 if (!crtc_state->wm.need_postvbl_update)
2260 return;
2261
2262 mutex_lock(&dev_priv->wm.wm_mutex);
2263 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2264 vlv_program_watermarks(dev_priv);
2265 mutex_unlock(&dev_priv->wm.wm_mutex);
2266}
2267
2268static void i965_update_wm(struct intel_crtc *unused_crtc)
2269{
2270 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2271 struct intel_crtc *crtc;
2272 int srwm = 1;
2273 int cursor_sr = 16;
2274 bool cxsr_enabled;
2275
2276
2277 crtc = single_enabled_crtc(dev_priv);
2278 if (crtc) {
2279
2280 static const int sr_latency_ns = 12000;
2281 const struct drm_display_mode *pipe_mode =
2282 &crtc->config->hw.pipe_mode;
2283 const struct drm_framebuffer *fb =
2284 crtc->base.primary->state->fb;
2285 int clock = pipe_mode->crtc_clock;
2286 int htotal = pipe_mode->crtc_htotal;
2287 int hdisplay = crtc->config->pipe_src_w;
2288 int cpp = fb->format->cpp[0];
2289 int entries;
2290
2291 entries = intel_wm_method2(clock, htotal,
2292 hdisplay, cpp, sr_latency_ns / 100);
2293 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2294 srwm = I965_FIFO_SIZE - entries;
2295 if (srwm < 0)
2296 srwm = 1;
2297 srwm &= 0x1ff;
2298 drm_dbg_kms(&dev_priv->drm,
2299 "self-refresh entries: %d, wm: %d\n",
2300 entries, srwm);
2301
2302 entries = intel_wm_method2(clock, htotal,
2303 crtc->base.cursor->state->crtc_w, 4,
2304 sr_latency_ns / 100);
2305 entries = DIV_ROUND_UP(entries,
2306 i965_cursor_wm_info.cacheline_size) +
2307 i965_cursor_wm_info.guard_size;
2308
2309 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2310 if (cursor_sr > i965_cursor_wm_info.max_wm)
2311 cursor_sr = i965_cursor_wm_info.max_wm;
2312
2313 drm_dbg_kms(&dev_priv->drm,
2314 "self-refresh watermark: display plane %d "
2315 "cursor %d\n", srwm, cursor_sr);
2316
2317 cxsr_enabled = true;
2318 } else {
2319 cxsr_enabled = false;
2320
2321 intel_set_memory_cxsr(dev_priv, false);
2322 }
2323
2324 drm_dbg_kms(&dev_priv->drm,
2325 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2326 srwm);
2327
2328
2329 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2330 FW_WM(8, CURSORB) |
2331 FW_WM(8, PLANEB) |
2332 FW_WM(8, PLANEA));
2333 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2334 FW_WM(8, PLANEC_OLD));
2335
2336 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2337
2338 if (cxsr_enabled)
2339 intel_set_memory_cxsr(dev_priv, true);
2340}
2341
2342#undef FW_WM
2343
2344static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2345{
2346 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2347 const struct intel_watermark_params *wm_info;
2348 u32 fwater_lo;
2349 u32 fwater_hi;
2350 int cwm, srwm = 1;
2351 int fifo_size;
2352 int planea_wm, planeb_wm;
2353 struct intel_crtc *crtc, *enabled = NULL;
2354
2355 if (IS_I945GM(dev_priv))
2356 wm_info = &i945_wm_info;
2357 else if (DISPLAY_VER(dev_priv) != 2)
2358 wm_info = &i915_wm_info;
2359 else
2360 wm_info = &i830_a_wm_info;
2361
2362 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2363 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2364 if (intel_crtc_active(crtc)) {
2365 const struct drm_display_mode *pipe_mode =
2366 &crtc->config->hw.pipe_mode;
2367 const struct drm_framebuffer *fb =
2368 crtc->base.primary->state->fb;
2369 int cpp;
2370
2371 if (DISPLAY_VER(dev_priv) == 2)
2372 cpp = 4;
2373 else
2374 cpp = fb->format->cpp[0];
2375
2376 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2377 wm_info, fifo_size, cpp,
2378 pessimal_latency_ns);
2379 enabled = crtc;
2380 } else {
2381 planea_wm = fifo_size - wm_info->guard_size;
2382 if (planea_wm > (long)wm_info->max_wm)
2383 planea_wm = wm_info->max_wm;
2384 }
2385
2386 if (DISPLAY_VER(dev_priv) == 2)
2387 wm_info = &i830_bc_wm_info;
2388
2389 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2390 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2391 if (intel_crtc_active(crtc)) {
2392 const struct drm_display_mode *pipe_mode =
2393 &crtc->config->hw.pipe_mode;
2394 const struct drm_framebuffer *fb =
2395 crtc->base.primary->state->fb;
2396 int cpp;
2397
2398 if (DISPLAY_VER(dev_priv) == 2)
2399 cpp = 4;
2400 else
2401 cpp = fb->format->cpp[0];
2402
2403 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2404 wm_info, fifo_size, cpp,
2405 pessimal_latency_ns);
2406 if (enabled == NULL)
2407 enabled = crtc;
2408 else
2409 enabled = NULL;
2410 } else {
2411 planeb_wm = fifo_size - wm_info->guard_size;
2412 if (planeb_wm > (long)wm_info->max_wm)
2413 planeb_wm = wm_info->max_wm;
2414 }
2415
2416 drm_dbg_kms(&dev_priv->drm,
2417 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2418
2419 if (IS_I915GM(dev_priv) && enabled) {
2420 struct drm_i915_gem_object *obj;
2421
2422 obj = intel_fb_obj(enabled->base.primary->state->fb);
2423
2424
2425 if (!i915_gem_object_is_tiled(obj))
2426 enabled = NULL;
2427 }
2428
2429
2430
2431
2432 cwm = 2;
2433
2434
2435 intel_set_memory_cxsr(dev_priv, false);
2436
2437
2438 if (HAS_FW_BLC(dev_priv) && enabled) {
2439
2440 static const int sr_latency_ns = 6000;
2441 const struct drm_display_mode *pipe_mode =
2442 &enabled->config->hw.pipe_mode;
2443 const struct drm_framebuffer *fb =
2444 enabled->base.primary->state->fb;
2445 int clock = pipe_mode->crtc_clock;
2446 int htotal = pipe_mode->crtc_htotal;
2447 int hdisplay = enabled->config->pipe_src_w;
2448 int cpp;
2449 int entries;
2450
2451 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2452 cpp = 4;
2453 else
2454 cpp = fb->format->cpp[0];
2455
2456 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2457 sr_latency_ns / 100);
2458 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2459 drm_dbg_kms(&dev_priv->drm,
2460 "self-refresh entries: %d\n", entries);
2461 srwm = wm_info->fifo_size - entries;
2462 if (srwm < 0)
2463 srwm = 1;
2464
2465 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2466 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2467 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2468 else
2469 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2470 }
2471
2472 drm_dbg_kms(&dev_priv->drm,
2473 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2474 planea_wm, planeb_wm, cwm, srwm);
2475
2476 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2477 fwater_hi = (cwm & 0x1f);
2478
2479
2480 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2481 fwater_hi = fwater_hi | (1 << 8);
2482
2483 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2484 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2485
2486 if (enabled)
2487 intel_set_memory_cxsr(dev_priv, true);
2488}
2489
2490static void i845_update_wm(struct intel_crtc *unused_crtc)
2491{
2492 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2493 struct intel_crtc *crtc;
2494 const struct drm_display_mode *pipe_mode;
2495 u32 fwater_lo;
2496 int planea_wm;
2497
2498 crtc = single_enabled_crtc(dev_priv);
2499 if (crtc == NULL)
2500 return;
2501
2502 pipe_mode = &crtc->config->hw.pipe_mode;
2503 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2504 &i845_wm_info,
2505 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2506 4, pessimal_latency_ns);
2507 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2508 fwater_lo |= (3<<8) | planea_wm;
2509
2510 drm_dbg_kms(&dev_priv->drm,
2511 "Setting FIFO watermarks - A: %d\n", planea_wm);
2512
2513 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2514}
2515
2516
2517static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2518 unsigned int cpp,
2519 unsigned int latency)
2520{
2521 unsigned int ret;
2522
2523 ret = intel_wm_method1(pixel_rate, cpp, latency);
2524 ret = DIV_ROUND_UP(ret, 64) + 2;
2525
2526 return ret;
2527}
2528
2529
2530static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2531 unsigned int htotal,
2532 unsigned int width,
2533 unsigned int cpp,
2534 unsigned int latency)
2535{
2536 unsigned int ret;
2537
2538 ret = intel_wm_method2(pixel_rate, htotal,
2539 width, cpp, latency);
2540 ret = DIV_ROUND_UP(ret, 64) + 2;
2541
2542 return ret;
2543}
2544
2545static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2546{
2547
2548
2549
2550
2551
2552
2553 if (WARN_ON(!cpp))
2554 return 0;
2555 if (WARN_ON(!horiz_pixels))
2556 return 0;
2557
2558 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2559}
2560
2561struct ilk_wm_maximums {
2562 u16 pri;
2563 u16 spr;
2564 u16 cur;
2565 u16 fbc;
2566};
2567
2568
2569
2570
2571
2572static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2573 const struct intel_plane_state *plane_state,
2574 u32 mem_value, bool is_lp)
2575{
2576 u32 method1, method2;
2577 int cpp;
2578
2579 if (mem_value == 0)
2580 return U32_MAX;
2581
2582 if (!intel_wm_plane_visible(crtc_state, plane_state))
2583 return 0;
2584
2585 cpp = plane_state->hw.fb->format->cpp[0];
2586
2587 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2588
2589 if (!is_lp)
2590 return method1;
2591
2592 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2593 crtc_state->hw.pipe_mode.crtc_htotal,
2594 drm_rect_width(&plane_state->uapi.dst),
2595 cpp, mem_value);
2596
2597 return min(method1, method2);
2598}
2599
2600
2601
2602
2603
2604static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2605 const struct intel_plane_state *plane_state,
2606 u32 mem_value)
2607{
2608 u32 method1, method2;
2609 int cpp;
2610
2611 if (mem_value == 0)
2612 return U32_MAX;
2613
2614 if (!intel_wm_plane_visible(crtc_state, plane_state))
2615 return 0;
2616
2617 cpp = plane_state->hw.fb->format->cpp[0];
2618
2619 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2620 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2621 crtc_state->hw.pipe_mode.crtc_htotal,
2622 drm_rect_width(&plane_state->uapi.dst),
2623 cpp, mem_value);
2624 return min(method1, method2);
2625}
2626
2627
2628
2629
2630
2631static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2632 const struct intel_plane_state *plane_state,
2633 u32 mem_value)
2634{
2635 int cpp;
2636
2637 if (mem_value == 0)
2638 return U32_MAX;
2639
2640 if (!intel_wm_plane_visible(crtc_state, plane_state))
2641 return 0;
2642
2643 cpp = plane_state->hw.fb->format->cpp[0];
2644
2645 return ilk_wm_method2(crtc_state->pixel_rate,
2646 crtc_state->hw.pipe_mode.crtc_htotal,
2647 drm_rect_width(&plane_state->uapi.dst),
2648 cpp, mem_value);
2649}
2650
2651
2652static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2653 const struct intel_plane_state *plane_state,
2654 u32 pri_val)
2655{
2656 int cpp;
2657
2658 if (!intel_wm_plane_visible(crtc_state, plane_state))
2659 return 0;
2660
2661 cpp = plane_state->hw.fb->format->cpp[0];
2662
2663 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2664 cpp);
2665}
2666
2667static unsigned int
2668ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2669{
2670 if (DISPLAY_VER(dev_priv) >= 8)
2671 return 3072;
2672 else if (DISPLAY_VER(dev_priv) >= 7)
2673 return 768;
2674 else
2675 return 512;
2676}
2677
2678static unsigned int
2679ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2680 int level, bool is_sprite)
2681{
2682 if (DISPLAY_VER(dev_priv) >= 8)
2683
2684 return level == 0 ? 255 : 2047;
2685 else if (DISPLAY_VER(dev_priv) >= 7)
2686
2687 return level == 0 ? 127 : 1023;
2688 else if (!is_sprite)
2689
2690 return level == 0 ? 127 : 511;
2691 else
2692
2693 return level == 0 ? 63 : 255;
2694}
2695
2696static unsigned int
2697ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2698{
2699 if (DISPLAY_VER(dev_priv) >= 7)
2700 return level == 0 ? 63 : 255;
2701 else
2702 return level == 0 ? 31 : 63;
2703}
2704
2705static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2706{
2707 if (DISPLAY_VER(dev_priv) >= 8)
2708 return 31;
2709 else
2710 return 15;
2711}
2712
2713
2714static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2715 int level,
2716 const struct intel_wm_config *config,
2717 enum intel_ddb_partitioning ddb_partitioning,
2718 bool is_sprite)
2719{
2720 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2721
2722
2723 if (is_sprite && !config->sprites_enabled)
2724 return 0;
2725
2726
2727 if (level == 0 || config->num_pipes_active > 1) {
2728 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2729
2730
2731
2732
2733
2734
2735 if (DISPLAY_VER(dev_priv) <= 6)
2736 fifo_size /= 2;
2737 }
2738
2739 if (config->sprites_enabled) {
2740
2741 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2742 if (is_sprite)
2743 fifo_size *= 5;
2744 fifo_size /= 6;
2745 } else {
2746 fifo_size /= 2;
2747 }
2748 }
2749
2750
2751 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2752}
2753
2754
2755static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2756 int level,
2757 const struct intel_wm_config *config)
2758{
2759
2760 if (level > 0 && config->num_pipes_active > 1)
2761 return 64;
2762
2763
2764 return ilk_cursor_wm_reg_max(dev_priv, level);
2765}
2766
2767static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2768 int level,
2769 const struct intel_wm_config *config,
2770 enum intel_ddb_partitioning ddb_partitioning,
2771 struct ilk_wm_maximums *max)
2772{
2773 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2774 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2775 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2776 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2777}
2778
2779static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2780 int level,
2781 struct ilk_wm_maximums *max)
2782{
2783 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2784 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2785 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2786 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2787}
2788
2789static bool ilk_validate_wm_level(int level,
2790 const struct ilk_wm_maximums *max,
2791 struct intel_wm_level *result)
2792{
2793 bool ret;
2794
2795
2796 if (!result->enable)
2797 return false;
2798
2799 result->enable = result->pri_val <= max->pri &&
2800 result->spr_val <= max->spr &&
2801 result->cur_val <= max->cur;
2802
2803 ret = result->enable;
2804
2805
2806
2807
2808
2809
2810 if (level == 0 && !result->enable) {
2811 if (result->pri_val > max->pri)
2812 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2813 level, result->pri_val, max->pri);
2814 if (result->spr_val > max->spr)
2815 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2816 level, result->spr_val, max->spr);
2817 if (result->cur_val > max->cur)
2818 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2819 level, result->cur_val, max->cur);
2820
2821 result->pri_val = min_t(u32, result->pri_val, max->pri);
2822 result->spr_val = min_t(u32, result->spr_val, max->spr);
2823 result->cur_val = min_t(u32, result->cur_val, max->cur);
2824 result->enable = true;
2825 }
2826
2827 return ret;
2828}
2829
2830static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2831 const struct intel_crtc *crtc,
2832 int level,
2833 struct intel_crtc_state *crtc_state,
2834 const struct intel_plane_state *pristate,
2835 const struct intel_plane_state *sprstate,
2836 const struct intel_plane_state *curstate,
2837 struct intel_wm_level *result)
2838{
2839 u16 pri_latency = dev_priv->wm.pri_latency[level];
2840 u16 spr_latency = dev_priv->wm.spr_latency[level];
2841 u16 cur_latency = dev_priv->wm.cur_latency[level];
2842
2843
2844 if (level > 0) {
2845 pri_latency *= 5;
2846 spr_latency *= 5;
2847 cur_latency *= 5;
2848 }
2849
2850 if (pristate) {
2851 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2852 pri_latency, level);
2853 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2854 }
2855
2856 if (sprstate)
2857 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2858
2859 if (curstate)
2860 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2861
2862 result->enable = true;
2863}
2864
2865static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2866 u16 wm[8])
2867{
2868 struct intel_uncore *uncore = &dev_priv->uncore;
2869
2870 if (DISPLAY_VER(dev_priv) >= 9) {
2871 u32 val;
2872 int ret, i;
2873 int level, max_level = ilk_wm_max_level(dev_priv);
2874
2875
2876 val = 0;
2877 ret = sandybridge_pcode_read(dev_priv,
2878 GEN9_PCODE_READ_MEM_LATENCY,
2879 &val, NULL);
2880
2881 if (ret) {
2882 drm_err(&dev_priv->drm,
2883 "SKL Mailbox read error = %d\n", ret);
2884 return;
2885 }
2886
2887 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2888 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2889 GEN9_MEM_LATENCY_LEVEL_MASK;
2890 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2891 GEN9_MEM_LATENCY_LEVEL_MASK;
2892 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2893 GEN9_MEM_LATENCY_LEVEL_MASK;
2894
2895
2896 val = 1;
2897 ret = sandybridge_pcode_read(dev_priv,
2898 GEN9_PCODE_READ_MEM_LATENCY,
2899 &val, NULL);
2900 if (ret) {
2901 drm_err(&dev_priv->drm,
2902 "SKL Mailbox read error = %d\n", ret);
2903 return;
2904 }
2905
2906 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2907 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2908 GEN9_MEM_LATENCY_LEVEL_MASK;
2909 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2910 GEN9_MEM_LATENCY_LEVEL_MASK;
2911 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2912 GEN9_MEM_LATENCY_LEVEL_MASK;
2913
2914
2915
2916
2917
2918
2919 for (level = 1; level <= max_level; level++) {
2920 if (wm[level] == 0) {
2921 for (i = level + 1; i <= max_level; i++)
2922 wm[i] = 0;
2923
2924 max_level = level - 1;
2925
2926 break;
2927 }
2928 }
2929
2930
2931
2932
2933
2934
2935
2936
2937 if (wm[0] == 0) {
2938 u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
2939
2940 for (level = 0; level <= max_level; level++)
2941 wm[level] += adjust;
2942 }
2943
2944
2945
2946
2947
2948
2949
2950 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2951 wm[0] += 1;
2952 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2953 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2954
2955 wm[0] = (sskpd >> 56) & 0xFF;
2956 if (wm[0] == 0)
2957 wm[0] = sskpd & 0xF;
2958 wm[1] = (sskpd >> 4) & 0xFF;
2959 wm[2] = (sskpd >> 12) & 0xFF;
2960 wm[3] = (sskpd >> 20) & 0x1FF;
2961 wm[4] = (sskpd >> 32) & 0x1FF;
2962 } else if (DISPLAY_VER(dev_priv) >= 6) {
2963 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2964
2965 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2966 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2967 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2968 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2969 } else if (DISPLAY_VER(dev_priv) >= 5) {
2970 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2971
2972
2973 wm[0] = 7;
2974 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2975 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2976 } else {
2977 MISSING_CASE(INTEL_DEVID(dev_priv));
2978 }
2979}
2980
2981static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2982 u16 wm[5])
2983{
2984
2985 if (DISPLAY_VER(dev_priv) == 5)
2986 wm[0] = 13;
2987}
2988
2989static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2990 u16 wm[5])
2991{
2992
2993 if (DISPLAY_VER(dev_priv) == 5)
2994 wm[0] = 13;
2995}
2996
2997int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2998{
2999
3000 if (HAS_HW_SAGV_WM(dev_priv))
3001 return 5;
3002 else if (DISPLAY_VER(dev_priv) >= 9)
3003 return 7;
3004 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3005 return 4;
3006 else if (DISPLAY_VER(dev_priv) >= 6)
3007 return 3;
3008 else
3009 return 2;
3010}
3011
3012static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3013 const char *name,
3014 const u16 wm[])
3015{
3016 int level, max_level = ilk_wm_max_level(dev_priv);
3017
3018 for (level = 0; level <= max_level; level++) {
3019 unsigned int latency = wm[level];
3020
3021 if (latency == 0) {
3022 drm_dbg_kms(&dev_priv->drm,
3023 "%s WM%d latency not provided\n",
3024 name, level);
3025 continue;
3026 }
3027
3028
3029
3030
3031
3032 if (DISPLAY_VER(dev_priv) >= 9)
3033 latency *= 10;
3034 else if (level > 0)
3035 latency *= 5;
3036
3037 drm_dbg_kms(&dev_priv->drm,
3038 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3039 wm[level], latency / 10, latency % 10);
3040 }
3041}
3042
3043static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3044 u16 wm[5], u16 min)
3045{
3046 int level, max_level = ilk_wm_max_level(dev_priv);
3047
3048 if (wm[0] >= min)
3049 return false;
3050
3051 wm[0] = max(wm[0], min);
3052 for (level = 1; level <= max_level; level++)
3053 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3054
3055 return true;
3056}
3057
3058static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3059{
3060 bool changed;
3061
3062
3063
3064
3065
3066 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3067 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3068 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3069
3070 if (!changed)
3071 return;
3072
3073 drm_dbg_kms(&dev_priv->drm,
3074 "WM latency values increased to avoid potential underruns\n");
3075 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3076 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3077 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3078}
3079
3080static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3081{
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093 if (dev_priv->wm.pri_latency[3] == 0 &&
3094 dev_priv->wm.spr_latency[3] == 0 &&
3095 dev_priv->wm.cur_latency[3] == 0)
3096 return;
3097
3098 dev_priv->wm.pri_latency[3] = 0;
3099 dev_priv->wm.spr_latency[3] = 0;
3100 dev_priv->wm.cur_latency[3] = 0;
3101
3102 drm_dbg_kms(&dev_priv->drm,
3103 "LP3 watermarks disabled due to potential for lost interrupts\n");
3104 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3105 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3106 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3107}
3108
3109static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3110{
3111 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3112
3113 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3114 sizeof(dev_priv->wm.pri_latency));
3115 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3116 sizeof(dev_priv->wm.pri_latency));
3117
3118 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3119 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3120
3121 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3122 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3123 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3124
3125 if (DISPLAY_VER(dev_priv) == 6) {
3126 snb_wm_latency_quirk(dev_priv);
3127 snb_wm_lp3_irq_quirk(dev_priv);
3128 }
3129}
3130
3131static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3132{
3133 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3134 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3135}
3136
3137static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3138 struct intel_pipe_wm *pipe_wm)
3139{
3140
3141 const struct intel_wm_config config = {
3142 .num_pipes_active = 1,
3143 .sprites_enabled = pipe_wm->sprites_enabled,
3144 .sprites_scaled = pipe_wm->sprites_scaled,
3145 };
3146 struct ilk_wm_maximums max;
3147
3148
3149 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3150
3151
3152 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3153 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3154 return false;
3155 }
3156
3157 return true;
3158}
3159
3160
3161static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3162 struct intel_crtc *crtc)
3163{
3164 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3165 struct intel_crtc_state *crtc_state =
3166 intel_atomic_get_new_crtc_state(state, crtc);
3167 struct intel_pipe_wm *pipe_wm;
3168 struct intel_plane *plane;
3169 const struct intel_plane_state *plane_state;
3170 const struct intel_plane_state *pristate = NULL;
3171 const struct intel_plane_state *sprstate = NULL;
3172 const struct intel_plane_state *curstate = NULL;
3173 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3174 struct ilk_wm_maximums max;
3175
3176 pipe_wm = &crtc_state->wm.ilk.optimal;
3177
3178 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3179 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3180 pristate = plane_state;
3181 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3182 sprstate = plane_state;
3183 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3184 curstate = plane_state;
3185 }
3186
3187 pipe_wm->pipe_enabled = crtc_state->hw.active;
3188 if (sprstate) {
3189 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3190 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3191 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3192 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3193 }
3194
3195 usable_level = max_level;
3196
3197
3198 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3199 usable_level = 1;
3200
3201
3202 if (pipe_wm->sprites_scaled)
3203 usable_level = 0;
3204
3205 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3206 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3207 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3208
3209 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3210 return -EINVAL;
3211
3212 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3213
3214 for (level = 1; level <= usable_level; level++) {
3215 struct intel_wm_level *wm = &pipe_wm->wm[level];
3216
3217 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3218 pristate, sprstate, curstate, wm);
3219
3220
3221
3222
3223
3224
3225 if (!ilk_validate_wm_level(level, &max, wm)) {
3226 memset(wm, 0, sizeof(*wm));
3227 break;
3228 }
3229 }
3230
3231 return 0;
3232}
3233
3234
3235
3236
3237
3238
3239static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3240 struct intel_crtc *crtc)
3241{
3242 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3243 struct intel_crtc_state *new_crtc_state =
3244 intel_atomic_get_new_crtc_state(state, crtc);
3245 const struct intel_crtc_state *old_crtc_state =
3246 intel_atomic_get_old_crtc_state(state, crtc);
3247 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3248 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3249 int level, max_level = ilk_wm_max_level(dev_priv);
3250
3251
3252
3253
3254
3255
3256 *a = new_crtc_state->wm.ilk.optimal;
3257 if (!new_crtc_state->hw.active ||
3258 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3259 state->skip_intermediate_wm)
3260 return 0;
3261
3262 a->pipe_enabled |= b->pipe_enabled;
3263 a->sprites_enabled |= b->sprites_enabled;
3264 a->sprites_scaled |= b->sprites_scaled;
3265
3266 for (level = 0; level <= max_level; level++) {
3267 struct intel_wm_level *a_wm = &a->wm[level];
3268 const struct intel_wm_level *b_wm = &b->wm[level];
3269
3270 a_wm->enable &= b_wm->enable;
3271 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3272 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3273 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3274 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3275 }
3276
3277
3278
3279
3280
3281
3282
3283 if (!ilk_validate_pipe_wm(dev_priv, a))
3284 return -EINVAL;
3285
3286
3287
3288
3289
3290 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3291 new_crtc_state->wm.need_postvbl_update = true;
3292
3293 return 0;
3294}
3295
3296
3297
3298
3299static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3300 int level,
3301 struct intel_wm_level *ret_wm)
3302{
3303 const struct intel_crtc *crtc;
3304
3305 ret_wm->enable = true;
3306
3307 for_each_intel_crtc(&dev_priv->drm, crtc) {
3308 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3309 const struct intel_wm_level *wm = &active->wm[level];
3310
3311 if (!active->pipe_enabled)
3312 continue;
3313
3314
3315
3316
3317
3318
3319 if (!wm->enable)
3320 ret_wm->enable = false;
3321
3322 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3323 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3324 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3325 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3326 }
3327}
3328
3329
3330
3331
3332static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3333 const struct intel_wm_config *config,
3334 const struct ilk_wm_maximums *max,
3335 struct intel_pipe_wm *merged)
3336{
3337 int level, max_level = ilk_wm_max_level(dev_priv);
3338 int last_enabled_level = max_level;
3339
3340
3341 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3342 config->num_pipes_active > 1)
3343 last_enabled_level = 0;
3344
3345
3346 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3347
3348
3349 for (level = 1; level <= max_level; level++) {
3350 struct intel_wm_level *wm = &merged->wm[level];
3351
3352 ilk_merge_wm_level(dev_priv, level, wm);
3353
3354 if (level > last_enabled_level)
3355 wm->enable = false;
3356 else if (!ilk_validate_wm_level(level, max, wm))
3357
3358 last_enabled_level = level - 1;
3359
3360
3361
3362
3363
3364 if (wm->fbc_val > max->fbc) {
3365 if (wm->enable)
3366 merged->fbc_wm_enabled = false;
3367 wm->fbc_val = 0;
3368 }
3369 }
3370
3371
3372
3373
3374
3375
3376
3377 if (DISPLAY_VER(dev_priv) == 5 && !merged->fbc_wm_enabled &&
3378 intel_fbc_is_active(dev_priv)) {
3379 for (level = 2; level <= max_level; level++) {
3380 struct intel_wm_level *wm = &merged->wm[level];
3381
3382 wm->enable = false;
3383 }
3384 }
3385}
3386
3387static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3388{
3389
3390 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3391}
3392
3393
3394static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3395 int level)
3396{
3397 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3398 return 2 * level;
3399 else
3400 return dev_priv->wm.pri_latency[level];
3401}
3402
3403static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3404 const struct intel_pipe_wm *merged,
3405 enum intel_ddb_partitioning partitioning,
3406 struct ilk_wm_values *results)
3407{
3408 struct intel_crtc *crtc;
3409 int level, wm_lp;
3410
3411 results->enable_fbc_wm = merged->fbc_wm_enabled;
3412 results->partitioning = partitioning;
3413
3414
3415 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3416 const struct intel_wm_level *r;
3417
3418 level = ilk_wm_lp_to_level(wm_lp, merged);
3419
3420 r = &merged->wm[level];
3421
3422
3423
3424
3425
3426 results->wm_lp[wm_lp - 1] =
3427 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3428 (r->pri_val << WM1_LP_SR_SHIFT) |
3429 r->cur_val;
3430
3431 if (r->enable)
3432 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3433
3434 if (DISPLAY_VER(dev_priv) >= 8)
3435 results->wm_lp[wm_lp - 1] |=
3436 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3437 else
3438 results->wm_lp[wm_lp - 1] |=
3439 r->fbc_val << WM1_LP_FBC_SHIFT;
3440
3441
3442
3443
3444
3445 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3446 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3447 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3448 } else
3449 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3450 }
3451
3452
3453 for_each_intel_crtc(&dev_priv->drm, crtc) {
3454 enum pipe pipe = crtc->pipe;
3455 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3456 const struct intel_wm_level *r = &pipe_wm->wm[0];
3457
3458 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3459 continue;
3460
3461 results->wm_pipe[pipe] =
3462 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3463 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3464 r->cur_val;
3465 }
3466}
3467
3468
3469
3470static struct intel_pipe_wm *
3471ilk_find_best_result(struct drm_i915_private *dev_priv,
3472 struct intel_pipe_wm *r1,
3473 struct intel_pipe_wm *r2)
3474{
3475 int level, max_level = ilk_wm_max_level(dev_priv);
3476 int level1 = 0, level2 = 0;
3477
3478 for (level = 1; level <= max_level; level++) {
3479 if (r1->wm[level].enable)
3480 level1 = level;
3481 if (r2->wm[level].enable)
3482 level2 = level;
3483 }
3484
3485 if (level1 == level2) {
3486 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3487 return r2;
3488 else
3489 return r1;
3490 } else if (level1 > level2) {
3491 return r1;
3492 } else {
3493 return r2;
3494 }
3495}
3496
3497
3498#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3499#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3500#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3501#define WM_DIRTY_FBC (1 << 24)
3502#define WM_DIRTY_DDB (1 << 25)
3503
3504static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3505 const struct ilk_wm_values *old,
3506 const struct ilk_wm_values *new)
3507{
3508 unsigned int dirty = 0;
3509 enum pipe pipe;
3510 int wm_lp;
3511
3512 for_each_pipe(dev_priv, pipe) {
3513 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3514 dirty |= WM_DIRTY_PIPE(pipe);
3515
3516 dirty |= WM_DIRTY_LP_ALL;
3517 }
3518 }
3519
3520 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3521 dirty |= WM_DIRTY_FBC;
3522
3523 dirty |= WM_DIRTY_LP_ALL;
3524 }
3525
3526 if (old->partitioning != new->partitioning) {
3527 dirty |= WM_DIRTY_DDB;
3528
3529 dirty |= WM_DIRTY_LP_ALL;
3530 }
3531
3532
3533 if (dirty & WM_DIRTY_LP_ALL)
3534 return dirty;
3535
3536
3537 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3538 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3539 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3540 break;
3541 }
3542
3543
3544 for (; wm_lp <= 3; wm_lp++)
3545 dirty |= WM_DIRTY_LP(wm_lp);
3546
3547 return dirty;
3548}
3549
3550static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3551 unsigned int dirty)
3552{
3553 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3554 bool changed = false;
3555
3556 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3557 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3558 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3559 changed = true;
3560 }
3561 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3562 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3563 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3564 changed = true;
3565 }
3566 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3567 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3568 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3569 changed = true;
3570 }
3571
3572
3573
3574
3575
3576
3577 return changed;
3578}
3579
3580
3581
3582
3583
3584static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3585 struct ilk_wm_values *results)
3586{
3587 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3588 unsigned int dirty;
3589 u32 val;
3590
3591 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3592 if (!dirty)
3593 return;
3594
3595 _ilk_disable_lp_wm(dev_priv, dirty);
3596
3597 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3598 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3599 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3600 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3601 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3602 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3603
3604 if (dirty & WM_DIRTY_DDB) {
3605 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3606 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3607 if (results->partitioning == INTEL_DDB_PART_1_2)
3608 val &= ~WM_MISC_DATA_PARTITION_5_6;
3609 else
3610 val |= WM_MISC_DATA_PARTITION_5_6;
3611 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3612 } else {
3613 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3614 if (results->partitioning == INTEL_DDB_PART_1_2)
3615 val &= ~DISP_DATA_PARTITION_5_6;
3616 else
3617 val |= DISP_DATA_PARTITION_5_6;
3618 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3619 }
3620 }
3621
3622 if (dirty & WM_DIRTY_FBC) {
3623 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3624 if (results->enable_fbc_wm)
3625 val &= ~DISP_FBC_WM_DIS;
3626 else
3627 val |= DISP_FBC_WM_DIS;
3628 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3629 }
3630
3631 if (dirty & WM_DIRTY_LP(1) &&
3632 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3633 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3634
3635 if (DISPLAY_VER(dev_priv) >= 7) {
3636 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3637 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3638 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3639 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3640 }
3641
3642 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3643 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3644 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3645 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3646 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3647 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3648
3649 dev_priv->wm.hw = *results;
3650}
3651
3652bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3653{
3654 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3655}
3656
3657u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3658{
3659 u8 enabled_slices = 0;
3660 enum dbuf_slice slice;
3661
3662 for_each_dbuf_slice(dev_priv, slice) {
3663 if (intel_uncore_read(&dev_priv->uncore,
3664 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3665 enabled_slices |= BIT(slice);
3666 }
3667
3668 return enabled_slices;
3669}
3670
3671
3672
3673
3674
3675static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3676{
3677 return DISPLAY_VER(dev_priv) == 9;
3678}
3679
3680static bool
3681intel_has_sagv(struct drm_i915_private *dev_priv)
3682{
3683 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3684 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3685}
3686
3687static void
3688skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3689{
3690 if (DISPLAY_VER(dev_priv) >= 12) {
3691 u32 val = 0;
3692 int ret;
3693
3694 ret = sandybridge_pcode_read(dev_priv,
3695 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3696 &val, NULL);
3697 if (!ret) {
3698 dev_priv->sagv_block_time_us = val;
3699 return;
3700 }
3701
3702 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3703 } else if (DISPLAY_VER(dev_priv) == 11) {
3704 dev_priv->sagv_block_time_us = 10;
3705 return;
3706 } else if (DISPLAY_VER(dev_priv) == 10) {
3707 dev_priv->sagv_block_time_us = 20;
3708 return;
3709 } else if (DISPLAY_VER(dev_priv) == 9) {
3710 dev_priv->sagv_block_time_us = 30;
3711 return;
3712 } else {
3713 MISSING_CASE(DISPLAY_VER(dev_priv));
3714 }
3715
3716
3717 dev_priv->sagv_block_time_us = -1;
3718}
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731static int
3732intel_enable_sagv(struct drm_i915_private *dev_priv)
3733{
3734 int ret;
3735
3736 if (!intel_has_sagv(dev_priv))
3737 return 0;
3738
3739 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3740 return 0;
3741
3742 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3743 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3744 GEN9_SAGV_ENABLE);
3745
3746
3747
3748
3749
3750
3751
3752 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3753 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3754 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3755 return 0;
3756 } else if (ret < 0) {
3757 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3758 return ret;
3759 }
3760
3761 dev_priv->sagv_status = I915_SAGV_ENABLED;
3762 return 0;
3763}
3764
3765static int
3766intel_disable_sagv(struct drm_i915_private *dev_priv)
3767{
3768 int ret;
3769
3770 if (!intel_has_sagv(dev_priv))
3771 return 0;
3772
3773 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3774 return 0;
3775
3776 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3777
3778 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3779 GEN9_SAGV_DISABLE,
3780 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3781 1);
3782
3783
3784
3785
3786 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3787 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3788 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3789 return 0;
3790 } else if (ret < 0) {
3791 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3792 return ret;
3793 }
3794
3795 dev_priv->sagv_status = I915_SAGV_DISABLED;
3796 return 0;
3797}
3798
3799void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3800{
3801 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3802 const struct intel_bw_state *new_bw_state;
3803 const struct intel_bw_state *old_bw_state;
3804 u32 new_mask = 0;
3805
3806
3807
3808
3809
3810
3811
3812
3813 if (!intel_has_sagv(dev_priv))
3814 return;
3815
3816 new_bw_state = intel_atomic_get_new_bw_state(state);
3817 if (!new_bw_state)
3818 return;
3819
3820 if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3821 intel_disable_sagv(dev_priv);
3822 return;
3823 }
3824
3825 old_bw_state = intel_atomic_get_old_bw_state(state);
3826
3827
3828
3829 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3830 return;
3831
3832 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3833
3834
3835
3836
3837
3838 if (!new_mask)
3839 return;
3840
3841
3842
3843
3844
3845
3846
3847 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3848}
3849
3850void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3851{
3852 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3853 const struct intel_bw_state *new_bw_state;
3854 const struct intel_bw_state *old_bw_state;
3855 u32 new_mask = 0;
3856
3857
3858
3859
3860
3861
3862
3863
3864 if (!intel_has_sagv(dev_priv))
3865 return;
3866
3867 new_bw_state = intel_atomic_get_new_bw_state(state);
3868 if (!new_bw_state)
3869 return;
3870
3871 if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3872 intel_enable_sagv(dev_priv);
3873 return;
3874 }
3875
3876 old_bw_state = intel_atomic_get_old_bw_state(state);
3877
3878
3879
3880 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3881 return;
3882
3883 new_mask = new_bw_state->qgv_points_mask;
3884
3885
3886
3887
3888
3889
3890
3891 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3892}
3893
3894static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3895{
3896 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3897 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3898 enum plane_id plane_id;
3899 int max_level = INT_MAX;
3900
3901 if (!intel_has_sagv(dev_priv))
3902 return false;
3903
3904 if (!crtc_state->hw.active)
3905 return true;
3906
3907 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3908 return false;
3909
3910 for_each_plane_id_on_crtc(crtc, plane_id) {
3911 const struct skl_plane_wm *wm =
3912 &crtc_state->wm.skl.optimal.planes[plane_id];
3913 int level;
3914
3915
3916 if (!wm->wm[0].enable)
3917 continue;
3918
3919
3920 for (level = ilk_wm_max_level(dev_priv);
3921 !wm->wm[level].enable; --level)
3922 { }
3923
3924
3925 max_level = min(level, max_level);
3926 }
3927
3928
3929 if (max_level == INT_MAX)
3930 return true;
3931
3932 for_each_plane_id_on_crtc(crtc, plane_id) {
3933 const struct skl_plane_wm *wm =
3934 &crtc_state->wm.skl.optimal.planes[plane_id];
3935
3936
3937
3938
3939
3940 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3941 return false;
3942 }
3943
3944 return true;
3945}
3946
3947static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3948{
3949 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3950 enum plane_id plane_id;
3951
3952 if (!crtc_state->hw.active)
3953 return true;
3954
3955 for_each_plane_id_on_crtc(crtc, plane_id) {
3956 const struct skl_plane_wm *wm =
3957 &crtc_state->wm.skl.optimal.planes[plane_id];
3958
3959 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3960 return false;
3961 }
3962
3963 return true;
3964}
3965
3966static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3967{
3968 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3969 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3970
3971 if (DISPLAY_VER(dev_priv) >= 12)
3972 return tgl_crtc_can_enable_sagv(crtc_state);
3973 else
3974 return skl_crtc_can_enable_sagv(crtc_state);
3975}
3976
3977bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3978 const struct intel_bw_state *bw_state)
3979{
3980 if (DISPLAY_VER(dev_priv) < 11 &&
3981 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3982 return false;
3983
3984 return bw_state->pipe_sagv_reject == 0;
3985}
3986
3987static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3988{
3989 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3990 int ret;
3991 struct intel_crtc *crtc;
3992 struct intel_crtc_state *new_crtc_state;
3993 struct intel_bw_state *new_bw_state = NULL;
3994 const struct intel_bw_state *old_bw_state = NULL;
3995 int i;
3996
3997 for_each_new_intel_crtc_in_state(state, crtc,
3998 new_crtc_state, i) {
3999 new_bw_state = intel_atomic_get_bw_state(state);
4000 if (IS_ERR(new_bw_state))
4001 return PTR_ERR(new_bw_state);
4002
4003 old_bw_state = intel_atomic_get_old_bw_state(state);
4004
4005 if (intel_crtc_can_enable_sagv(new_crtc_state))
4006 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4007 else
4008 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4009 }
4010
4011 if (!new_bw_state)
4012 return 0;
4013
4014 new_bw_state->active_pipes =
4015 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4016
4017 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4018 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4019 if (ret)
4020 return ret;
4021 }
4022
4023 for_each_new_intel_crtc_in_state(state, crtc,
4024 new_crtc_state, i) {
4025 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4026
4027
4028
4029
4030
4031
4032
4033 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4034 DISPLAY_VER(dev_priv) >= 12 &&
4035 intel_can_enable_sagv(dev_priv, new_bw_state);
4036 }
4037
4038 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4039 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4040 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4041 if (ret)
4042 return ret;
4043 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4044 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4045 if (ret)
4046 return ret;
4047 }
4048
4049 return 0;
4050}
4051
4052static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4053{
4054 return INTEL_INFO(dev_priv)->dbuf.size /
4055 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4056}
4057
4058static void
4059skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4060 struct skl_ddb_entry *ddb)
4061{
4062 int slice_size = intel_dbuf_slice_size(dev_priv);
4063
4064 if (!slice_mask) {
4065 ddb->start = 0;
4066 ddb->end = 0;
4067 return;
4068 }
4069
4070 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4071 ddb->end = fls(slice_mask) * slice_size;
4072
4073 WARN_ON(ddb->start >= ddb->end);
4074 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4075}
4076
4077static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4078{
4079 struct skl_ddb_entry ddb;
4080
4081 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4082 slice_mask = BIT(DBUF_S1);
4083 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4084 slice_mask = BIT(DBUF_S3);
4085
4086 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4087
4088 return ddb.start;
4089}
4090
4091u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4092 const struct skl_ddb_entry *entry)
4093{
4094 int slice_size = intel_dbuf_slice_size(dev_priv);
4095 enum dbuf_slice start_slice, end_slice;
4096 u8 slice_mask = 0;
4097
4098 if (!skl_ddb_entry_size(entry))
4099 return 0;
4100
4101 start_slice = entry->start / slice_size;
4102 end_slice = (entry->end - 1) / slice_size;
4103
4104
4105
4106
4107
4108 while (start_slice <= end_slice) {
4109 slice_mask |= BIT(start_slice);
4110 start_slice++;
4111 }
4112
4113 return slice_mask;
4114}
4115
4116static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4117{
4118 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4119 int hdisplay, vdisplay;
4120
4121 if (!crtc_state->hw.active)
4122 return 0;
4123
4124
4125
4126
4127
4128
4129 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4130
4131 return hdisplay;
4132}
4133
4134static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4135 enum pipe for_pipe,
4136 unsigned int *weight_start,
4137 unsigned int *weight_end,
4138 unsigned int *weight_total)
4139{
4140 struct drm_i915_private *dev_priv =
4141 to_i915(dbuf_state->base.state->base.dev);
4142 enum pipe pipe;
4143
4144 *weight_start = 0;
4145 *weight_end = 0;
4146 *weight_total = 0;
4147
4148 for_each_pipe(dev_priv, pipe) {
4149 int weight = dbuf_state->weight[pipe];
4150
4151
4152
4153
4154
4155
4156
4157
4158 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4159 continue;
4160
4161 *weight_total += weight;
4162 if (pipe < for_pipe) {
4163 *weight_start += weight;
4164 *weight_end += weight;
4165 } else if (pipe == for_pipe) {
4166 *weight_end += weight;
4167 }
4168 }
4169}
4170
4171static int
4172skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4173{
4174 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4175 unsigned int weight_total, weight_start, weight_end;
4176 const struct intel_dbuf_state *old_dbuf_state =
4177 intel_atomic_get_old_dbuf_state(state);
4178 struct intel_dbuf_state *new_dbuf_state =
4179 intel_atomic_get_new_dbuf_state(state);
4180 struct intel_crtc_state *crtc_state;
4181 struct skl_ddb_entry ddb_slices;
4182 enum pipe pipe = crtc->pipe;
4183 unsigned int mbus_offset = 0;
4184 u32 ddb_range_size;
4185 u32 dbuf_slice_mask;
4186 u32 start, end;
4187 int ret;
4188
4189 if (new_dbuf_state->weight[pipe] == 0) {
4190 new_dbuf_state->ddb[pipe].start = 0;
4191 new_dbuf_state->ddb[pipe].end = 0;
4192 goto out;
4193 }
4194
4195 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4196
4197 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4198 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4199 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4200
4201 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4202 &weight_start, &weight_end, &weight_total);
4203
4204 start = ddb_range_size * weight_start / weight_total;
4205 end = ddb_range_size * weight_end / weight_total;
4206
4207 new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start;
4208 new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end;
4209out:
4210 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4211 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4212 &new_dbuf_state->ddb[pipe]))
4213 return 0;
4214
4215 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4216 if (ret)
4217 return ret;
4218
4219 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4220 if (IS_ERR(crtc_state))
4221 return PTR_ERR(crtc_state);
4222
4223
4224
4225
4226
4227 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4228 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4229
4230 drm_dbg_kms(&dev_priv->drm,
4231 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4232 crtc->base.base.id, crtc->base.name,
4233 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4234 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4235 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4236 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4237
4238 return 0;
4239}
4240
4241static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4242 int width, const struct drm_format_info *format,
4243 u64 modifier, unsigned int rotation,
4244 u32 plane_pixel_rate, struct skl_wm_params *wp,
4245 int color_plane);
4246static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4247 int level,
4248 unsigned int latency,
4249 const struct skl_wm_params *wp,
4250 const struct skl_wm_level *result_prev,
4251 struct skl_wm_level *result );
4252
4253static unsigned int
4254skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4255 int num_active)
4256{
4257 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4258 int level, max_level = ilk_wm_max_level(dev_priv);
4259 struct skl_wm_level wm = {};
4260 int ret, min_ddb_alloc = 0;
4261 struct skl_wm_params wp;
4262
4263 ret = skl_compute_wm_params(crtc_state, 256,
4264 drm_format_info(DRM_FORMAT_ARGB8888),
4265 DRM_FORMAT_MOD_LINEAR,
4266 DRM_MODE_ROTATE_0,
4267 crtc_state->pixel_rate, &wp, 0);
4268 drm_WARN_ON(&dev_priv->drm, ret);
4269
4270 for (level = 0; level <= max_level; level++) {
4271 unsigned int latency = dev_priv->wm.skl_latency[level];
4272
4273 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4274 if (wm.min_ddb_alloc == U16_MAX)
4275 break;
4276
4277 min_ddb_alloc = wm.min_ddb_alloc;
4278 }
4279
4280 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4281}
4282
4283static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4284 struct skl_ddb_entry *entry, u32 reg)
4285{
4286 entry->start = reg & DDB_ENTRY_MASK;
4287 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4288
4289 if (entry->end)
4290 entry->end += 1;
4291}
4292
4293static void
4294skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4295 const enum pipe pipe,
4296 const enum plane_id plane_id,
4297 struct skl_ddb_entry *ddb_y,
4298 struct skl_ddb_entry *ddb_uv)
4299{
4300 u32 val, val2;
4301 u32 fourcc = 0;
4302
4303
4304 if (plane_id == PLANE_CURSOR) {
4305 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4306 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4307 return;
4308 }
4309
4310 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4311
4312
4313 if (val & PLANE_CTL_ENABLE)
4314 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4315 val & PLANE_CTL_ORDER_RGBX,
4316 val & PLANE_CTL_ALPHA_MASK);
4317
4318 if (DISPLAY_VER(dev_priv) >= 11) {
4319 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4320 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4321 } else {
4322 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4323 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4324
4325 if (fourcc &&
4326 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4327 swap(val, val2);
4328
4329 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4330 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4331 }
4332}
4333
4334void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4335 struct skl_ddb_entry *ddb_y,
4336 struct skl_ddb_entry *ddb_uv)
4337{
4338 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4339 enum intel_display_power_domain power_domain;
4340 enum pipe pipe = crtc->pipe;
4341 intel_wakeref_t wakeref;
4342 enum plane_id plane_id;
4343
4344 power_domain = POWER_DOMAIN_PIPE(pipe);
4345 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4346 if (!wakeref)
4347 return;
4348
4349 for_each_plane_id_on_crtc(crtc, plane_id)
4350 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4351 plane_id,
4352 &ddb_y[plane_id],
4353 &ddb_uv[plane_id]);
4354
4355 intel_display_power_put(dev_priv, power_domain, wakeref);
4356}
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374static uint_fixed_16_16_t
4375skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4376 const struct intel_plane_state *plane_state)
4377{
4378 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4379 u32 src_w, src_h, dst_w, dst_h;
4380 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4381 uint_fixed_16_16_t downscale_h, downscale_w;
4382
4383 if (drm_WARN_ON(&dev_priv->drm,
4384 !intel_wm_plane_visible(crtc_state, plane_state)))
4385 return u32_to_fixed16(0);
4386
4387
4388
4389
4390
4391
4392
4393
4394 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4395 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4396 dst_w = drm_rect_width(&plane_state->uapi.dst);
4397 dst_h = drm_rect_height(&plane_state->uapi.dst);
4398
4399 fp_w_ratio = div_fixed16(src_w, dst_w);
4400 fp_h_ratio = div_fixed16(src_h, dst_h);
4401 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4402 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4403
4404 return mul_fixed16(downscale_w, downscale_h);
4405}
4406
4407struct dbuf_slice_conf_entry {
4408 u8 active_pipes;
4409 u8 dbuf_mask[I915_MAX_PIPES];
4410 bool join_mbus;
4411};
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4424
4425{
4426 {
4427 .active_pipes = BIT(PIPE_A),
4428 .dbuf_mask = {
4429 [PIPE_A] = BIT(DBUF_S1),
4430 },
4431 },
4432 {
4433 .active_pipes = BIT(PIPE_B),
4434 .dbuf_mask = {
4435 [PIPE_B] = BIT(DBUF_S1),
4436 },
4437 },
4438 {
4439 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4440 .dbuf_mask = {
4441 [PIPE_A] = BIT(DBUF_S1),
4442 [PIPE_B] = BIT(DBUF_S2),
4443 },
4444 },
4445 {
4446 .active_pipes = BIT(PIPE_C),
4447 .dbuf_mask = {
4448 [PIPE_C] = BIT(DBUF_S2),
4449 },
4450 },
4451 {
4452 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4453 .dbuf_mask = {
4454 [PIPE_A] = BIT(DBUF_S1),
4455 [PIPE_C] = BIT(DBUF_S2),
4456 },
4457 },
4458 {
4459 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4460 .dbuf_mask = {
4461 [PIPE_B] = BIT(DBUF_S1),
4462 [PIPE_C] = BIT(DBUF_S2),
4463 },
4464 },
4465 {
4466 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4467 .dbuf_mask = {
4468 [PIPE_A] = BIT(DBUF_S1),
4469 [PIPE_B] = BIT(DBUF_S1),
4470 [PIPE_C] = BIT(DBUF_S2),
4471 },
4472 },
4473 {}
4474};
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4487
4488{
4489 {
4490 .active_pipes = BIT(PIPE_A),
4491 .dbuf_mask = {
4492 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4493 },
4494 },
4495 {
4496 .active_pipes = BIT(PIPE_B),
4497 .dbuf_mask = {
4498 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4499 },
4500 },
4501 {
4502 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4503 .dbuf_mask = {
4504 [PIPE_A] = BIT(DBUF_S2),
4505 [PIPE_B] = BIT(DBUF_S1),
4506 },
4507 },
4508 {
4509 .active_pipes = BIT(PIPE_C),
4510 .dbuf_mask = {
4511 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4512 },
4513 },
4514 {
4515 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4516 .dbuf_mask = {
4517 [PIPE_A] = BIT(DBUF_S1),
4518 [PIPE_C] = BIT(DBUF_S2),
4519 },
4520 },
4521 {
4522 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4523 .dbuf_mask = {
4524 [PIPE_B] = BIT(DBUF_S1),
4525 [PIPE_C] = BIT(DBUF_S2),
4526 },
4527 },
4528 {
4529 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4530 .dbuf_mask = {
4531 [PIPE_A] = BIT(DBUF_S1),
4532 [PIPE_B] = BIT(DBUF_S1),
4533 [PIPE_C] = BIT(DBUF_S2),
4534 },
4535 },
4536 {
4537 .active_pipes = BIT(PIPE_D),
4538 .dbuf_mask = {
4539 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4540 },
4541 },
4542 {
4543 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4544 .dbuf_mask = {
4545 [PIPE_A] = BIT(DBUF_S1),
4546 [PIPE_D] = BIT(DBUF_S2),
4547 },
4548 },
4549 {
4550 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4551 .dbuf_mask = {
4552 [PIPE_B] = BIT(DBUF_S1),
4553 [PIPE_D] = BIT(DBUF_S2),
4554 },
4555 },
4556 {
4557 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4558 .dbuf_mask = {
4559 [PIPE_A] = BIT(DBUF_S1),
4560 [PIPE_B] = BIT(DBUF_S1),
4561 [PIPE_D] = BIT(DBUF_S2),
4562 },
4563 },
4564 {
4565 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4566 .dbuf_mask = {
4567 [PIPE_C] = BIT(DBUF_S1),
4568 [PIPE_D] = BIT(DBUF_S2),
4569 },
4570 },
4571 {
4572 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4573 .dbuf_mask = {
4574 [PIPE_A] = BIT(DBUF_S1),
4575 [PIPE_C] = BIT(DBUF_S2),
4576 [PIPE_D] = BIT(DBUF_S2),
4577 },
4578 },
4579 {
4580 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4581 .dbuf_mask = {
4582 [PIPE_B] = BIT(DBUF_S1),
4583 [PIPE_C] = BIT(DBUF_S2),
4584 [PIPE_D] = BIT(DBUF_S2),
4585 },
4586 },
4587 {
4588 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4589 .dbuf_mask = {
4590 [PIPE_A] = BIT(DBUF_S1),
4591 [PIPE_B] = BIT(DBUF_S1),
4592 [PIPE_C] = BIT(DBUF_S2),
4593 [PIPE_D] = BIT(DBUF_S2),
4594 },
4595 },
4596 {}
4597};
4598
4599static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
4600 {
4601 .active_pipes = BIT(PIPE_A),
4602 .dbuf_mask = {
4603 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4604 },
4605 },
4606 {
4607 .active_pipes = BIT(PIPE_B),
4608 .dbuf_mask = {
4609 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4610 },
4611 },
4612 {
4613 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4614 .dbuf_mask = {
4615 [PIPE_A] = BIT(DBUF_S1),
4616 [PIPE_B] = BIT(DBUF_S2),
4617 },
4618 },
4619 {
4620 .active_pipes = BIT(PIPE_C),
4621 .dbuf_mask = {
4622 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4623 },
4624 },
4625 {
4626 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4627 .dbuf_mask = {
4628 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4629 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4630 },
4631 },
4632 {
4633 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4634 .dbuf_mask = {
4635 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4636 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4637 },
4638 },
4639 {
4640 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4641 .dbuf_mask = {
4642 [PIPE_A] = BIT(DBUF_S1),
4643 [PIPE_B] = BIT(DBUF_S2),
4644 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4645 },
4646 },
4647 {
4648 .active_pipes = BIT(PIPE_D),
4649 .dbuf_mask = {
4650 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4651 },
4652 },
4653 {
4654 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4655 .dbuf_mask = {
4656 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4657 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4658 },
4659 },
4660 {
4661 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4662 .dbuf_mask = {
4663 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4664 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4665 },
4666 },
4667 {
4668 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4669 .dbuf_mask = {
4670 [PIPE_A] = BIT(DBUF_S1),
4671 [PIPE_B] = BIT(DBUF_S2),
4672 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4673 },
4674 },
4675 {
4676 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4677 .dbuf_mask = {
4678 [PIPE_C] = BIT(DBUF_S3),
4679 [PIPE_D] = BIT(DBUF_S4),
4680 },
4681 },
4682 {
4683 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4684 .dbuf_mask = {
4685 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4686 [PIPE_C] = BIT(DBUF_S3),
4687 [PIPE_D] = BIT(DBUF_S4),
4688 },
4689 },
4690 {
4691 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4692 .dbuf_mask = {
4693 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4694 [PIPE_C] = BIT(DBUF_S3),
4695 [PIPE_D] = BIT(DBUF_S4),
4696 },
4697 },
4698 {
4699 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4700 .dbuf_mask = {
4701 [PIPE_A] = BIT(DBUF_S1),
4702 [PIPE_B] = BIT(DBUF_S2),
4703 [PIPE_C] = BIT(DBUF_S3),
4704 [PIPE_D] = BIT(DBUF_S4),
4705 },
4706 },
4707 {}
4708};
4709
4710static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4711 {
4712 .active_pipes = BIT(PIPE_A),
4713 .dbuf_mask = {
4714 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4715 },
4716 .join_mbus = true,
4717 },
4718 {
4719 .active_pipes = BIT(PIPE_B),
4720 .dbuf_mask = {
4721 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4722 },
4723 .join_mbus = true,
4724 },
4725 {
4726 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4727 .dbuf_mask = {
4728 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4729 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4730 },
4731 },
4732 {
4733 .active_pipes = BIT(PIPE_C),
4734 .dbuf_mask = {
4735 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4736 },
4737 },
4738 {
4739 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4740 .dbuf_mask = {
4741 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4742 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4743 },
4744 },
4745 {
4746 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4747 .dbuf_mask = {
4748 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4749 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4750 },
4751 },
4752 {
4753 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4754 .dbuf_mask = {
4755 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4756 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4757 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4758 },
4759 },
4760 {
4761 .active_pipes = BIT(PIPE_D),
4762 .dbuf_mask = {
4763 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4764 },
4765 },
4766 {
4767 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4768 .dbuf_mask = {
4769 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4770 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4771 },
4772 },
4773 {
4774 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4775 .dbuf_mask = {
4776 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4777 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4778 },
4779 },
4780 {
4781 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4782 .dbuf_mask = {
4783 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4784 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4785 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4786 },
4787 },
4788 {
4789 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4790 .dbuf_mask = {
4791 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4792 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4793 },
4794 },
4795 {
4796 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4797 .dbuf_mask = {
4798 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4799 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4800 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4801 },
4802 },
4803 {
4804 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4805 .dbuf_mask = {
4806 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4807 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4808 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4809 },
4810 },
4811 {
4812 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4813 .dbuf_mask = {
4814 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4815 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4816 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4817 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4818 },
4819 },
4820 {}
4821
4822};
4823
4824static bool check_mbus_joined(u8 active_pipes,
4825 const struct dbuf_slice_conf_entry *dbuf_slices)
4826{
4827 int i;
4828
4829 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4830 if (dbuf_slices[i].active_pipes == active_pipes)
4831 return dbuf_slices[i].join_mbus;
4832 }
4833 return false;
4834}
4835
4836static bool adlp_check_mbus_joined(u8 active_pipes)
4837{
4838 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4839}
4840
4841static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4842 const struct dbuf_slice_conf_entry *dbuf_slices)
4843{
4844 int i;
4845
4846 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4847 if (dbuf_slices[i].active_pipes == active_pipes)
4848 return dbuf_slices[i].dbuf_mask[pipe];
4849 }
4850 return 0;
4851}
4852
4853
4854
4855
4856
4857
4858static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4859{
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4873}
4874
4875static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4876{
4877 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4878}
4879
4880static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
4881{
4882 return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs);
4883}
4884
4885static u32 dg2_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
4886{
4887 return compute_dbuf_slices(pipe, active_pipes, dg2_allowed_dbufs);
4888}
4889
4890static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4891{
4892 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4893 enum pipe pipe = crtc->pipe;
4894
4895 if (IS_DG2(dev_priv))
4896 return dg2_compute_dbuf_slices(pipe, active_pipes);
4897 else if (IS_ALDERLAKE_P(dev_priv))
4898 return adlp_compute_dbuf_slices(pipe, active_pipes);
4899 else if (DISPLAY_VER(dev_priv) == 12)
4900 return tgl_compute_dbuf_slices(pipe, active_pipes);
4901 else if (DISPLAY_VER(dev_priv) == 11)
4902 return icl_compute_dbuf_slices(pipe, active_pipes);
4903
4904
4905
4906
4907 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4908}
4909
4910static u64
4911skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4912 const struct intel_plane_state *plane_state,
4913 int color_plane)
4914{
4915 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4916 const struct drm_framebuffer *fb = plane_state->hw.fb;
4917 u32 data_rate;
4918 u32 width = 0, height = 0;
4919 uint_fixed_16_16_t down_scale_amount;
4920 u64 rate;
4921
4922 if (!plane_state->uapi.visible)
4923 return 0;
4924
4925 if (plane->id == PLANE_CURSOR)
4926 return 0;
4927
4928 if (color_plane == 1 &&
4929 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4930 return 0;
4931
4932
4933
4934
4935
4936
4937 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4938 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4939
4940
4941 if (color_plane == 1) {
4942 width /= 2;
4943 height /= 2;
4944 }
4945
4946 data_rate = width * height;
4947
4948 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4949
4950 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4951
4952 rate *= fb->format->cpp[color_plane];
4953 return rate;
4954}
4955
4956static u64
4957skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4958 struct intel_crtc *crtc)
4959{
4960 struct intel_crtc_state *crtc_state =
4961 intel_atomic_get_new_crtc_state(state, crtc);
4962 const struct intel_plane_state *plane_state;
4963 struct intel_plane *plane;
4964 u64 total_data_rate = 0;
4965 enum plane_id plane_id;
4966 int i;
4967
4968
4969 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4970 if (plane->pipe != crtc->pipe)
4971 continue;
4972
4973 plane_id = plane->id;
4974
4975
4976 crtc_state->plane_data_rate[plane_id] =
4977 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4978
4979
4980 crtc_state->uv_plane_data_rate[plane_id] =
4981 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4982 }
4983
4984 for_each_plane_id_on_crtc(crtc, plane_id) {
4985 total_data_rate += crtc_state->plane_data_rate[plane_id];
4986 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4987 }
4988
4989 return total_data_rate;
4990}
4991
4992static u64
4993icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4994 struct intel_crtc *crtc)
4995{
4996 struct intel_crtc_state *crtc_state =
4997 intel_atomic_get_new_crtc_state(state, crtc);
4998 const struct intel_plane_state *plane_state;
4999 struct intel_plane *plane;
5000 u64 total_data_rate = 0;
5001 enum plane_id plane_id;
5002 int i;
5003
5004
5005 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5006 if (plane->pipe != crtc->pipe)
5007 continue;
5008
5009 plane_id = plane->id;
5010
5011 if (!plane_state->planar_linked_plane) {
5012 crtc_state->plane_data_rate[plane_id] =
5013 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5014 } else {
5015 enum plane_id y_plane_id;
5016
5017
5018
5019
5020
5021
5022
5023
5024 if (plane_state->planar_slave)
5025 continue;
5026
5027
5028 y_plane_id = plane_state->planar_linked_plane->id;
5029 crtc_state->plane_data_rate[y_plane_id] =
5030 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5031
5032 crtc_state->plane_data_rate[plane_id] =
5033 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5034 }
5035 }
5036
5037 for_each_plane_id_on_crtc(crtc, plane_id)
5038 total_data_rate += crtc_state->plane_data_rate[plane_id];
5039
5040 return total_data_rate;
5041}
5042
5043const struct skl_wm_level *
5044skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
5045 enum plane_id plane_id,
5046 int level)
5047{
5048 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5049
5050 if (level == 0 && pipe_wm->use_sagv_wm)
5051 return &wm->sagv.wm0;
5052
5053 return &wm->wm[level];
5054}
5055
5056const struct skl_wm_level *
5057skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
5058 enum plane_id plane_id)
5059{
5060 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5061
5062 if (pipe_wm->use_sagv_wm)
5063 return &wm->sagv.trans_wm;
5064
5065 return &wm->trans_wm;
5066}
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080static void
5081skl_check_wm_level(struct skl_wm_level *wm, u64 total)
5082{
5083 if (wm->min_ddb_alloc > total)
5084 memset(wm, 0, sizeof(*wm));
5085}
5086
5087static void
5088skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
5089 u64 total, u64 uv_total)
5090{
5091 if (wm->min_ddb_alloc > total ||
5092 uv_wm->min_ddb_alloc > uv_total) {
5093 memset(wm, 0, sizeof(*wm));
5094 memset(uv_wm, 0, sizeof(*uv_wm));
5095 }
5096}
5097
5098static int
5099skl_allocate_plane_ddb(struct intel_atomic_state *state,
5100 struct intel_crtc *crtc)
5101{
5102 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5103 struct intel_crtc_state *crtc_state =
5104 intel_atomic_get_new_crtc_state(state, crtc);
5105 const struct intel_dbuf_state *dbuf_state =
5106 intel_atomic_get_new_dbuf_state(state);
5107 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
5108 int num_active = hweight8(dbuf_state->active_pipes);
5109 u16 alloc_size, start = 0;
5110 u16 total[I915_MAX_PLANES] = {};
5111 u16 uv_total[I915_MAX_PLANES] = {};
5112 u64 total_data_rate;
5113 enum plane_id plane_id;
5114 u32 blocks;
5115 int level;
5116
5117
5118 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
5119 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
5120
5121 if (!crtc_state->hw.active)
5122 return 0;
5123
5124 if (DISPLAY_VER(dev_priv) >= 11)
5125 total_data_rate =
5126 icl_get_total_relative_data_rate(state, crtc);
5127 else
5128 total_data_rate =
5129 skl_get_total_relative_data_rate(state, crtc);
5130
5131 alloc_size = skl_ddb_entry_size(alloc);
5132 if (alloc_size == 0)
5133 return 0;
5134
5135
5136 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5137 alloc_size -= total[PLANE_CURSOR];
5138 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
5139 alloc->end - total[PLANE_CURSOR];
5140 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
5141
5142 if (total_data_rate == 0)
5143 return 0;
5144
5145
5146
5147
5148
5149 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5150 blocks = 0;
5151 for_each_plane_id_on_crtc(crtc, plane_id) {
5152 const struct skl_plane_wm *wm =
5153 &crtc_state->wm.skl.optimal.planes[plane_id];
5154
5155 if (plane_id == PLANE_CURSOR) {
5156 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
5157 drm_WARN_ON(&dev_priv->drm,
5158 wm->wm[level].min_ddb_alloc != U16_MAX);
5159 blocks = U32_MAX;
5160 break;
5161 }
5162 continue;
5163 }
5164
5165 blocks += wm->wm[level].min_ddb_alloc;
5166 blocks += wm->uv_wm[level].min_ddb_alloc;
5167 }
5168
5169 if (blocks <= alloc_size) {
5170 alloc_size -= blocks;
5171 break;
5172 }
5173 }
5174
5175 if (level < 0) {
5176 drm_dbg_kms(&dev_priv->drm,
5177 "Requested display configuration exceeds system DDB limitations");
5178 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5179 blocks, alloc_size);
5180 return -EINVAL;
5181 }
5182
5183
5184
5185
5186
5187
5188 for_each_plane_id_on_crtc(crtc, plane_id) {
5189 const struct skl_plane_wm *wm =
5190 &crtc_state->wm.skl.optimal.planes[plane_id];
5191 u64 rate;
5192 u16 extra;
5193
5194 if (plane_id == PLANE_CURSOR)
5195 continue;
5196
5197
5198
5199
5200
5201 if (total_data_rate == 0)
5202 break;
5203
5204 rate = crtc_state->plane_data_rate[plane_id];
5205 extra = min_t(u16, alloc_size,
5206 DIV64_U64_ROUND_UP(alloc_size * rate,
5207 total_data_rate));
5208 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
5209 alloc_size -= extra;
5210 total_data_rate -= rate;
5211
5212 if (total_data_rate == 0)
5213 break;
5214
5215 rate = crtc_state->uv_plane_data_rate[plane_id];
5216 extra = min_t(u16, alloc_size,
5217 DIV64_U64_ROUND_UP(alloc_size * rate,
5218 total_data_rate));
5219 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
5220 alloc_size -= extra;
5221 total_data_rate -= rate;
5222 }
5223 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
5224
5225
5226 start = alloc->start;
5227 for_each_plane_id_on_crtc(crtc, plane_id) {
5228 struct skl_ddb_entry *plane_alloc =
5229 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5230 struct skl_ddb_entry *uv_plane_alloc =
5231 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5232
5233 if (plane_id == PLANE_CURSOR)
5234 continue;
5235
5236
5237 drm_WARN_ON(&dev_priv->drm,
5238 DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
5239
5240
5241 if (total[plane_id]) {
5242 plane_alloc->start = start;
5243 start += total[plane_id];
5244 plane_alloc->end = start;
5245 }
5246
5247 if (uv_total[plane_id]) {
5248 uv_plane_alloc->start = start;
5249 start += uv_total[plane_id];
5250 uv_plane_alloc->end = start;
5251 }
5252 }
5253
5254
5255
5256
5257
5258
5259
5260 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5261 for_each_plane_id_on_crtc(crtc, plane_id) {
5262 struct skl_plane_wm *wm =
5263 &crtc_state->wm.skl.optimal.planes[plane_id];
5264
5265 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
5266 total[plane_id], uv_total[plane_id]);
5267
5268
5269
5270
5271
5272 if (DISPLAY_VER(dev_priv) == 11 &&
5273 level == 1 && wm->wm[0].enable) {
5274 wm->wm[level].blocks = wm->wm[0].blocks;
5275 wm->wm[level].lines = wm->wm[0].lines;
5276 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5277 }
5278 }
5279 }
5280
5281
5282
5283
5284
5285 for_each_plane_id_on_crtc(crtc, plane_id) {
5286 struct skl_plane_wm *wm =
5287 &crtc_state->wm.skl.optimal.planes[plane_id];
5288
5289 skl_check_wm_level(&wm->trans_wm, total[plane_id]);
5290 skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
5291 skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5292 }
5293
5294 return 0;
5295}
5296
5297
5298
5299
5300
5301
5302
5303static uint_fixed_16_16_t
5304skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5305 u8 cpp, u32 latency, u32 dbuf_block_size)
5306{
5307 u32 wm_intermediate_val;
5308 uint_fixed_16_16_t ret;
5309
5310 if (latency == 0)
5311 return FP_16_16_MAX;
5312
5313 wm_intermediate_val = latency * pixel_rate * cpp;
5314 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5315
5316 if (DISPLAY_VER(dev_priv) >= 10)
5317 ret = add_fixed16_u32(ret, 1);
5318
5319 return ret;
5320}
5321
5322static uint_fixed_16_16_t
5323skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5324 uint_fixed_16_16_t plane_blocks_per_line)
5325{
5326 u32 wm_intermediate_val;
5327 uint_fixed_16_16_t ret;
5328
5329 if (latency == 0)
5330 return FP_16_16_MAX;
5331
5332 wm_intermediate_val = latency * pixel_rate;
5333 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5334 pipe_htotal * 1000);
5335 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5336 return ret;
5337}
5338
5339static uint_fixed_16_16_t
5340intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5341{
5342 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5343 u32 pixel_rate;
5344 u32 crtc_htotal;
5345 uint_fixed_16_16_t linetime_us;
5346
5347 if (!crtc_state->hw.active)
5348 return u32_to_fixed16(0);
5349
5350 pixel_rate = crtc_state->pixel_rate;
5351
5352 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5353 return u32_to_fixed16(0);
5354
5355 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5356 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5357
5358 return linetime_us;
5359}
5360
5361static int
5362skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5363 int width, const struct drm_format_info *format,
5364 u64 modifier, unsigned int rotation,
5365 u32 plane_pixel_rate, struct skl_wm_params *wp,
5366 int color_plane)
5367{
5368 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5369 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5370 u32 interm_pbpl;
5371
5372
5373 if (color_plane == 1 &&
5374 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5375 drm_dbg_kms(&dev_priv->drm,
5376 "Non planar format have single plane\n");
5377 return -EINVAL;
5378 }
5379
5380 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5381 modifier == I915_FORMAT_MOD_Yf_TILED ||
5382 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5383 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5384 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5385 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5386 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5387 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5388
5389 wp->width = width;
5390 if (color_plane == 1 && wp->is_planar)
5391 wp->width /= 2;
5392
5393 wp->cpp = format->cpp[color_plane];
5394 wp->plane_pixel_rate = plane_pixel_rate;
5395
5396 if (DISPLAY_VER(dev_priv) >= 11 &&
5397 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5398 wp->dbuf_block_size = 256;
5399 else
5400 wp->dbuf_block_size = 512;
5401
5402 if (drm_rotation_90_or_270(rotation)) {
5403 switch (wp->cpp) {
5404 case 1:
5405 wp->y_min_scanlines = 16;
5406 break;
5407 case 2:
5408 wp->y_min_scanlines = 8;
5409 break;
5410 case 4:
5411 wp->y_min_scanlines = 4;
5412 break;
5413 default:
5414 MISSING_CASE(wp->cpp);
5415 return -EINVAL;
5416 }
5417 } else {
5418 wp->y_min_scanlines = 4;
5419 }
5420
5421 if (skl_needs_memory_bw_wa(dev_priv))
5422 wp->y_min_scanlines *= 2;
5423
5424 wp->plane_bytes_per_line = wp->width * wp->cpp;
5425 if (wp->y_tiled) {
5426 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5427 wp->y_min_scanlines,
5428 wp->dbuf_block_size);
5429
5430 if (DISPLAY_VER(dev_priv) >= 10)
5431 interm_pbpl++;
5432
5433 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5434 wp->y_min_scanlines);
5435 } else {
5436 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5437 wp->dbuf_block_size);
5438
5439 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5440 interm_pbpl++;
5441
5442 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5443 }
5444
5445 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5446 wp->plane_blocks_per_line);
5447
5448 wp->linetime_us = fixed16_to_u32_round_up(
5449 intel_get_linetime_us(crtc_state));
5450
5451 return 0;
5452}
5453
5454static int
5455skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5456 const struct intel_plane_state *plane_state,
5457 struct skl_wm_params *wp, int color_plane)
5458{
5459 const struct drm_framebuffer *fb = plane_state->hw.fb;
5460 int width;
5461
5462
5463
5464
5465
5466
5467 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5468
5469 return skl_compute_wm_params(crtc_state, width,
5470 fb->format, fb->modifier,
5471 plane_state->hw.rotation,
5472 intel_plane_pixel_rate(crtc_state, plane_state),
5473 wp, color_plane);
5474}
5475
5476static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5477{
5478 if (DISPLAY_VER(dev_priv) >= 10)
5479 return true;
5480
5481
5482 return level > 0;
5483}
5484
5485static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5486{
5487 if (DISPLAY_VER(dev_priv) >= 13)
5488 return 255;
5489 else
5490 return 31;
5491}
5492
5493static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5494 int level,
5495 unsigned int latency,
5496 const struct skl_wm_params *wp,
5497 const struct skl_wm_level *result_prev,
5498 struct skl_wm_level *result )
5499{
5500 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5501 uint_fixed_16_16_t method1, method2;
5502 uint_fixed_16_16_t selected_result;
5503 u32 blocks, lines, min_ddb_alloc = 0;
5504
5505 if (latency == 0) {
5506
5507 result->min_ddb_alloc = U16_MAX;
5508 return;
5509 }
5510
5511
5512
5513
5514
5515 if ((IS_KABYLAKE(dev_priv) ||
5516 IS_COFFEELAKE(dev_priv) ||
5517 IS_COMETLAKE(dev_priv)) &&
5518 dev_priv->ipc_enabled)
5519 latency += 4;
5520
5521 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5522 latency += 15;
5523
5524 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5525 wp->cpp, latency, wp->dbuf_block_size);
5526 method2 = skl_wm_method2(wp->plane_pixel_rate,
5527 crtc_state->hw.pipe_mode.crtc_htotal,
5528 latency,
5529 wp->plane_blocks_per_line);
5530
5531 if (wp->y_tiled) {
5532 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5533 } else {
5534 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5535 wp->dbuf_block_size < 1) &&
5536 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5537 selected_result = method2;
5538 } else if (latency >= wp->linetime_us) {
5539 if (DISPLAY_VER(dev_priv) == 9)
5540 selected_result = min_fixed16(method1, method2);
5541 else
5542 selected_result = method2;
5543 } else {
5544 selected_result = method1;
5545 }
5546 }
5547
5548 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5549 lines = div_round_up_fixed16(selected_result,
5550 wp->plane_blocks_per_line);
5551
5552 if (DISPLAY_VER(dev_priv) == 9) {
5553
5554 if (level == 0 && wp->rc_surface)
5555 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5556
5557
5558 if (level >= 1 && level <= 7) {
5559 if (wp->y_tiled) {
5560 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5561 lines += wp->y_min_scanlines;
5562 } else {
5563 blocks++;
5564 }
5565
5566
5567
5568
5569
5570
5571
5572 if (result_prev->blocks > blocks)
5573 blocks = result_prev->blocks;
5574 }
5575 }
5576
5577 if (DISPLAY_VER(dev_priv) >= 11) {
5578 if (wp->y_tiled) {
5579 int extra_lines;
5580
5581 if (lines % wp->y_min_scanlines == 0)
5582 extra_lines = wp->y_min_scanlines;
5583 else
5584 extra_lines = wp->y_min_scanlines * 2 -
5585 lines % wp->y_min_scanlines;
5586
5587 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5588 wp->plane_blocks_per_line);
5589 } else {
5590 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5591 }
5592 }
5593
5594 if (!skl_wm_has_lines(dev_priv, level))
5595 lines = 0;
5596
5597 if (lines > skl_wm_max_lines(dev_priv)) {
5598
5599 result->min_ddb_alloc = U16_MAX;
5600 return;
5601 }
5602
5603
5604
5605
5606
5607
5608
5609 result->blocks = blocks;
5610 result->lines = lines;
5611
5612 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5613 result->enable = true;
5614
5615 if (DISPLAY_VER(dev_priv) < 12)
5616 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5617}
5618
5619static void
5620skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5621 const struct skl_wm_params *wm_params,
5622 struct skl_wm_level *levels)
5623{
5624 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5625 int level, max_level = ilk_wm_max_level(dev_priv);
5626 struct skl_wm_level *result_prev = &levels[0];
5627
5628 for (level = 0; level <= max_level; level++) {
5629 struct skl_wm_level *result = &levels[level];
5630 unsigned int latency = dev_priv->wm.skl_latency[level];
5631
5632 skl_compute_plane_wm(crtc_state, level, latency,
5633 wm_params, result_prev, result);
5634
5635 result_prev = result;
5636 }
5637}
5638
5639static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5640 const struct skl_wm_params *wm_params,
5641 struct skl_plane_wm *plane_wm)
5642{
5643 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5644 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5645 struct skl_wm_level *levels = plane_wm->wm;
5646 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5647
5648 skl_compute_plane_wm(crtc_state, 0, latency,
5649 wm_params, &levels[0],
5650 sagv_wm);
5651}
5652
5653static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5654 struct skl_wm_level *trans_wm,
5655 const struct skl_wm_level *wm0,
5656 const struct skl_wm_params *wp)
5657{
5658 u16 trans_min, trans_amount, trans_y_tile_min;
5659 u16 wm0_blocks, trans_offset, blocks;
5660
5661
5662 if (!dev_priv->ipc_enabled)
5663 return;
5664
5665
5666
5667
5668
5669 if (DISPLAY_VER(dev_priv) == 9)
5670 return;
5671
5672 if (DISPLAY_VER(dev_priv) >= 11)
5673 trans_min = 4;
5674 else
5675 trans_min = 14;
5676
5677
5678 if (DISPLAY_VER(dev_priv) == 10)
5679 trans_amount = 0;
5680 else
5681 trans_amount = 10;
5682
5683 trans_offset = trans_min + trans_amount;
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695 wm0_blocks = wm0->blocks - 1;
5696
5697 if (wp->y_tiled) {
5698 trans_y_tile_min =
5699 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5700 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5701 } else {
5702 blocks = wm0_blocks + trans_offset;
5703 }
5704 blocks++;
5705
5706
5707
5708
5709
5710
5711 trans_wm->blocks = blocks;
5712 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5713 trans_wm->enable = true;
5714}
5715
5716static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5717 const struct intel_plane_state *plane_state,
5718 enum plane_id plane_id, int color_plane)
5719{
5720 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5721 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5722 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5723 struct skl_wm_params wm_params;
5724 int ret;
5725
5726 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5727 &wm_params, color_plane);
5728 if (ret)
5729 return ret;
5730
5731 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5732
5733 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5734 &wm->wm[0], &wm_params);
5735
5736 if (DISPLAY_VER(dev_priv) >= 12) {
5737 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5738
5739 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5740 &wm->sagv.wm0, &wm_params);
5741 }
5742
5743 return 0;
5744}
5745
5746static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5747 const struct intel_plane_state *plane_state,
5748 enum plane_id plane_id)
5749{
5750 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5751 struct skl_wm_params wm_params;
5752 int ret;
5753
5754 wm->is_planar = true;
5755
5756
5757 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5758 &wm_params, 1);
5759 if (ret)
5760 return ret;
5761
5762 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5763
5764 return 0;
5765}
5766
5767static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5768 const struct intel_plane_state *plane_state)
5769{
5770 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5771 enum plane_id plane_id = plane->id;
5772 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5773 const struct drm_framebuffer *fb = plane_state->hw.fb;
5774 int ret;
5775
5776 memset(wm, 0, sizeof(*wm));
5777
5778 if (!intel_wm_plane_visible(crtc_state, plane_state))
5779 return 0;
5780
5781 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5782 plane_id, 0);
5783 if (ret)
5784 return ret;
5785
5786 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5787 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5788 plane_id);
5789 if (ret)
5790 return ret;
5791 }
5792
5793 return 0;
5794}
5795
5796static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5797 const struct intel_plane_state *plane_state)
5798{
5799 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5800 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5801 enum plane_id plane_id = plane->id;
5802 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5803 int ret;
5804
5805
5806 if (plane_state->planar_slave)
5807 return 0;
5808
5809 memset(wm, 0, sizeof(*wm));
5810
5811 if (plane_state->planar_linked_plane) {
5812 const struct drm_framebuffer *fb = plane_state->hw.fb;
5813 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5814
5815 drm_WARN_ON(&dev_priv->drm,
5816 !intel_wm_plane_visible(crtc_state, plane_state));
5817 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5818 fb->format->num_planes == 1);
5819
5820 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5821 y_plane_id, 0);
5822 if (ret)
5823 return ret;
5824
5825 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5826 plane_id, 1);
5827 if (ret)
5828 return ret;
5829 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5830 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5831 plane_id, 0);
5832 if (ret)
5833 return ret;
5834 }
5835
5836 return 0;
5837}
5838
5839static int skl_build_pipe_wm(struct intel_atomic_state *state,
5840 struct intel_crtc *crtc)
5841{
5842 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5843 struct intel_crtc_state *crtc_state =
5844 intel_atomic_get_new_crtc_state(state, crtc);
5845 const struct intel_plane_state *plane_state;
5846 struct intel_plane *plane;
5847 int ret, i;
5848
5849 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5850
5851
5852
5853
5854
5855 if (plane->pipe != crtc->pipe)
5856 continue;
5857
5858 if (DISPLAY_VER(dev_priv) >= 11)
5859 ret = icl_build_plane_wm(crtc_state, plane_state);
5860 else
5861 ret = skl_build_plane_wm(crtc_state, plane_state);
5862 if (ret)
5863 return ret;
5864 }
5865
5866 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5867
5868 return 0;
5869}
5870
5871static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5872 i915_reg_t reg,
5873 const struct skl_ddb_entry *entry)
5874{
5875 if (entry->end)
5876 intel_de_write_fw(dev_priv, reg,
5877 (entry->end - 1) << 16 | entry->start);
5878 else
5879 intel_de_write_fw(dev_priv, reg, 0);
5880}
5881
5882static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5883 i915_reg_t reg,
5884 const struct skl_wm_level *level)
5885{
5886 u32 val = 0;
5887
5888 if (level->enable)
5889 val |= PLANE_WM_EN;
5890 if (level->ignore_lines)
5891 val |= PLANE_WM_IGNORE_LINES;
5892 val |= level->blocks;
5893 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5894
5895 intel_de_write_fw(dev_priv, reg, val);
5896}
5897
5898void skl_write_plane_wm(struct intel_plane *plane,
5899 const struct intel_crtc_state *crtc_state)
5900{
5901 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5902 int level, max_level = ilk_wm_max_level(dev_priv);
5903 enum plane_id plane_id = plane->id;
5904 enum pipe pipe = plane->pipe;
5905 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5906 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5907 const struct skl_ddb_entry *ddb_y =
5908 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5909 const struct skl_ddb_entry *ddb_uv =
5910 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5911
5912 for (level = 0; level <= max_level; level++)
5913 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5914 skl_plane_wm_level(pipe_wm, plane_id, level));
5915
5916 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5917 skl_plane_trans_wm(pipe_wm, plane_id));
5918
5919 if (HAS_HW_SAGV_WM(dev_priv)) {
5920 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5921 &wm->sagv.wm0);
5922 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5923 &wm->sagv.trans_wm);
5924 }
5925
5926 if (DISPLAY_VER(dev_priv) >= 11) {
5927 skl_ddb_entry_write(dev_priv,
5928 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5929 return;
5930 }
5931
5932 if (wm->is_planar)
5933 swap(ddb_y, ddb_uv);
5934
5935 skl_ddb_entry_write(dev_priv,
5936 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5937 skl_ddb_entry_write(dev_priv,
5938 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5939}
5940
5941void skl_write_cursor_wm(struct intel_plane *plane,
5942 const struct intel_crtc_state *crtc_state)
5943{
5944 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5945 int level, max_level = ilk_wm_max_level(dev_priv);
5946 enum plane_id plane_id = plane->id;
5947 enum pipe pipe = plane->pipe;
5948 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5949 const struct skl_ddb_entry *ddb =
5950 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5951
5952 for (level = 0; level <= max_level; level++)
5953 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5954 skl_plane_wm_level(pipe_wm, plane_id, level));
5955
5956 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5957 skl_plane_trans_wm(pipe_wm, plane_id));
5958
5959 if (HAS_HW_SAGV_WM(dev_priv)) {
5960 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5961
5962 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5963 &wm->sagv.wm0);
5964 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
5965 &wm->sagv.trans_wm);
5966 }
5967
5968 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5969}
5970
5971bool skl_wm_level_equals(const struct skl_wm_level *l1,
5972 const struct skl_wm_level *l2)
5973{
5974 return l1->enable == l2->enable &&
5975 l1->ignore_lines == l2->ignore_lines &&
5976 l1->lines == l2->lines &&
5977 l1->blocks == l2->blocks;
5978}
5979
5980static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5981 const struct skl_plane_wm *wm1,
5982 const struct skl_plane_wm *wm2)
5983{
5984 int level, max_level = ilk_wm_max_level(dev_priv);
5985
5986 for (level = 0; level <= max_level; level++) {
5987
5988
5989
5990
5991
5992 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5993 return false;
5994 }
5995
5996 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5997 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5998 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5999}
6000
6001static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
6002 const struct skl_ddb_entry *b)
6003{
6004 return a->start < b->end && b->start < a->end;
6005}
6006
6007static void skl_ddb_entry_union(struct skl_ddb_entry *a,
6008 const struct skl_ddb_entry *b)
6009{
6010 if (a->end && b->end) {
6011 a->start = min(a->start, b->start);
6012 a->end = max(a->end, b->end);
6013 } else if (b->end) {
6014 a->start = b->start;
6015 a->end = b->end;
6016 }
6017}
6018
6019bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
6020 const struct skl_ddb_entry *entries,
6021 int num_entries, int ignore_idx)
6022{
6023 int i;
6024
6025 for (i = 0; i < num_entries; i++) {
6026 if (i != ignore_idx &&
6027 skl_ddb_entries_overlap(ddb, &entries[i]))
6028 return true;
6029 }
6030
6031 return false;
6032}
6033
6034static int
6035skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
6036 struct intel_crtc_state *new_crtc_state)
6037{
6038 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
6039 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6040 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6041 struct intel_plane *plane;
6042
6043 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6044 struct intel_plane_state *plane_state;
6045 enum plane_id plane_id = plane->id;
6046
6047 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
6048 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
6049 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
6050 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
6051 continue;
6052
6053 plane_state = intel_atomic_get_plane_state(state, plane);
6054 if (IS_ERR(plane_state))
6055 return PTR_ERR(plane_state);
6056
6057 new_crtc_state->update_planes |= BIT(plane_id);
6058 }
6059
6060 return 0;
6061}
6062
6063static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
6064{
6065 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
6066 u8 enabled_slices;
6067 enum pipe pipe;
6068
6069
6070
6071
6072
6073 enabled_slices = BIT(DBUF_S1);
6074
6075 for_each_pipe(dev_priv, pipe)
6076 enabled_slices |= dbuf_state->slices[pipe];
6077
6078 return enabled_slices;
6079}
6080
6081static int
6082skl_compute_ddb(struct intel_atomic_state *state)
6083{
6084 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6085 const struct intel_dbuf_state *old_dbuf_state;
6086 struct intel_dbuf_state *new_dbuf_state = NULL;
6087 const struct intel_crtc_state *old_crtc_state;
6088 struct intel_crtc_state *new_crtc_state;
6089 struct intel_crtc *crtc;
6090 int ret, i;
6091
6092 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6093 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6094 if (IS_ERR(new_dbuf_state))
6095 return PTR_ERR(new_dbuf_state);
6096
6097 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6098 break;
6099 }
6100
6101 if (!new_dbuf_state)
6102 return 0;
6103
6104 new_dbuf_state->active_pipes =
6105 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6106
6107 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
6108 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6109 if (ret)
6110 return ret;
6111 }
6112
6113 for_each_intel_crtc(&dev_priv->drm, crtc) {
6114 enum pipe pipe = crtc->pipe;
6115
6116 new_dbuf_state->slices[pipe] =
6117 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
6118
6119 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
6120 continue;
6121
6122 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6123 if (ret)
6124 return ret;
6125 }
6126
6127 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6128
6129 if (IS_ALDERLAKE_P(dev_priv))
6130 new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6131
6132 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6133 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6134 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6135 if (ret)
6136 return ret;
6137
6138 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6139
6140 ret = intel_modeset_all_pipes(state);
6141 if (ret)
6142 return ret;
6143 }
6144
6145 drm_dbg_kms(&dev_priv->drm,
6146 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6147 old_dbuf_state->enabled_slices,
6148 new_dbuf_state->enabled_slices,
6149 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6150 yesno(old_dbuf_state->joined_mbus),
6151 yesno(new_dbuf_state->joined_mbus));
6152 }
6153
6154 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6155 enum pipe pipe = crtc->pipe;
6156
6157 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6158
6159 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6160 continue;
6161
6162 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6163 if (ret)
6164 return ret;
6165 }
6166
6167 for_each_intel_crtc(&dev_priv->drm, crtc) {
6168 ret = skl_crtc_allocate_ddb(state, crtc);
6169 if (ret)
6170 return ret;
6171 }
6172
6173 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6174 new_crtc_state, i) {
6175 ret = skl_allocate_plane_ddb(state, crtc);
6176 if (ret)
6177 return ret;
6178
6179 ret = skl_ddb_add_affected_planes(old_crtc_state,
6180 new_crtc_state);
6181 if (ret)
6182 return ret;
6183 }
6184
6185 return 0;
6186}
6187
6188static char enast(bool enable)
6189{
6190 return enable ? '*' : ' ';
6191}
6192
6193static void
6194skl_print_wm_changes(struct intel_atomic_state *state)
6195{
6196 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6197 const struct intel_crtc_state *old_crtc_state;
6198 const struct intel_crtc_state *new_crtc_state;
6199 struct intel_plane *plane;
6200 struct intel_crtc *crtc;
6201 int i;
6202
6203 if (!drm_debug_enabled(DRM_UT_KMS))
6204 return;
6205
6206 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6207 new_crtc_state, i) {
6208 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6209
6210 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6211 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6212
6213 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6214 enum plane_id plane_id = plane->id;
6215 const struct skl_ddb_entry *old, *new;
6216
6217 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
6218 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6219
6220 if (skl_ddb_entry_equal(old, new))
6221 continue;
6222
6223 drm_dbg_kms(&dev_priv->drm,
6224 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6225 plane->base.base.id, plane->base.name,
6226 old->start, old->end, new->start, new->end,
6227 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6228 }
6229
6230 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6231 enum plane_id plane_id = plane->id;
6232 const struct skl_plane_wm *old_wm, *new_wm;
6233
6234 old_wm = &old_pipe_wm->planes[plane_id];
6235 new_wm = &new_pipe_wm->planes[plane_id];
6236
6237 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6238 continue;
6239
6240 drm_dbg_kms(&dev_priv->drm,
6241 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6242 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6243 plane->base.base.id, plane->base.name,
6244 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6245 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6246 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6247 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6248 enast(old_wm->trans_wm.enable),
6249 enast(old_wm->sagv.wm0.enable),
6250 enast(old_wm->sagv.trans_wm.enable),
6251 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6252 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6253 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6254 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6255 enast(new_wm->trans_wm.enable),
6256 enast(new_wm->sagv.wm0.enable),
6257 enast(new_wm->sagv.trans_wm.enable));
6258
6259 drm_dbg_kms(&dev_priv->drm,
6260 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6261 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6262 plane->base.base.id, plane->base.name,
6263 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6264 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6265 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6266 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6267 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6268 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6269 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6270 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6271 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6272 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6273 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6274 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6275 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6276 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6277 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6278 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6279 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6280 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6281 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6282 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6283 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6284 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6285
6286 drm_dbg_kms(&dev_priv->drm,
6287 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6288 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6289 plane->base.base.id, plane->base.name,
6290 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6291 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6292 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6293 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6294 old_wm->trans_wm.blocks,
6295 old_wm->sagv.wm0.blocks,
6296 old_wm->sagv.trans_wm.blocks,
6297 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6298 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6299 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6300 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6301 new_wm->trans_wm.blocks,
6302 new_wm->sagv.wm0.blocks,
6303 new_wm->sagv.trans_wm.blocks);
6304
6305 drm_dbg_kms(&dev_priv->drm,
6306 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6307 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6308 plane->base.base.id, plane->base.name,
6309 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6310 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6311 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6312 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6313 old_wm->trans_wm.min_ddb_alloc,
6314 old_wm->sagv.wm0.min_ddb_alloc,
6315 old_wm->sagv.trans_wm.min_ddb_alloc,
6316 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6317 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6318 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6319 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6320 new_wm->trans_wm.min_ddb_alloc,
6321 new_wm->sagv.wm0.min_ddb_alloc,
6322 new_wm->sagv.trans_wm.min_ddb_alloc);
6323 }
6324 }
6325}
6326
6327static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6328 const struct skl_pipe_wm *old_pipe_wm,
6329 const struct skl_pipe_wm *new_pipe_wm)
6330{
6331 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6332 int level, max_level = ilk_wm_max_level(i915);
6333
6334 for (level = 0; level <= max_level; level++) {
6335
6336
6337
6338
6339
6340 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6341 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6342 return false;
6343 }
6344
6345 if (HAS_HW_SAGV_WM(i915)) {
6346 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6347 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6348
6349 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6350 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6351 return false;
6352 }
6353
6354 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6355 skl_plane_trans_wm(new_pipe_wm, plane->id));
6356}
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6381 struct intel_crtc *crtc)
6382{
6383 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6384 const struct intel_crtc_state *old_crtc_state =
6385 intel_atomic_get_old_crtc_state(state, crtc);
6386 struct intel_crtc_state *new_crtc_state =
6387 intel_atomic_get_new_crtc_state(state, crtc);
6388 struct intel_plane *plane;
6389
6390 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6391 struct intel_plane_state *plane_state;
6392 enum plane_id plane_id = plane->id;
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6403 skl_plane_selected_wm_equals(plane,
6404 &old_crtc_state->wm.skl.optimal,
6405 &new_crtc_state->wm.skl.optimal))
6406 continue;
6407
6408 plane_state = intel_atomic_get_plane_state(state, plane);
6409 if (IS_ERR(plane_state))
6410 return PTR_ERR(plane_state);
6411
6412 new_crtc_state->update_planes |= BIT(plane_id);
6413 }
6414
6415 return 0;
6416}
6417
6418static int
6419skl_compute_wm(struct intel_atomic_state *state)
6420{
6421 struct intel_crtc *crtc;
6422 struct intel_crtc_state *new_crtc_state;
6423 int ret, i;
6424
6425 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6426 ret = skl_build_pipe_wm(state, crtc);
6427 if (ret)
6428 return ret;
6429 }
6430
6431 ret = skl_compute_ddb(state);
6432 if (ret)
6433 return ret;
6434
6435 ret = intel_compute_sagv_mask(state);
6436 if (ret)
6437 return ret;
6438
6439
6440
6441
6442
6443
6444 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6445 ret = skl_wm_add_affected_planes(state, crtc);
6446 if (ret)
6447 return ret;
6448 }
6449
6450 skl_print_wm_changes(state);
6451
6452 return 0;
6453}
6454
6455static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6456 struct intel_wm_config *config)
6457{
6458 struct intel_crtc *crtc;
6459
6460
6461 for_each_intel_crtc(&dev_priv->drm, crtc) {
6462 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6463
6464 if (!wm->pipe_enabled)
6465 continue;
6466
6467 config->sprites_enabled |= wm->sprites_enabled;
6468 config->sprites_scaled |= wm->sprites_scaled;
6469 config->num_pipes_active++;
6470 }
6471}
6472
6473static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6474{
6475 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6476 struct ilk_wm_maximums max;
6477 struct intel_wm_config config = {};
6478 struct ilk_wm_values results = {};
6479 enum intel_ddb_partitioning partitioning;
6480
6481 ilk_compute_wm_config(dev_priv, &config);
6482
6483 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6484 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6485
6486
6487 if (DISPLAY_VER(dev_priv) >= 7 &&
6488 config.num_pipes_active == 1 && config.sprites_enabled) {
6489 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6490 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6491
6492 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6493 } else {
6494 best_lp_wm = &lp_wm_1_2;
6495 }
6496
6497 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6498 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6499
6500 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6501
6502 ilk_write_wm_values(dev_priv, &results);
6503}
6504
6505static void ilk_initial_watermarks(struct intel_atomic_state *state,
6506 struct intel_crtc *crtc)
6507{
6508 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6509 const struct intel_crtc_state *crtc_state =
6510 intel_atomic_get_new_crtc_state(state, crtc);
6511
6512 mutex_lock(&dev_priv->wm.wm_mutex);
6513 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6514 ilk_program_watermarks(dev_priv);
6515 mutex_unlock(&dev_priv->wm.wm_mutex);
6516}
6517
6518static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6519 struct intel_crtc *crtc)
6520{
6521 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6522 const struct intel_crtc_state *crtc_state =
6523 intel_atomic_get_new_crtc_state(state, crtc);
6524
6525 if (!crtc_state->wm.need_postvbl_update)
6526 return;
6527
6528 mutex_lock(&dev_priv->wm.wm_mutex);
6529 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6530 ilk_program_watermarks(dev_priv);
6531 mutex_unlock(&dev_priv->wm.wm_mutex);
6532}
6533
6534static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6535{
6536 level->enable = val & PLANE_WM_EN;
6537 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6538 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6539 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6540}
6541
6542void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6543 struct skl_pipe_wm *out)
6544{
6545 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6546 enum pipe pipe = crtc->pipe;
6547 int level, max_level;
6548 enum plane_id plane_id;
6549 u32 val;
6550
6551 max_level = ilk_wm_max_level(dev_priv);
6552
6553 for_each_plane_id_on_crtc(crtc, plane_id) {
6554 struct skl_plane_wm *wm = &out->planes[plane_id];
6555
6556 for (level = 0; level <= max_level; level++) {
6557 if (plane_id != PLANE_CURSOR)
6558 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6559 else
6560 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6561
6562 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6563 }
6564
6565 if (plane_id != PLANE_CURSOR)
6566 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6567 else
6568 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6569
6570 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6571
6572 if (HAS_HW_SAGV_WM(dev_priv)) {
6573 if (plane_id != PLANE_CURSOR)
6574 val = intel_uncore_read(&dev_priv->uncore,
6575 PLANE_WM_SAGV(pipe, plane_id));
6576 else
6577 val = intel_uncore_read(&dev_priv->uncore,
6578 CUR_WM_SAGV(pipe));
6579
6580 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6581
6582 if (plane_id != PLANE_CURSOR)
6583 val = intel_uncore_read(&dev_priv->uncore,
6584 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6585 else
6586 val = intel_uncore_read(&dev_priv->uncore,
6587 CUR_WM_SAGV_TRANS(pipe));
6588
6589 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6590 } else if (DISPLAY_VER(dev_priv) >= 12) {
6591 wm->sagv.wm0 = wm->wm[0];
6592 wm->sagv.trans_wm = wm->trans_wm;
6593 }
6594 }
6595}
6596
6597void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6598{
6599 struct intel_dbuf_state *dbuf_state =
6600 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6601 struct intel_crtc *crtc;
6602
6603 if (IS_ALDERLAKE_P(dev_priv))
6604 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6605
6606 for_each_intel_crtc(&dev_priv->drm, crtc) {
6607 struct intel_crtc_state *crtc_state =
6608 to_intel_crtc_state(crtc->base.state);
6609 enum pipe pipe = crtc->pipe;
6610 unsigned int mbus_offset;
6611 enum plane_id plane_id;
6612
6613 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6614 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6615
6616 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6617
6618 for_each_plane_id_on_crtc(crtc, plane_id) {
6619 struct skl_ddb_entry *ddb_y =
6620 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6621 struct skl_ddb_entry *ddb_uv =
6622 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6623
6624 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6625 plane_id, ddb_y, ddb_uv);
6626
6627 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6628 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6629 }
6630
6631 dbuf_state->slices[pipe] =
6632 skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6633
6634 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6635
6636
6637
6638
6639
6640 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]);
6641 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6642 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6643
6644 drm_dbg_kms(&dev_priv->drm,
6645 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6646 crtc->base.base.id, crtc->base.name,
6647 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6648 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6649 yesno(dbuf_state->joined_mbus));
6650 }
6651
6652 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6653}
6654
6655static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6656{
6657 struct drm_device *dev = crtc->base.dev;
6658 struct drm_i915_private *dev_priv = to_i915(dev);
6659 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6660 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6661 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6662 enum pipe pipe = crtc->pipe;
6663
6664 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6665
6666 memset(active, 0, sizeof(*active));
6667
6668 active->pipe_enabled = crtc->active;
6669
6670 if (active->pipe_enabled) {
6671 u32 tmp = hw->wm_pipe[pipe];
6672
6673
6674
6675
6676
6677
6678
6679 active->wm[0].enable = true;
6680 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6681 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6682 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6683 } else {
6684 int level, max_level = ilk_wm_max_level(dev_priv);
6685
6686
6687
6688
6689
6690
6691 for (level = 0; level <= max_level; level++)
6692 active->wm[level].enable = true;
6693 }
6694
6695 crtc->wm.active.ilk = *active;
6696}
6697
6698#define _FW_WM(value, plane) \
6699 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6700#define _FW_WM_VLV(value, plane) \
6701 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6702
6703static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6704 struct g4x_wm_values *wm)
6705{
6706 u32 tmp;
6707
6708 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6709 wm->sr.plane = _FW_WM(tmp, SR);
6710 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6711 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6712 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6713
6714 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6715 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6716 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6717 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6718 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6719 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6720 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6721
6722 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6723 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6724 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6725 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6726 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6727}
6728
6729static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6730 struct vlv_wm_values *wm)
6731{
6732 enum pipe pipe;
6733 u32 tmp;
6734
6735 for_each_pipe(dev_priv, pipe) {
6736 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6737
6738 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6739 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6740 wm->ddl[pipe].plane[PLANE_CURSOR] =
6741 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6742 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6743 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6744 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6745 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6746 }
6747
6748 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6749 wm->sr.plane = _FW_WM(tmp, SR);
6750 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6751 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6752 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6753
6754 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6755 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6756 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6757 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6758
6759 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6760 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6761
6762 if (IS_CHERRYVIEW(dev_priv)) {
6763 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6764 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6765 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6766
6767 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6768 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6769 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6770
6771 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6772 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6773 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6774
6775 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6776 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6777 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6778 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6779 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6780 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6781 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6782 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6783 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6784 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6785 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6786 } else {
6787 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6788 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6789 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6790
6791 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6792 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6793 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6794 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6795 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6796 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6797 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6798 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6799 }
6800}
6801
6802#undef _FW_WM
6803#undef _FW_WM_VLV
6804
6805void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6806{
6807 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6808 struct intel_crtc *crtc;
6809
6810 g4x_read_wm_values(dev_priv, wm);
6811
6812 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6813
6814 for_each_intel_crtc(&dev_priv->drm, crtc) {
6815 struct intel_crtc_state *crtc_state =
6816 to_intel_crtc_state(crtc->base.state);
6817 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6818 struct g4x_pipe_wm *raw;
6819 enum pipe pipe = crtc->pipe;
6820 enum plane_id plane_id;
6821 int level, max_level;
6822
6823 active->cxsr = wm->cxsr;
6824 active->hpll_en = wm->hpll_en;
6825 active->fbc_en = wm->fbc_en;
6826
6827 active->sr = wm->sr;
6828 active->hpll = wm->hpll;
6829
6830 for_each_plane_id_on_crtc(crtc, plane_id) {
6831 active->wm.plane[plane_id] =
6832 wm->pipe[pipe].plane[plane_id];
6833 }
6834
6835 if (wm->cxsr && wm->hpll_en)
6836 max_level = G4X_WM_LEVEL_HPLL;
6837 else if (wm->cxsr)
6838 max_level = G4X_WM_LEVEL_SR;
6839 else
6840 max_level = G4X_WM_LEVEL_NORMAL;
6841
6842 level = G4X_WM_LEVEL_NORMAL;
6843 raw = &crtc_state->wm.g4x.raw[level];
6844 for_each_plane_id_on_crtc(crtc, plane_id)
6845 raw->plane[plane_id] = active->wm.plane[plane_id];
6846
6847 if (++level > max_level)
6848 goto out;
6849
6850 raw = &crtc_state->wm.g4x.raw[level];
6851 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6852 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6853 raw->plane[PLANE_SPRITE0] = 0;
6854 raw->fbc = active->sr.fbc;
6855
6856 if (++level > max_level)
6857 goto out;
6858
6859 raw = &crtc_state->wm.g4x.raw[level];
6860 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6861 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6862 raw->plane[PLANE_SPRITE0] = 0;
6863 raw->fbc = active->hpll.fbc;
6864
6865 out:
6866 for_each_plane_id_on_crtc(crtc, plane_id)
6867 g4x_raw_plane_wm_set(crtc_state, level,
6868 plane_id, USHRT_MAX);
6869 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6870
6871 crtc_state->wm.g4x.optimal = *active;
6872 crtc_state->wm.g4x.intermediate = *active;
6873
6874 drm_dbg_kms(&dev_priv->drm,
6875 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6876 pipe_name(pipe),
6877 wm->pipe[pipe].plane[PLANE_PRIMARY],
6878 wm->pipe[pipe].plane[PLANE_CURSOR],
6879 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6880 }
6881
6882 drm_dbg_kms(&dev_priv->drm,
6883 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6884 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6885 drm_dbg_kms(&dev_priv->drm,
6886 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6887 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6888 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6889 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6890}
6891
6892void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6893{
6894 struct intel_plane *plane;
6895 struct intel_crtc *crtc;
6896
6897 mutex_lock(&dev_priv->wm.wm_mutex);
6898
6899 for_each_intel_plane(&dev_priv->drm, plane) {
6900 struct intel_crtc *crtc =
6901 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6902 struct intel_crtc_state *crtc_state =
6903 to_intel_crtc_state(crtc->base.state);
6904 struct intel_plane_state *plane_state =
6905 to_intel_plane_state(plane->base.state);
6906 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6907 enum plane_id plane_id = plane->id;
6908 int level;
6909
6910 if (plane_state->uapi.visible)
6911 continue;
6912
6913 for (level = 0; level < 3; level++) {
6914 struct g4x_pipe_wm *raw =
6915 &crtc_state->wm.g4x.raw[level];
6916
6917 raw->plane[plane_id] = 0;
6918 wm_state->wm.plane[plane_id] = 0;
6919 }
6920
6921 if (plane_id == PLANE_PRIMARY) {
6922 for (level = 0; level < 3; level++) {
6923 struct g4x_pipe_wm *raw =
6924 &crtc_state->wm.g4x.raw[level];
6925 raw->fbc = 0;
6926 }
6927
6928 wm_state->sr.fbc = 0;
6929 wm_state->hpll.fbc = 0;
6930 wm_state->fbc_en = false;
6931 }
6932 }
6933
6934 for_each_intel_crtc(&dev_priv->drm, crtc) {
6935 struct intel_crtc_state *crtc_state =
6936 to_intel_crtc_state(crtc->base.state);
6937
6938 crtc_state->wm.g4x.intermediate =
6939 crtc_state->wm.g4x.optimal;
6940 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6941 }
6942
6943 g4x_program_watermarks(dev_priv);
6944
6945 mutex_unlock(&dev_priv->wm.wm_mutex);
6946}
6947
6948void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6949{
6950 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6951 struct intel_crtc *crtc;
6952 u32 val;
6953
6954 vlv_read_wm_values(dev_priv, wm);
6955
6956 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6957 wm->level = VLV_WM_LEVEL_PM2;
6958
6959 if (IS_CHERRYVIEW(dev_priv)) {
6960 vlv_punit_get(dev_priv);
6961
6962 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6963 if (val & DSP_MAXFIFO_PM5_ENABLE)
6964 wm->level = VLV_WM_LEVEL_PM5;
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6976 val |= FORCE_DDR_FREQ_REQ_ACK;
6977 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6978
6979 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6980 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6981 drm_dbg_kms(&dev_priv->drm,
6982 "Punit not acking DDR DVFS request, "
6983 "assuming DDR DVFS is disabled\n");
6984 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6985 } else {
6986 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6987 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6988 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6989 }
6990
6991 vlv_punit_put(dev_priv);
6992 }
6993
6994 for_each_intel_crtc(&dev_priv->drm, crtc) {
6995 struct intel_crtc_state *crtc_state =
6996 to_intel_crtc_state(crtc->base.state);
6997 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6998 const struct vlv_fifo_state *fifo_state =
6999 &crtc_state->wm.vlv.fifo_state;
7000 enum pipe pipe = crtc->pipe;
7001 enum plane_id plane_id;
7002 int level;
7003
7004 vlv_get_fifo_size(crtc_state);
7005
7006 active->num_levels = wm->level + 1;
7007 active->cxsr = wm->cxsr;
7008
7009 for (level = 0; level < active->num_levels; level++) {
7010 struct g4x_pipe_wm *raw =
7011 &crtc_state->wm.vlv.raw[level];
7012
7013 active->sr[level].plane = wm->sr.plane;
7014 active->sr[level].cursor = wm->sr.cursor;
7015
7016 for_each_plane_id_on_crtc(crtc, plane_id) {
7017 active->wm[level].plane[plane_id] =
7018 wm->pipe[pipe].plane[plane_id];
7019
7020 raw->plane[plane_id] =
7021 vlv_invert_wm_value(active->wm[level].plane[plane_id],
7022 fifo_state->plane[plane_id]);
7023 }
7024 }
7025
7026 for_each_plane_id_on_crtc(crtc, plane_id)
7027 vlv_raw_plane_wm_set(crtc_state, level,
7028 plane_id, USHRT_MAX);
7029 vlv_invalidate_wms(crtc, active, level);
7030
7031 crtc_state->wm.vlv.optimal = *active;
7032 crtc_state->wm.vlv.intermediate = *active;
7033
7034 drm_dbg_kms(&dev_priv->drm,
7035 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
7036 pipe_name(pipe),
7037 wm->pipe[pipe].plane[PLANE_PRIMARY],
7038 wm->pipe[pipe].plane[PLANE_CURSOR],
7039 wm->pipe[pipe].plane[PLANE_SPRITE0],
7040 wm->pipe[pipe].plane[PLANE_SPRITE1]);
7041 }
7042
7043 drm_dbg_kms(&dev_priv->drm,
7044 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
7045 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7046}
7047
7048void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
7049{
7050 struct intel_plane *plane;
7051 struct intel_crtc *crtc;
7052
7053 mutex_lock(&dev_priv->wm.wm_mutex);
7054
7055 for_each_intel_plane(&dev_priv->drm, plane) {
7056 struct intel_crtc *crtc =
7057 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
7058 struct intel_crtc_state *crtc_state =
7059 to_intel_crtc_state(crtc->base.state);
7060 struct intel_plane_state *plane_state =
7061 to_intel_plane_state(plane->base.state);
7062 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
7063 const struct vlv_fifo_state *fifo_state =
7064 &crtc_state->wm.vlv.fifo_state;
7065 enum plane_id plane_id = plane->id;
7066 int level;
7067
7068 if (plane_state->uapi.visible)
7069 continue;
7070
7071 for (level = 0; level < wm_state->num_levels; level++) {
7072 struct g4x_pipe_wm *raw =
7073 &crtc_state->wm.vlv.raw[level];
7074
7075 raw->plane[plane_id] = 0;
7076
7077 wm_state->wm[level].plane[plane_id] =
7078 vlv_invert_wm_value(raw->plane[plane_id],
7079 fifo_state->plane[plane_id]);
7080 }
7081 }
7082
7083 for_each_intel_crtc(&dev_priv->drm, crtc) {
7084 struct intel_crtc_state *crtc_state =
7085 to_intel_crtc_state(crtc->base.state);
7086
7087 crtc_state->wm.vlv.intermediate =
7088 crtc_state->wm.vlv.optimal;
7089 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
7090 }
7091
7092 vlv_program_watermarks(dev_priv);
7093
7094 mutex_unlock(&dev_priv->wm.wm_mutex);
7095}
7096
7097
7098
7099
7100
7101static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7102{
7103 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
7104 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
7105 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
7106
7107
7108
7109
7110
7111}
7112
7113void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7114{
7115 struct ilk_wm_values *hw = &dev_priv->wm.hw;
7116 struct intel_crtc *crtc;
7117
7118 ilk_init_lp_watermarks(dev_priv);
7119
7120 for_each_intel_crtc(&dev_priv->drm, crtc)
7121 ilk_pipe_wm_get_hw_state(crtc);
7122
7123 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7124 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7125 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7126
7127 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7128 if (DISPLAY_VER(dev_priv) >= 7) {
7129 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7130 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7131 }
7132
7133 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7134 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7135 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7136 else if (IS_IVYBRIDGE(dev_priv))
7137 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7138 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7139
7140 hw->enable_fbc_wm =
7141 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7142}
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177void intel_update_watermarks(struct intel_crtc *crtc)
7178{
7179 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7180
7181 if (dev_priv->display.update_wm)
7182 dev_priv->display.update_wm(crtc);
7183}
7184
7185void intel_enable_ipc(struct drm_i915_private *dev_priv)
7186{
7187 u32 val;
7188
7189 if (!HAS_IPC(dev_priv))
7190 return;
7191
7192 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7193
7194 if (dev_priv->ipc_enabled)
7195 val |= DISP_IPC_ENABLE;
7196 else
7197 val &= ~DISP_IPC_ENABLE;
7198
7199 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7200}
7201
7202static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7203{
7204
7205 if (IS_SKYLAKE(dev_priv))
7206 return false;
7207
7208
7209 if (IS_KABYLAKE(dev_priv) ||
7210 IS_COFFEELAKE(dev_priv) ||
7211 IS_COMETLAKE(dev_priv))
7212 return dev_priv->dram_info.symmetric_memory;
7213
7214 return true;
7215}
7216
7217void intel_init_ipc(struct drm_i915_private *dev_priv)
7218{
7219 if (!HAS_IPC(dev_priv))
7220 return;
7221
7222 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7223
7224 intel_enable_ipc(dev_priv);
7225}
7226
7227static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7228{
7229
7230
7231
7232
7233
7234 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7235}
7236
7237static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7238{
7239 enum pipe pipe;
7240
7241 for_each_pipe(dev_priv, pipe) {
7242 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7243 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7244 DISPPLANE_TRICKLE_FEED_DISABLE);
7245
7246 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7247 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7248 }
7249}
7250
7251static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7252{
7253 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7254
7255
7256
7257
7258
7259 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7260 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7261 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7262
7263 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7264 MARIUNIT_CLOCK_GATE_DISABLE |
7265 SVSMUNIT_CLOCK_GATE_DISABLE);
7266 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7267 VFMUNIT_CLOCK_GATE_DISABLE);
7268
7269
7270
7271
7272
7273
7274
7275
7276 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7277 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7278 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7279 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7280 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7281 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7282 DISP_FBC_WM_DIS));
7283
7284
7285
7286
7287
7288
7289
7290
7291 if (IS_IRONLAKE_M(dev_priv)) {
7292
7293 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7294 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7295 ILK_FBCQ_DIS);
7296 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7297 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7298 ILK_DPARB_GATE);
7299 }
7300
7301 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7302
7303 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7304 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7305 ILK_ELPIN_409_SELECT);
7306
7307 g4x_disable_trickle_feed(dev_priv);
7308
7309 ibx_init_clock_gating(dev_priv);
7310}
7311
7312static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7313{
7314 enum pipe pipe;
7315 u32 val;
7316
7317
7318
7319
7320
7321
7322 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7323 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7324 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7325 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7326 DPLS_EDP_PPS_FIX_DIS);
7327
7328
7329
7330 for_each_pipe(dev_priv, pipe) {
7331 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7332 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7333 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7334 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7335 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7336 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7337 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7338 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7339 }
7340
7341 for_each_pipe(dev_priv, pipe) {
7342 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7343 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7344 }
7345}
7346
7347static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7348{
7349 u32 tmp;
7350
7351 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7352 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
7353 drm_dbg_kms(&dev_priv->drm,
7354 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7355 tmp);
7356}
7357
7358static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7359{
7360 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7361
7362 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7363
7364 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7365 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7366 ILK_ELPIN_409_SELECT);
7367
7368 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7369 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7370 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7371 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7387 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7388 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7402 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7403 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7404 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7405 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7406 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7407 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7408 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7409 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7410 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7411
7412 g4x_disable_trickle_feed(dev_priv);
7413
7414 cpt_init_clock_gating(dev_priv);
7415
7416 gen6_check_mch_setup(dev_priv);
7417}
7418
7419static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7420{
7421
7422
7423
7424
7425 if (HAS_PCH_LPT_LP(dev_priv))
7426 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7427 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7428 PCH_LP_PARTITION_LEVEL_DISABLE);
7429
7430
7431 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7432 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7433 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7434}
7435
7436static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7437{
7438 if (HAS_PCH_LPT_LP(dev_priv)) {
7439 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7440
7441 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7442 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7443 }
7444}
7445
7446static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7447 int general_prio_credits,
7448 int high_prio_credits)
7449{
7450 u32 misccpctl;
7451 u32 val;
7452
7453
7454 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7455 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7456
7457 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7458 val &= ~L3_PRIO_CREDITS_MASK;
7459 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7460 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7461 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7462
7463
7464
7465
7466
7467 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7468 udelay(1);
7469 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7470}
7471
7472static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7473{
7474
7475 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7476 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7477
7478
7479 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7480 0, ICL_DELAY_PMRSP);
7481}
7482
7483static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7484{
7485
7486 if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
7487 IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv))
7488 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7489 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7490
7491
7492 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7493 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7494 TGL_VRH_GATING_DIS);
7495
7496
7497 if (DISPLAY_VER(dev_priv) == 12)
7498 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7499 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7500}
7501
7502static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7503{
7504 gen12lp_init_clock_gating(dev_priv);
7505
7506
7507 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7508}
7509
7510static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7511{
7512 gen12lp_init_clock_gating(dev_priv);
7513
7514
7515 if (IS_DG1_GT_STEP(dev_priv, STEP_A0, STEP_B0))
7516 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7517 DPT_GATING_DIS);
7518}
7519
7520static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7521{
7522 if (!HAS_PCH_CNP(dev_priv))
7523 return;
7524
7525
7526 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7527 CNP_PWM_CGE_GATING_DISABLE);
7528}
7529
7530static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7531{
7532 cnp_init_clock_gating(dev_priv);
7533 gen9_init_clock_gating(dev_priv);
7534
7535
7536 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7537 FBC_LLC_FULLY_OPEN);
7538
7539
7540
7541
7542
7543 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7544 DISP_FBC_WM_DIS);
7545
7546
7547
7548
7549
7550 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7551 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7552}
7553
7554static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7555{
7556 gen9_init_clock_gating(dev_priv);
7557
7558
7559 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7560 FBC_LLC_FULLY_OPEN);
7561
7562
7563 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_C0))
7564 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7565 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7566
7567
7568 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_C0))
7569 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7570 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7571
7572
7573
7574
7575
7576 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7577 DISP_FBC_WM_DIS);
7578
7579
7580
7581
7582
7583 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7584 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7585}
7586
7587static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7588{
7589 gen9_init_clock_gating(dev_priv);
7590
7591
7592 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7593 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7594
7595
7596 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7597 FBC_LLC_FULLY_OPEN);
7598
7599
7600
7601
7602
7603 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7604 DISP_FBC_WM_DIS);
7605
7606
7607
7608
7609
7610 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7611 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7612
7613
7614
7615
7616
7617 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7618 ILK_DPFC_DISABLE_DUMMY0);
7619}
7620
7621static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7622{
7623 enum pipe pipe;
7624
7625
7626 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7627 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7628 HSW_FBCQ_DIS);
7629
7630
7631 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7632
7633
7634 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7635 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7636
7637 for_each_pipe(dev_priv, pipe) {
7638
7639 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7640 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7641 BDW_DPRS_MASK_VBLANK_SRD);
7642
7643
7644 if (intel_vtd_active())
7645 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7646 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7647 }
7648
7649
7650
7651 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7652 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7653 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7654
7655 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7656 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7657
7658
7659 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7660 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7661
7662
7663 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7664
7665
7666 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7667 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7668
7669 lpt_init_clock_gating(dev_priv);
7670
7671
7672
7673
7674
7675
7676 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7677 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7678}
7679
7680static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7681{
7682 enum pipe pipe;
7683
7684
7685 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7686 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7687 HSW_FBCQ_DIS);
7688
7689 for_each_pipe(dev_priv, pipe) {
7690
7691 if (intel_vtd_active())
7692 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7693 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7694 }
7695
7696
7697 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7698 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7699 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7700
7701
7702 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7703
7704 lpt_init_clock_gating(dev_priv);
7705}
7706
7707static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7708{
7709 u32 snpcr;
7710
7711 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7712
7713
7714 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7715 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7716 ILK_FBCQ_DIS);
7717
7718
7719 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7720 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7721 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7722
7723 if (IS_IVB_GT1(dev_priv))
7724 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7725 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7726 else {
7727
7728 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7729 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7730 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7731 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7732 }
7733
7734
7735
7736
7737
7738 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7739 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7740
7741
7742 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7743 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7744 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7745
7746 g4x_disable_trickle_feed(dev_priv);
7747
7748 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7749 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7750 snpcr |= GEN6_MBC_SNPCR_MED;
7751 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7752
7753 if (!HAS_PCH_NOP(dev_priv))
7754 cpt_init_clock_gating(dev_priv);
7755
7756 gen6_check_mch_setup(dev_priv);
7757}
7758
7759static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7760{
7761
7762 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7763 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7764 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7765
7766
7767 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7768 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7769
7770
7771 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7772 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7773 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7774
7775
7776
7777
7778
7779 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7780 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7781
7782
7783
7784
7785 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7786 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7787
7788
7789
7790
7791
7792
7793 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7794}
7795
7796static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7797{
7798
7799
7800 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7801 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7802 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7803
7804
7805 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7806 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7807
7808
7809 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7810 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7811
7812
7813 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7814 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7815
7816
7817
7818
7819
7820
7821 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7822}
7823
7824static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7825{
7826 u32 dspclk_gate;
7827
7828 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7829 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7830 GS_UNIT_CLOCK_GATE_DISABLE |
7831 CL_UNIT_CLOCK_GATE_DISABLE);
7832 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7833 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7834 OVRUNIT_CLOCK_GATE_DISABLE |
7835 OVCUNIT_CLOCK_GATE_DISABLE;
7836 if (IS_GM45(dev_priv))
7837 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7838 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7839
7840 g4x_disable_trickle_feed(dev_priv);
7841}
7842
7843static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7844{
7845 struct intel_uncore *uncore = &dev_priv->uncore;
7846
7847 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7848 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7849 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7850 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7851 intel_uncore_write16(uncore, DEUC, 0);
7852 intel_uncore_write(uncore,
7853 MI_ARB_STATE,
7854 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7855}
7856
7857static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7858{
7859 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7860 I965_RCC_CLOCK_GATE_DISABLE |
7861 I965_RCPB_CLOCK_GATE_DISABLE |
7862 I965_ISC_CLOCK_GATE_DISABLE |
7863 I965_FBC_CLOCK_GATE_DISABLE);
7864 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7865 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7866 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7867}
7868
7869static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7870{
7871 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7872
7873 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7874 DSTATE_DOT_CLOCK_GATING;
7875 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7876
7877 if (IS_PINEVIEW(dev_priv))
7878 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7879
7880
7881 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7882
7883
7884 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7885
7886
7887 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7888
7889 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7890 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7891}
7892
7893static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7894{
7895 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7896
7897
7898 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7899 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7900
7901 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7902 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7903
7904
7905
7906
7907
7908
7909
7910
7911 intel_uncore_write(&dev_priv->uncore, SCPD0,
7912 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7913}
7914
7915static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7916{
7917 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7918 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7919 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7920}
7921
7922void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7923{
7924 dev_priv->display.init_clock_gating(dev_priv);
7925}
7926
7927void intel_suspend_hw(struct drm_i915_private *dev_priv)
7928{
7929 if (HAS_PCH_LPT(dev_priv))
7930 lpt_suspend_hw(dev_priv);
7931}
7932
7933static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7934{
7935 drm_dbg_kms(&dev_priv->drm,
7936 "No clock gating settings or workarounds applied.\n");
7937}
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7949{
7950 if (IS_ALDERLAKE_P(dev_priv))
7951 dev_priv->display.init_clock_gating = adlp_init_clock_gating;
7952 else if (IS_DG1(dev_priv))
7953 dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7954 else if (GRAPHICS_VER(dev_priv) == 12)
7955 dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
7956 else if (GRAPHICS_VER(dev_priv) == 11)
7957 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7958 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7959 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7960 else if (IS_SKYLAKE(dev_priv))
7961 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7962 else if (IS_KABYLAKE(dev_priv))
7963 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7964 else if (IS_BROXTON(dev_priv))
7965 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7966 else if (IS_GEMINILAKE(dev_priv))
7967 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7968 else if (IS_BROADWELL(dev_priv))
7969 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7970 else if (IS_CHERRYVIEW(dev_priv))
7971 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7972 else if (IS_HASWELL(dev_priv))
7973 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7974 else if (IS_IVYBRIDGE(dev_priv))
7975 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7976 else if (IS_VALLEYVIEW(dev_priv))
7977 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7978 else if (GRAPHICS_VER(dev_priv) == 6)
7979 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7980 else if (GRAPHICS_VER(dev_priv) == 5)
7981 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7982 else if (IS_G4X(dev_priv))
7983 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7984 else if (IS_I965GM(dev_priv))
7985 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7986 else if (IS_I965G(dev_priv))
7987 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7988 else if (GRAPHICS_VER(dev_priv) == 3)
7989 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7990 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7991 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7992 else if (GRAPHICS_VER(dev_priv) == 2)
7993 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7994 else {
7995 MISSING_CASE(INTEL_DEVID(dev_priv));
7996 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7997 }
7998}
7999
8000
8001void intel_init_pm(struct drm_i915_private *dev_priv)
8002{
8003
8004 if (IS_PINEVIEW(dev_priv))
8005 pnv_get_mem_freq(dev_priv);
8006 else if (GRAPHICS_VER(dev_priv) == 5)
8007 ilk_get_mem_freq(dev_priv);
8008
8009 if (intel_has_sagv(dev_priv))
8010 skl_setup_sagv_block_time(dev_priv);
8011
8012
8013 if (DISPLAY_VER(dev_priv) >= 9) {
8014 skl_setup_wm_latency(dev_priv);
8015 dev_priv->display.compute_global_watermarks = skl_compute_wm;
8016 } else if (HAS_PCH_SPLIT(dev_priv)) {
8017 ilk_setup_wm_latency(dev_priv);
8018
8019 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
8020 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8021 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
8022 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8023 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
8024 dev_priv->display.compute_intermediate_wm =
8025 ilk_compute_intermediate_wm;
8026 dev_priv->display.initial_watermarks =
8027 ilk_initial_watermarks;
8028 dev_priv->display.optimize_watermarks =
8029 ilk_optimize_watermarks;
8030 } else {
8031 drm_dbg_kms(&dev_priv->drm,
8032 "Failed to read display plane latency. "
8033 "Disable CxSR\n");
8034 }
8035 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8036 vlv_setup_wm_latency(dev_priv);
8037 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
8038 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
8039 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
8040 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
8041 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
8042 } else if (IS_G4X(dev_priv)) {
8043 g4x_setup_wm_latency(dev_priv);
8044 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
8045 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
8046 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
8047 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
8048 } else if (IS_PINEVIEW(dev_priv)) {
8049 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8050 dev_priv->is_ddr3,
8051 dev_priv->fsb_freq,
8052 dev_priv->mem_freq)) {
8053 drm_info(&dev_priv->drm,
8054 "failed to find known CxSR latency "
8055 "(found ddr%s fsb freq %d, mem freq %d), "
8056 "disabling CxSR\n",
8057 (dev_priv->is_ddr3 == 1) ? "3" : "2",
8058 dev_priv->fsb_freq, dev_priv->mem_freq);
8059
8060 intel_set_memory_cxsr(dev_priv, false);
8061 dev_priv->display.update_wm = NULL;
8062 } else
8063 dev_priv->display.update_wm = pnv_update_wm;
8064 } else if (DISPLAY_VER(dev_priv) == 4) {
8065 dev_priv->display.update_wm = i965_update_wm;
8066 } else if (DISPLAY_VER(dev_priv) == 3) {
8067 dev_priv->display.update_wm = i9xx_update_wm;
8068 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
8069 } else if (DISPLAY_VER(dev_priv) == 2) {
8070 if (INTEL_NUM_PIPES(dev_priv) == 1) {
8071 dev_priv->display.update_wm = i845_update_wm;
8072 dev_priv->display.get_fifo_size = i845_get_fifo_size;
8073 } else {
8074 dev_priv->display.update_wm = i9xx_update_wm;
8075 dev_priv->display.get_fifo_size = i830_get_fifo_size;
8076 }
8077 } else {
8078 drm_err(&dev_priv->drm,
8079 "unexpected fall-through in %s\n", __func__);
8080 }
8081}
8082
8083void intel_pm_setup(struct drm_i915_private *dev_priv)
8084{
8085 dev_priv->runtime_pm.suspended = false;
8086 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8087}
8088
8089static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8090{
8091 struct intel_dbuf_state *dbuf_state;
8092
8093 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8094 if (!dbuf_state)
8095 return NULL;
8096
8097 return &dbuf_state->base;
8098}
8099
8100static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8101 struct intel_global_state *state)
8102{
8103 kfree(state);
8104}
8105
8106static const struct intel_global_state_funcs intel_dbuf_funcs = {
8107 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8108 .atomic_destroy_state = intel_dbuf_destroy_state,
8109};
8110
8111struct intel_dbuf_state *
8112intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8113{
8114 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8115 struct intel_global_state *dbuf_state;
8116
8117 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8118 if (IS_ERR(dbuf_state))
8119 return ERR_CAST(dbuf_state);
8120
8121 return to_intel_dbuf_state(dbuf_state);
8122}
8123
8124int intel_dbuf_init(struct drm_i915_private *dev_priv)
8125{
8126 struct intel_dbuf_state *dbuf_state;
8127
8128 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8129 if (!dbuf_state)
8130 return -ENOMEM;
8131
8132 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8133 &dbuf_state->base, &intel_dbuf_funcs);
8134
8135 return 0;
8136}
8137
8138
8139
8140
8141
8142static void update_mbus_pre_enable(struct intel_atomic_state *state)
8143{
8144 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8145 u32 mbus_ctl, dbuf_min_tracker_val;
8146 enum dbuf_slice slice;
8147 const struct intel_dbuf_state *dbuf_state =
8148 intel_atomic_get_new_dbuf_state(state);
8149
8150 if (!IS_ALDERLAKE_P(dev_priv))
8151 return;
8152
8153
8154
8155
8156
8157 if (dbuf_state->joined_mbus) {
8158 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8159 MBUS_JOIN_PIPE_SELECT_NONE;
8160 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8161 } else {
8162 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8163 MBUS_JOIN_PIPE_SELECT_NONE;
8164 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8165 }
8166
8167 intel_de_rmw(dev_priv, MBUS_CTL,
8168 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8169 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8170
8171 for_each_dbuf_slice(dev_priv, slice)
8172 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8173 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8174 dbuf_min_tracker_val);
8175}
8176
8177void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8178{
8179 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8180 const struct intel_dbuf_state *new_dbuf_state =
8181 intel_atomic_get_new_dbuf_state(state);
8182 const struct intel_dbuf_state *old_dbuf_state =
8183 intel_atomic_get_old_dbuf_state(state);
8184
8185 if (!new_dbuf_state ||
8186 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8187 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8188 return;
8189
8190 WARN_ON(!new_dbuf_state->base.changed);
8191
8192 update_mbus_pre_enable(state);
8193 gen9_dbuf_slices_update(dev_priv,
8194 old_dbuf_state->enabled_slices |
8195 new_dbuf_state->enabled_slices);
8196}
8197
8198void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8199{
8200 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8201 const struct intel_dbuf_state *new_dbuf_state =
8202 intel_atomic_get_new_dbuf_state(state);
8203 const struct intel_dbuf_state *old_dbuf_state =
8204 intel_atomic_get_old_dbuf_state(state);
8205
8206 if (!new_dbuf_state ||
8207 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8208 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8209 return;
8210
8211 WARN_ON(!new_dbuf_state->base.changed);
8212
8213 gen9_dbuf_slices_update(dev_priv,
8214 new_dbuf_state->enabled_slices);
8215}
8216