1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58#include "display/intel_dp.h"
59
60#include "i915_drv.h"
61#include "i915_trace.h"
62#include "intel_display_types.h"
63#include "intel_fbc.h"
64#include "intel_frontbuffer.h"
65#include "intel_psr.h"
66
67
68
69
70
71
72
73
74
75
76
77
78
79static void frontbuffer_flush(struct drm_i915_private *i915,
80 unsigned int frontbuffer_bits,
81 enum fb_op_origin origin)
82{
83
84 spin_lock(&i915->fb_tracking.lock);
85 frontbuffer_bits &= ~i915->fb_tracking.busy_bits;
86 spin_unlock(&i915->fb_tracking.lock);
87
88 if (!frontbuffer_bits)
89 return;
90
91 trace_intel_frontbuffer_flush(frontbuffer_bits, origin);
92
93 might_sleep();
94 intel_edp_drrs_flush(i915, frontbuffer_bits);
95 intel_psr_flush(i915, frontbuffer_bits, origin);
96 intel_fbc_flush(i915, frontbuffer_bits, origin);
97}
98
99
100
101
102
103
104
105
106
107
108
109
110
111void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915,
112 unsigned frontbuffer_bits)
113{
114 spin_lock(&i915->fb_tracking.lock);
115 i915->fb_tracking.flip_bits |= frontbuffer_bits;
116
117 i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
118 spin_unlock(&i915->fb_tracking.lock);
119}
120
121
122
123
124
125
126
127
128
129
130
131void intel_frontbuffer_flip_complete(struct drm_i915_private *i915,
132 unsigned frontbuffer_bits)
133{
134 spin_lock(&i915->fb_tracking.lock);
135
136 frontbuffer_bits &= i915->fb_tracking.flip_bits;
137 i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
138 spin_unlock(&i915->fb_tracking.lock);
139
140 if (frontbuffer_bits)
141 frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
142}
143
144
145
146
147
148
149
150
151
152
153
154
155void intel_frontbuffer_flip(struct drm_i915_private *i915,
156 unsigned frontbuffer_bits)
157{
158 spin_lock(&i915->fb_tracking.lock);
159
160 i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
161 spin_unlock(&i915->fb_tracking.lock);
162
163 frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
164}
165
166void __intel_fb_invalidate(struct intel_frontbuffer *front,
167 enum fb_op_origin origin,
168 unsigned int frontbuffer_bits)
169{
170 struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
171
172 if (origin == ORIGIN_CS) {
173 spin_lock(&i915->fb_tracking.lock);
174 i915->fb_tracking.busy_bits |= frontbuffer_bits;
175 i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
176 spin_unlock(&i915->fb_tracking.lock);
177 }
178
179 trace_intel_frontbuffer_invalidate(frontbuffer_bits, origin);
180
181 might_sleep();
182 intel_psr_invalidate(i915, frontbuffer_bits, origin);
183 intel_edp_drrs_invalidate(i915, frontbuffer_bits);
184 intel_fbc_invalidate(i915, frontbuffer_bits, origin);
185}
186
187void __intel_fb_flush(struct intel_frontbuffer *front,
188 enum fb_op_origin origin,
189 unsigned int frontbuffer_bits)
190{
191 struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
192
193 if (origin == ORIGIN_CS) {
194 spin_lock(&i915->fb_tracking.lock);
195
196 frontbuffer_bits &= i915->fb_tracking.busy_bits;
197 i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
198 spin_unlock(&i915->fb_tracking.lock);
199 }
200
201 if (frontbuffer_bits)
202 frontbuffer_flush(i915, frontbuffer_bits, origin);
203}
204
205static int frontbuffer_active(struct i915_active *ref)
206{
207 struct intel_frontbuffer *front =
208 container_of(ref, typeof(*front), write);
209
210 kref_get(&front->ref);
211 return 0;
212}
213
214static void frontbuffer_retire(struct i915_active *ref)
215{
216 struct intel_frontbuffer *front =
217 container_of(ref, typeof(*front), write);
218
219 intel_frontbuffer_flush(front, ORIGIN_CS);
220 intel_frontbuffer_put(front);
221}
222
223static void frontbuffer_release(struct kref *ref)
224 __releases(&to_i915(front->obj->base.dev)->fb_tracking.lock)
225{
226 struct intel_frontbuffer *front =
227 container_of(ref, typeof(*front), ref);
228 struct drm_i915_gem_object *obj = front->obj;
229 struct i915_vma *vma;
230
231 drm_WARN_ON(obj->base.dev, atomic_read(&front->bits));
232
233 spin_lock(&obj->vma.lock);
234 for_each_ggtt_vma(vma, obj) {
235 i915_vma_clear_scanout(vma);
236 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
237 }
238 spin_unlock(&obj->vma.lock);
239
240 RCU_INIT_POINTER(obj->frontbuffer, NULL);
241 spin_unlock(&to_i915(obj->base.dev)->fb_tracking.lock);
242
243 i915_active_fini(&front->write);
244
245 i915_gem_object_put(obj);
246 kfree_rcu(front, rcu);
247}
248
249struct intel_frontbuffer *
250intel_frontbuffer_get(struct drm_i915_gem_object *obj)
251{
252 struct drm_i915_private *i915 = to_i915(obj->base.dev);
253 struct intel_frontbuffer *front;
254
255 front = __intel_frontbuffer_get(obj);
256 if (front)
257 return front;
258
259 front = kmalloc(sizeof(*front), GFP_KERNEL);
260 if (!front)
261 return NULL;
262
263 front->obj = obj;
264 kref_init(&front->ref);
265 atomic_set(&front->bits, 0);
266 i915_active_init(&front->write,
267 frontbuffer_active,
268 frontbuffer_retire,
269 I915_ACTIVE_RETIRE_SLEEPS);
270
271 spin_lock(&i915->fb_tracking.lock);
272 if (rcu_access_pointer(obj->frontbuffer)) {
273 kfree(front);
274 front = rcu_dereference_protected(obj->frontbuffer, true);
275 kref_get(&front->ref);
276 } else {
277 i915_gem_object_get(obj);
278 rcu_assign_pointer(obj->frontbuffer, front);
279 }
280 spin_unlock(&i915->fb_tracking.lock);
281
282 return front;
283}
284
285void intel_frontbuffer_put(struct intel_frontbuffer *front)
286{
287 kref_put_lock(&front->ref,
288 frontbuffer_release,
289 &to_i915(front->obj->base.dev)->fb_tracking.lock);
290}
291
292
293
294
295
296
297
298
299
300
301void intel_frontbuffer_track(struct intel_frontbuffer *old,
302 struct intel_frontbuffer *new,
303 unsigned int frontbuffer_bits)
304{
305
306
307
308
309
310
311
312 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
313 BITS_PER_TYPE(atomic_t));
314
315 if (old) {
316 drm_WARN_ON(old->obj->base.dev,
317 !(atomic_read(&old->bits) & frontbuffer_bits));
318 atomic_andnot(frontbuffer_bits, &old->bits);
319 }
320
321 if (new) {
322 drm_WARN_ON(new->obj->base.dev,
323 atomic_read(&new->bits) & frontbuffer_bits);
324 atomic_or(frontbuffer_bits, &new->bits);
325 }
326}
327