1
2
3
4
5
6#include <linux/string_helpers.h>
7#include <linux/suspend.h>
8
9#include "i915_drv.h"
10#include "i915_params.h"
11#include "intel_context.h"
12#include "intel_engine_pm.h"
13#include "intel_gt.h"
14#include "intel_gt_clock_utils.h"
15#include "intel_gt_pm.h"
16#include "intel_gt_requests.h"
17#include "intel_llc.h"
18#include "intel_pm.h"
19#include "intel_rc6.h"
20#include "intel_rps.h"
21#include "intel_wakeref.h"
22#include "pxp/intel_pxp_pm.h"
23
24#define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
25
26static void user_forcewake(struct intel_gt *gt, bool suspend)
27{
28 int count = atomic_read(>->user_wakeref);
29
30
31 if (likely(!count))
32 return;
33
34 intel_gt_pm_get(gt);
35 if (suspend) {
36 GEM_BUG_ON(count > atomic_read(>->wakeref.count));
37 atomic_sub(count, >->wakeref.count);
38 } else {
39 atomic_add(count, >->wakeref.count);
40 }
41 intel_gt_pm_put(gt);
42}
43
44static void runtime_begin(struct intel_gt *gt)
45{
46 local_irq_disable();
47 write_seqcount_begin(>->stats.lock);
48 gt->stats.start = ktime_get();
49 gt->stats.active = true;
50 write_seqcount_end(>->stats.lock);
51 local_irq_enable();
52}
53
54static void runtime_end(struct intel_gt *gt)
55{
56 local_irq_disable();
57 write_seqcount_begin(>->stats.lock);
58 gt->stats.active = false;
59 gt->stats.total =
60 ktime_add(gt->stats.total,
61 ktime_sub(ktime_get(), gt->stats.start));
62 write_seqcount_end(>->stats.lock);
63 local_irq_enable();
64}
65
66static int __gt_unpark(struct intel_wakeref *wf)
67{
68 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
69 struct drm_i915_private *i915 = gt->i915;
70
71 GT_TRACE(gt, "\n");
72
73
74
75
76
77
78
79
80
81
82
83
84 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
85 GEM_BUG_ON(!gt->awake);
86
87 intel_rc6_unpark(>->rc6);
88 intel_rps_unpark(>->rps);
89 i915_pmu_gt_unparked(i915);
90 intel_guc_busyness_unpark(gt);
91
92 intel_gt_unpark_requests(gt);
93 runtime_begin(gt);
94
95 return 0;
96}
97
98static int __gt_park(struct intel_wakeref *wf)
99{
100 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
101 intel_wakeref_t wakeref = fetch_and_zero(>->awake);
102 struct drm_i915_private *i915 = gt->i915;
103
104 GT_TRACE(gt, "\n");
105
106 runtime_end(gt);
107 intel_gt_park_requests(gt);
108
109 intel_guc_busyness_park(gt);
110 i915_vma_parked(gt);
111 i915_pmu_gt_parked(i915);
112 intel_rps_park(>->rps);
113 intel_rc6_park(>->rc6);
114
115
116 intel_synchronize_irq(i915);
117
118
119 GEM_BUG_ON(!wakeref);
120 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
121
122 return 0;
123}
124
125static const struct intel_wakeref_ops wf_ops = {
126 .get = __gt_unpark,
127 .put = __gt_park,
128};
129
130void intel_gt_pm_init_early(struct intel_gt *gt)
131{
132
133
134
135
136
137
138
139 intel_wakeref_init(>->wakeref, >->i915->runtime_pm, &wf_ops);
140 seqcount_mutex_init(>->stats.lock, >->wakeref.mutex);
141}
142
143void intel_gt_pm_init(struct intel_gt *gt)
144{
145
146
147
148
149
150 intel_rc6_init(>->rc6);
151 intel_rps_init(>->rps);
152}
153
154static bool reset_engines(struct intel_gt *gt)
155{
156 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
157 return false;
158
159 return __intel_gt_reset(gt, ALL_ENGINES) == 0;
160}
161
162static void gt_sanitize(struct intel_gt *gt, bool force)
163{
164 struct intel_engine_cs *engine;
165 enum intel_engine_id id;
166 intel_wakeref_t wakeref;
167
168 GT_TRACE(gt, "force:%s", str_yes_no(force));
169
170
171 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
172 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
173
174 intel_gt_check_clock_frequency(gt);
175
176
177
178
179
180
181
182 if (intel_gt_is_wedged(gt))
183 intel_gt_unset_wedged(gt);
184
185
186 intel_uc_reset_prepare(>->uc);
187
188 for_each_engine(engine, gt, id) {
189 if (engine->reset.prepare)
190 engine->reset.prepare(engine);
191
192 if (engine->sanitize)
193 engine->sanitize(engine);
194 }
195
196 if (reset_engines(gt) || force) {
197 for_each_engine(engine, gt, id)
198 __intel_engine_reset(engine, false);
199 }
200
201 intel_uc_reset(>->uc, false);
202
203 for_each_engine(engine, gt, id)
204 if (engine->reset.finish)
205 engine->reset.finish(engine);
206
207 intel_rps_sanitize(>->rps);
208
209 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
210 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
211}
212
213void intel_gt_pm_fini(struct intel_gt *gt)
214{
215 intel_rc6_fini(>->rc6);
216}
217
218int intel_gt_resume(struct intel_gt *gt)
219{
220 struct intel_engine_cs *engine;
221 enum intel_engine_id id;
222 int err;
223
224 err = intel_gt_has_unrecoverable_error(gt);
225 if (err)
226 return err;
227
228 GT_TRACE(gt, "\n");
229
230
231
232
233
234
235
236 gt_sanitize(gt, true);
237
238 intel_gt_pm_get(gt);
239
240 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
241 intel_rc6_sanitize(>->rc6);
242 if (intel_gt_is_wedged(gt)) {
243 err = -EIO;
244 goto out_fw;
245 }
246
247
248 err = intel_gt_init_hw(gt);
249 if (err) {
250 i915_probe_error(gt->i915,
251 "Failed to initialize GPU, declaring it wedged!\n");
252 goto err_wedged;
253 }
254
255 intel_uc_reset_finish(>->uc);
256
257 intel_rps_enable(>->rps);
258 intel_llc_enable(>->llc);
259
260 for_each_engine(engine, gt, id) {
261 intel_engine_pm_get(engine);
262
263 engine->serial++;
264 err = intel_engine_resume(engine);
265
266 intel_engine_pm_put(engine);
267 if (err) {
268 drm_err(>->i915->drm,
269 "Failed to restart %s (%d)\n",
270 engine->name, err);
271 goto err_wedged;
272 }
273 }
274
275 intel_rc6_enable(>->rc6);
276
277 intel_uc_resume(>->uc);
278
279 intel_pxp_resume(>->pxp);
280
281 user_forcewake(gt, false);
282
283out_fw:
284 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
285 intel_gt_pm_put(gt);
286 return err;
287
288err_wedged:
289 intel_gt_set_wedged(gt);
290 goto out_fw;
291}
292
293static void wait_for_suspend(struct intel_gt *gt)
294{
295 if (!intel_gt_pm_is_awake(gt))
296 return;
297
298 if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
299
300
301
302
303 intel_gt_set_wedged(gt);
304 intel_gt_retire_requests(gt);
305 }
306
307 intel_gt_pm_wait_for_idle(gt);
308}
309
310void intel_gt_suspend_prepare(struct intel_gt *gt)
311{
312 user_forcewake(gt, true);
313 wait_for_suspend(gt);
314
315 intel_pxp_suspend_prepare(>->pxp);
316}
317
318static suspend_state_t pm_suspend_target(void)
319{
320#if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
321 return pm_suspend_target_state;
322#else
323 return PM_SUSPEND_TO_IDLE;
324#endif
325}
326
327void intel_gt_suspend_late(struct intel_gt *gt)
328{
329 intel_wakeref_t wakeref;
330
331
332 wait_for_suspend(gt);
333
334 if (is_mock_gt(gt))
335 return;
336
337 GEM_BUG_ON(gt->awake);
338
339 intel_uc_suspend(>->uc);
340 intel_pxp_suspend(>->pxp);
341
342
343
344
345
346
347
348
349
350
351
352 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
353 return;
354
355 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
356 intel_rps_disable(>->rps);
357 intel_rc6_disable(>->rc6);
358 intel_llc_disable(>->llc);
359 }
360
361 gt_sanitize(gt, false);
362
363 GT_TRACE(gt, "\n");
364}
365
366void intel_gt_runtime_suspend(struct intel_gt *gt)
367{
368 intel_pxp_runtime_suspend(>->pxp);
369 intel_uc_runtime_suspend(>->uc);
370
371 GT_TRACE(gt, "\n");
372}
373
374int intel_gt_runtime_resume(struct intel_gt *gt)
375{
376 int ret;
377
378 GT_TRACE(gt, "\n");
379 intel_gt_init_swizzling(gt);
380 intel_ggtt_restore_fences(gt->ggtt);
381
382 ret = intel_uc_runtime_resume(>->uc);
383 if (ret)
384 return ret;
385
386 intel_pxp_runtime_resume(>->pxp);
387
388 return 0;
389}
390
391static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
392{
393 ktime_t total = gt->stats.total;
394
395 if (gt->stats.active)
396 total = ktime_add(total,
397 ktime_sub(ktime_get(), gt->stats.start));
398
399 return total;
400}
401
402ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
403{
404 unsigned int seq;
405 ktime_t total;
406
407 do {
408 seq = read_seqcount_begin(>->stats.lock);
409 total = __intel_gt_get_awake_time(gt);
410 } while (read_seqcount_retry(>->stats.lock, seq));
411
412 return total;
413}
414
415#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
416#include "selftest_gt_pm.c"
417#endif
418