1
2
3
4
5
6
7#include <linux/sort.h>
8
9#include "i915_drv.h"
10
11#include "intel_gt_requests.h"
12#include "i915_selftest.h"
13#include "selftest_engine_heartbeat.h"
14
15static int timeline_sync(struct intel_timeline *tl)
16{
17 struct dma_fence *fence;
18 long timeout;
19
20 fence = i915_active_fence_get(&tl->last_request);
21 if (!fence)
22 return 0;
23
24 timeout = dma_fence_wait_timeout(fence, true, HZ / 2);
25 dma_fence_put(fence);
26 if (timeout < 0)
27 return timeout;
28
29 return 0;
30}
31
32static int engine_sync_barrier(struct intel_engine_cs *engine)
33{
34 return timeline_sync(engine->kernel_context->timeline);
35}
36
37struct pulse {
38 struct i915_active active;
39 struct kref kref;
40};
41
42static int pulse_active(struct i915_active *active)
43{
44 kref_get(&container_of(active, struct pulse, active)->kref);
45 return 0;
46}
47
48static void pulse_free(struct kref *kref)
49{
50 kfree(container_of(kref, struct pulse, kref));
51}
52
53static void pulse_put(struct pulse *p)
54{
55 kref_put(&p->kref, pulse_free);
56}
57
58static void pulse_retire(struct i915_active *active)
59{
60 pulse_put(container_of(active, struct pulse, active));
61}
62
63static struct pulse *pulse_create(void)
64{
65 struct pulse *p;
66
67 p = kmalloc(sizeof(*p), GFP_KERNEL);
68 if (!p)
69 return p;
70
71 kref_init(&p->kref);
72 i915_active_init(&p->active, pulse_active, pulse_retire);
73
74 return p;
75}
76
77static void pulse_unlock_wait(struct pulse *p)
78{
79 i915_active_unlock_wait(&p->active);
80}
81
82static int __live_idle_pulse(struct intel_engine_cs *engine,
83 int (*fn)(struct intel_engine_cs *cs))
84{
85 struct pulse *p;
86 int err;
87
88 GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
89
90 p = pulse_create();
91 if (!p)
92 return -ENOMEM;
93
94 err = i915_active_acquire(&p->active);
95 if (err)
96 goto out;
97
98 err = i915_active_acquire_preallocate_barrier(&p->active, engine);
99 if (err) {
100 i915_active_release(&p->active);
101 goto out;
102 }
103
104 i915_active_acquire_barrier(&p->active);
105 i915_active_release(&p->active);
106
107 GEM_BUG_ON(i915_active_is_idle(&p->active));
108 GEM_BUG_ON(llist_empty(&engine->barrier_tasks));
109
110 err = fn(engine);
111 if (err)
112 goto out;
113
114 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
115
116 if (engine_sync_barrier(engine)) {
117 struct drm_printer m = drm_err_printer("pulse");
118
119 pr_err("%s: no heartbeat pulse?\n", engine->name);
120 intel_engine_dump(engine, &m, "%s", engine->name);
121
122 err = -ETIME;
123 goto out;
124 }
125
126 GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial);
127
128 pulse_unlock_wait(p);
129
130 if (!i915_active_is_idle(&p->active)) {
131 struct drm_printer m = drm_err_printer("pulse");
132
133 pr_err("%s: heartbeat pulse did not flush idle tasks\n",
134 engine->name);
135 i915_active_print(&p->active, &m);
136
137 err = -EINVAL;
138 goto out;
139 }
140
141out:
142 pulse_put(p);
143 return err;
144}
145
146static int live_idle_flush(void *arg)
147{
148 struct intel_gt *gt = arg;
149 struct intel_engine_cs *engine;
150 enum intel_engine_id id;
151 int err = 0;
152
153
154
155 for_each_engine(engine, gt, id) {
156 st_engine_heartbeat_disable(engine);
157 err = __live_idle_pulse(engine, intel_engine_flush_barriers);
158 st_engine_heartbeat_enable(engine);
159 if (err)
160 break;
161 }
162
163 return err;
164}
165
166static int live_idle_pulse(void *arg)
167{
168 struct intel_gt *gt = arg;
169 struct intel_engine_cs *engine;
170 enum intel_engine_id id;
171 int err = 0;
172
173
174
175 for_each_engine(engine, gt, id) {
176 st_engine_heartbeat_disable(engine);
177 err = __live_idle_pulse(engine, intel_engine_pulse);
178 st_engine_heartbeat_enable(engine);
179 if (err && err != -ENODEV)
180 break;
181
182 err = 0;
183 }
184
185 return err;
186}
187
188static int cmp_u32(const void *_a, const void *_b)
189{
190 const u32 *a = _a, *b = _b;
191
192 return *a - *b;
193}
194
195static int __live_heartbeat_fast(struct intel_engine_cs *engine)
196{
197 struct intel_context *ce;
198 struct i915_request *rq;
199 ktime_t t0, t1;
200 u32 times[5];
201 int err;
202 int i;
203
204 ce = intel_context_create(engine);
205 if (IS_ERR(ce))
206 return PTR_ERR(ce);
207
208 intel_engine_pm_get(engine);
209
210 err = intel_engine_set_heartbeat(engine, 1);
211 if (err)
212 goto err_pm;
213
214 for (i = 0; i < ARRAY_SIZE(times); i++) {
215
216 do {
217 while (READ_ONCE(engine->heartbeat.systole))
218 flush_delayed_work(&engine->heartbeat.work);
219
220 engine->serial++;
221 flush_delayed_work(&engine->heartbeat.work);
222 if (!delayed_work_pending(&engine->heartbeat.work)) {
223 pr_err("%s: heartbeat did not start\n",
224 engine->name);
225 err = -EINVAL;
226 goto err_pm;
227 }
228
229 rcu_read_lock();
230 rq = READ_ONCE(engine->heartbeat.systole);
231 if (rq)
232 rq = i915_request_get_rcu(rq);
233 rcu_read_unlock();
234 } while (!rq);
235
236 t0 = ktime_get();
237 while (rq == READ_ONCE(engine->heartbeat.systole))
238 yield();
239 t1 = ktime_get();
240
241 i915_request_put(rq);
242 times[i] = ktime_us_delta(t1, t0);
243 }
244
245 sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
246
247 pr_info("%s: Heartbeat delay: %uus [%u, %u]\n",
248 engine->name,
249 times[ARRAY_SIZE(times) / 2],
250 times[0],
251 times[ARRAY_SIZE(times) - 1]);
252
253
254 if (times[ARRAY_SIZE(times) / 2] > jiffies_to_usecs(6)) {
255 pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n",
256 engine->name,
257 times[ARRAY_SIZE(times) / 2],
258 jiffies_to_usecs(6));
259 err = -EINVAL;
260 }
261
262 intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
263err_pm:
264 intel_engine_pm_put(engine);
265 intel_context_put(ce);
266 return err;
267}
268
269static int live_heartbeat_fast(void *arg)
270{
271 struct intel_gt *gt = arg;
272 struct intel_engine_cs *engine;
273 enum intel_engine_id id;
274 int err = 0;
275
276
277 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
278 return 0;
279
280 for_each_engine(engine, gt, id) {
281 err = __live_heartbeat_fast(engine);
282 if (err)
283 break;
284 }
285
286 return err;
287}
288
289static int __live_heartbeat_off(struct intel_engine_cs *engine)
290{
291 int err;
292
293 intel_engine_pm_get(engine);
294
295 engine->serial++;
296 flush_delayed_work(&engine->heartbeat.work);
297 if (!delayed_work_pending(&engine->heartbeat.work)) {
298 pr_err("%s: heartbeat not running\n",
299 engine->name);
300 err = -EINVAL;
301 goto err_pm;
302 }
303
304 err = intel_engine_set_heartbeat(engine, 0);
305 if (err)
306 goto err_pm;
307
308 engine->serial++;
309 flush_delayed_work(&engine->heartbeat.work);
310 if (delayed_work_pending(&engine->heartbeat.work)) {
311 pr_err("%s: heartbeat still running\n",
312 engine->name);
313 err = -EINVAL;
314 goto err_beat;
315 }
316
317 if (READ_ONCE(engine->heartbeat.systole)) {
318 pr_err("%s: heartbeat still allocated\n",
319 engine->name);
320 err = -EINVAL;
321 goto err_beat;
322 }
323
324err_beat:
325 intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
326err_pm:
327 intel_engine_pm_put(engine);
328 return err;
329}
330
331static int live_heartbeat_off(void *arg)
332{
333 struct intel_gt *gt = arg;
334 struct intel_engine_cs *engine;
335 enum intel_engine_id id;
336 int err = 0;
337
338
339 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
340 return 0;
341
342 for_each_engine(engine, gt, id) {
343 if (!intel_engine_has_preemption(engine))
344 continue;
345
346 err = __live_heartbeat_off(engine);
347 if (err)
348 break;
349 }
350
351 return err;
352}
353
354int intel_heartbeat_live_selftests(struct drm_i915_private *i915)
355{
356 static const struct i915_subtest tests[] = {
357 SUBTEST(live_idle_flush),
358 SUBTEST(live_idle_pulse),
359 SUBTEST(live_heartbeat_fast),
360 SUBTEST(live_heartbeat_off),
361 };
362 int saved_hangcheck;
363 int err;
364
365 if (intel_gt_is_wedged(&i915->gt))
366 return 0;
367
368 saved_hangcheck = i915->params.enable_hangcheck;
369 i915->params.enable_hangcheck = INT_MAX;
370
371 err = intel_gt_live_subtests(tests, &i915->gt);
372
373 i915->params.enable_hangcheck = saved_hangcheck;
374 return err;
375}
376
377void st_engine_heartbeat_disable(struct intel_engine_cs *engine)
378{
379 engine->props.heartbeat_interval_ms = 0;
380
381 intel_engine_pm_get(engine);
382 intel_engine_park_heartbeat(engine);
383}
384
385void st_engine_heartbeat_enable(struct intel_engine_cs *engine)
386{
387 intel_engine_pm_put(engine);
388
389 engine->props.heartbeat_interval_ms =
390 engine->defaults.heartbeat_interval_ms;
391}
392
