1
2
3
4
5
6#include <linux/kthread.h>
7#include <trace/events/dma_fence.h>
8#include <uapi/linux/sched/types.h>
9
10#include "i915_drv.h"
11#include "i915_trace.h"
12#include "intel_breadcrumbs.h"
13#include "intel_context.h"
14#include "intel_engine_pm.h"
15#include "intel_gt_pm.h"
16#include "intel_gt_requests.h"
17
18static bool irq_enable(struct intel_breadcrumbs *b)
19{
20 return intel_engine_irq_enable(b->irq_engine);
21}
22
23static void irq_disable(struct intel_breadcrumbs *b)
24{
25 intel_engine_irq_disable(b->irq_engine);
26}
27
28static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
29{
30
31
32
33
34 if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt)))
35 return;
36
37
38
39
40
41
42
43 WRITE_ONCE(b->irq_armed, true);
44
45
46 if (!b->irq_enabled++ && b->irq_enable(b))
47 irq_work_queue(&b->irq_work);
48}
49
50static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
51{
52 if (!b->irq_engine)
53 return;
54
55 spin_lock(&b->irq_lock);
56 if (!b->irq_armed)
57 __intel_breadcrumbs_arm_irq(b);
58 spin_unlock(&b->irq_lock);
59}
60
61static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
62{
63 GEM_BUG_ON(!b->irq_enabled);
64 if (!--b->irq_enabled)
65 b->irq_disable(b);
66
67 WRITE_ONCE(b->irq_armed, false);
68 intel_gt_pm_put_async(b->irq_engine->gt);
69}
70
71static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
72{
73 spin_lock(&b->irq_lock);
74 if (b->irq_armed)
75 __intel_breadcrumbs_disarm_irq(b);
76 spin_unlock(&b->irq_lock);
77}
78
79static void add_signaling_context(struct intel_breadcrumbs *b,
80 struct intel_context *ce)
81{
82 lockdep_assert_held(&ce->signal_lock);
83
84 spin_lock(&b->signalers_lock);
85 list_add_rcu(&ce->signal_link, &b->signalers);
86 spin_unlock(&b->signalers_lock);
87}
88
89static bool remove_signaling_context(struct intel_breadcrumbs *b,
90 struct intel_context *ce)
91{
92 lockdep_assert_held(&ce->signal_lock);
93
94 if (!list_empty(&ce->signals))
95 return false;
96
97 spin_lock(&b->signalers_lock);
98 list_del_rcu(&ce->signal_link);
99 spin_unlock(&b->signalers_lock);
100
101 return true;
102}
103
104__maybe_unused static bool
105check_signal_order(struct intel_context *ce, struct i915_request *rq)
106{
107 if (rq->context != ce)
108 return false;
109
110 if (!list_is_last(&rq->signal_link, &ce->signals) &&
111 i915_seqno_passed(rq->fence.seqno,
112 list_next_entry(rq, signal_link)->fence.seqno))
113 return false;
114
115 if (!list_is_first(&rq->signal_link, &ce->signals) &&
116 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
117 rq->fence.seqno))
118 return false;
119
120 return true;
121}
122
123static bool
124__dma_fence_signal(struct dma_fence *fence)
125{
126 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
127}
128
129static void
130__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
131{
132 fence->timestamp = timestamp;
133 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
134 trace_dma_fence_signaled(fence);
135}
136
137static void
138__dma_fence_signal__notify(struct dma_fence *fence,
139 const struct list_head *list)
140{
141 struct dma_fence_cb *cur, *tmp;
142
143 lockdep_assert_held(fence->lock);
144
145 list_for_each_entry_safe(cur, tmp, list, node) {
146 INIT_LIST_HEAD(&cur->node);
147 cur->func(fence, cur);
148 }
149}
150
151static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
152{
153 if (b->irq_engine)
154 intel_engine_add_retire(b->irq_engine, tl);
155}
156
157static struct llist_node *
158slist_add(struct llist_node *node, struct llist_node *head)
159{
160 node->next = head;
161 return node;
162}
163
164static void signal_irq_work(struct irq_work *work)
165{
166 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
167 const ktime_t timestamp = ktime_get();
168 struct llist_node *signal, *sn;
169 struct intel_context *ce;
170
171 signal = NULL;
172 if (unlikely(!llist_empty(&b->signaled_requests)))
173 signal = llist_del_all(&b->signaled_requests);
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200 if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers))
201 intel_breadcrumbs_disarm_irq(b);
202
203 rcu_read_lock();
204 atomic_inc(&b->signaler_active);
205 list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
206 struct i915_request *rq;
207
208 list_for_each_entry_rcu(rq, &ce->signals, signal_link) {
209 bool release;
210
211 if (!__i915_request_is_complete(rq))
212 break;
213
214 if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL,
215 &rq->fence.flags))
216 break;
217
218
219
220
221
222
223 spin_lock(&ce->signal_lock);
224 list_del_rcu(&rq->signal_link);
225 release = remove_signaling_context(b, ce);
226 spin_unlock(&ce->signal_lock);
227 if (release) {
228 if (intel_timeline_is_last(ce->timeline, rq))
229 add_retire(b, ce->timeline);
230 intel_context_put(ce);
231 }
232
233 if (__dma_fence_signal(&rq->fence))
234
235 signal = slist_add(&rq->signal_node, signal);
236 else
237 i915_request_put(rq);
238 }
239 }
240 atomic_dec(&b->signaler_active);
241 rcu_read_unlock();
242
243 llist_for_each_safe(signal, sn, signal) {
244 struct i915_request *rq =
245 llist_entry(signal, typeof(*rq), signal_node);
246 struct list_head cb_list;
247
248 if (rq->engine->sched_engine->retire_inflight_request_prio)
249 rq->engine->sched_engine->retire_inflight_request_prio(rq);
250
251 spin_lock(&rq->lock);
252 list_replace(&rq->fence.cb_list, &cb_list);
253 __dma_fence_signal__timestamp(&rq->fence, timestamp);
254 __dma_fence_signal__notify(&rq->fence, &cb_list);
255 spin_unlock(&rq->lock);
256
257 i915_request_put(rq);
258 }
259
260 if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))
261 intel_breadcrumbs_arm_irq(b);
262}
263
264struct intel_breadcrumbs *
265intel_breadcrumbs_create(struct intel_engine_cs *irq_engine)
266{
267 struct intel_breadcrumbs *b;
268
269 b = kzalloc(sizeof(*b), GFP_KERNEL);
270 if (!b)
271 return NULL;
272
273 kref_init(&b->ref);
274
275 spin_lock_init(&b->signalers_lock);
276 INIT_LIST_HEAD(&b->signalers);
277 init_llist_head(&b->signaled_requests);
278
279 spin_lock_init(&b->irq_lock);
280 init_irq_work(&b->irq_work, signal_irq_work);
281
282 b->irq_engine = irq_engine;
283 b->irq_enable = irq_enable;
284 b->irq_disable = irq_disable;
285
286 return b;
287}
288
289void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)
290{
291 unsigned long flags;
292
293 if (!b->irq_engine)
294 return;
295
296 spin_lock_irqsave(&b->irq_lock, flags);
297
298 if (b->irq_enabled)
299 b->irq_enable(b);
300 else
301 b->irq_disable(b);
302
303 spin_unlock_irqrestore(&b->irq_lock, flags);
304}
305
306void __intel_breadcrumbs_park(struct intel_breadcrumbs *b)
307{
308 if (!READ_ONCE(b->irq_armed))
309 return;
310
311
312 irq_work_sync(&b->irq_work);
313 while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {
314 local_irq_disable();
315 signal_irq_work(&b->irq_work);
316 local_irq_enable();
317 cond_resched();
318 }
319}
320
321void intel_breadcrumbs_free(struct kref *kref)
322{
323 struct intel_breadcrumbs *b = container_of(kref, typeof(*b), ref);
324
325 irq_work_sync(&b->irq_work);
326 GEM_BUG_ON(!list_empty(&b->signalers));
327 GEM_BUG_ON(b->irq_armed);
328
329 kfree(b);
330}
331
332static void irq_signal_request(struct i915_request *rq,
333 struct intel_breadcrumbs *b)
334{
335 if (!__dma_fence_signal(&rq->fence))
336 return;
337
338 i915_request_get(rq);
339 if (llist_add(&rq->signal_node, &b->signaled_requests))
340 irq_work_queue(&b->irq_work);
341}
342
343static void insert_breadcrumb(struct i915_request *rq)
344{
345 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
346 struct intel_context *ce = rq->context;
347 struct list_head *pos;
348
349 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
350 return;
351
352
353
354
355
356
357 if (__i915_request_is_complete(rq)) {
358 irq_signal_request(rq, b);
359 return;
360 }
361
362 if (list_empty(&ce->signals)) {
363 intel_context_get(ce);
364 add_signaling_context(b, ce);
365 pos = &ce->signals;
366 } else {
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381 list_for_each_prev(pos, &ce->signals) {
382 struct i915_request *it =
383 list_entry(pos, typeof(*it), signal_link);
384
385 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
386 break;
387 }
388 }
389
390 i915_request_get(rq);
391 list_add_rcu(&rq->signal_link, pos);
392 GEM_BUG_ON(!check_signal_order(ce, rq));
393 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));
394 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
395
396
397
398
399
400
401 irq_work_queue(&b->irq_work);
402}
403
404bool i915_request_enable_breadcrumb(struct i915_request *rq)
405{
406 struct intel_context *ce = rq->context;
407
408
409 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
410 return true;
411
412
413
414
415
416
417
418 if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
419 return true;
420
421 spin_lock(&ce->signal_lock);
422 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
423 insert_breadcrumb(rq);
424 spin_unlock(&ce->signal_lock);
425
426 return true;
427}
428
429void i915_request_cancel_breadcrumb(struct i915_request *rq)
430{
431 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
432 struct intel_context *ce = rq->context;
433 bool release;
434
435 spin_lock(&ce->signal_lock);
436 if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
437 spin_unlock(&ce->signal_lock);
438 return;
439 }
440
441 list_del_rcu(&rq->signal_link);
442 release = remove_signaling_context(b, ce);
443 spin_unlock(&ce->signal_lock);
444 if (release)
445 intel_context_put(ce);
446
447 if (__i915_request_is_complete(rq))
448 irq_signal_request(rq, b);
449
450 i915_request_put(rq);
451}
452
453void intel_context_remove_breadcrumbs(struct intel_context *ce,
454 struct intel_breadcrumbs *b)
455{
456 struct i915_request *rq, *rn;
457 bool release = false;
458 unsigned long flags;
459
460 spin_lock_irqsave(&ce->signal_lock, flags);
461
462 if (list_empty(&ce->signals))
463 goto unlock;
464
465 list_for_each_entry_safe(rq, rn, &ce->signals, signal_link) {
466 GEM_BUG_ON(!__i915_request_is_complete(rq));
467 if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL,
468 &rq->fence.flags))
469 continue;
470
471 list_del_rcu(&rq->signal_link);
472 irq_signal_request(rq, b);
473 i915_request_put(rq);
474 }
475 release = remove_signaling_context(b, ce);
476
477unlock:
478 spin_unlock_irqrestore(&ce->signal_lock, flags);
479 if (release)
480 intel_context_put(ce);
481
482 while (atomic_read(&b->signaler_active))
483 cpu_relax();
484}
485
486static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)
487{
488 struct intel_context *ce;
489 struct i915_request *rq;
490
491 drm_printf(p, "Signals:\n");
492
493 rcu_read_lock();
494 list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
495 list_for_each_entry_rcu(rq, &ce->signals, signal_link)
496 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
497 rq->fence.context, rq->fence.seqno,
498 __i915_request_is_complete(rq) ? "!" :
499 __i915_request_has_started(rq) ? "*" :
500 "",
501 jiffies_to_msecs(jiffies - rq->emitted_jiffies));
502 }
503 rcu_read_unlock();
504}
505
506void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
507 struct drm_printer *p)
508{
509 struct intel_breadcrumbs *b;
510
511 b = engine->breadcrumbs;
512 if (!b)
513 return;
514
515 drm_printf(p, "IRQ: %s\n", enableddisabled(b->irq_armed));
516 if (!list_empty(&b->signalers))
517 print_signals(b, p);
518}
519