1#ifndef _INTEL_RINGBUFFER_H_
2#define _INTEL_RINGBUFFER_H_
3
4#include <linux/hashtable.h>
5#include "i915_gem_batch_pool.h"
6#include "i915_gem_request.h"
7#include "i915_gem_timeline.h"
8#include "i915_selftest.h"
9
10#define I915_CMD_HASH_ORDER 9
11
12
13
14
15
16
17#define CACHELINE_BYTES 64
18#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
19
20struct intel_hw_status_page {
21 struct i915_vma *vma;
22 u32 *page_addr;
23 u32 ggtt_offset;
24};
25
26#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
27#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
28
29#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
30#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
31
32#define I915_READ_HEAD(engine) I915_READ(RING_HEAD((engine)->mmio_base))
33#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
34
35#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
36#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
37
38#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
39#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
40
41#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
42#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
43
44
45
46
47#define gen8_semaphore_seqno_size sizeof(uint64_t)
48#define GEN8_SEMAPHORE_OFFSET(__from, __to) \
49 (((__from) * I915_NUM_ENGINES + (__to)) * gen8_semaphore_seqno_size)
50#define GEN8_SIGNAL_OFFSET(__ring, to) \
51 (dev_priv->semaphore->node.start + \
52 GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
53#define GEN8_WAIT_OFFSET(__ring, from) \
54 (dev_priv->semaphore->node.start + \
55 GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
56
57enum intel_engine_hangcheck_action {
58 ENGINE_IDLE = 0,
59 ENGINE_WAIT,
60 ENGINE_ACTIVE_SEQNO,
61 ENGINE_ACTIVE_HEAD,
62 ENGINE_ACTIVE_SUBUNITS,
63 ENGINE_WAIT_KICK,
64 ENGINE_DEAD,
65};
66
67static inline const char *
68hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
69{
70 switch (a) {
71 case ENGINE_IDLE:
72 return "idle";
73 case ENGINE_WAIT:
74 return "wait";
75 case ENGINE_ACTIVE_SEQNO:
76 return "active seqno";
77 case ENGINE_ACTIVE_HEAD:
78 return "active head";
79 case ENGINE_ACTIVE_SUBUNITS:
80 return "active subunits";
81 case ENGINE_WAIT_KICK:
82 return "wait kick";
83 case ENGINE_DEAD:
84 return "dead";
85 }
86
87 return "unknown";
88}
89
90#define I915_MAX_SLICES 3
91#define I915_MAX_SUBSLICES 3
92
93#define instdone_slice_mask(dev_priv__) \
94 (INTEL_GEN(dev_priv__) == 7 ? \
95 1 : INTEL_INFO(dev_priv__)->sseu.slice_mask)
96
97#define instdone_subslice_mask(dev_priv__) \
98 (INTEL_GEN(dev_priv__) == 7 ? \
99 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask)
100
101#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
102 for ((slice__) = 0, (subslice__) = 0; \
103 (slice__) < I915_MAX_SLICES; \
104 (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
105 (slice__) += ((subslice__) == 0)) \
106 for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
107 (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
108
109struct intel_instdone {
110 u32 instdone;
111
112 u32 slice_common;
113 u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
114 u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
115};
116
117struct intel_engine_hangcheck {
118 u64 acthd;
119 u32 seqno;
120 enum intel_engine_hangcheck_action action;
121 unsigned long action_timestamp;
122 int deadlock;
123 struct intel_instdone instdone;
124 struct drm_i915_gem_request *active_request;
125 bool stalled;
126};
127
128struct intel_ring {
129 struct i915_vma *vma;
130 void *vaddr;
131
132 struct list_head request_list;
133
134 u32 head;
135 u32 tail;
136 u32 emit;
137
138 u32 space;
139 u32 size;
140 u32 effective_size;
141};
142
143struct i915_gem_context;
144struct drm_i915_reg_table;
145
146
147
148
149
150
151
152
153
154
155
156
157struct i915_ctx_workarounds {
158 struct i915_wa_ctx_bb {
159 u32 offset;
160 u32 size;
161 } indirect_ctx, per_ctx;
162 struct i915_vma *vma;
163};
164
165struct drm_i915_gem_request;
166struct intel_render_state;
167
168
169
170
171
172enum intel_engine_id {
173 RCS = 0,
174 BCS,
175 VCS,
176 VCS2,
177#define _VCS(n) (VCS + (n))
178 VECS
179};
180
181struct i915_priolist {
182 struct rb_node node;
183 struct list_head requests;
184 int priority;
185};
186
187#define INTEL_ENGINE_CS_MAX_NAME 8
188
189struct intel_engine_cs {
190 struct drm_i915_private *i915;
191 char name[INTEL_ENGINE_CS_MAX_NAME];
192 enum intel_engine_id id;
193 unsigned int uabi_id;
194 unsigned int hw_id;
195 unsigned int guc_id;
196
197 u8 class;
198 u8 instance;
199 u32 context_size;
200 u32 mmio_base;
201 unsigned int irq_shift;
202
203 struct intel_ring *buffer;
204 struct intel_timeline *timeline;
205
206 struct intel_render_state *render_state;
207
208 atomic_t irq_count;
209 unsigned long irq_posted;
210#define ENGINE_IRQ_BREADCRUMB 0
211#define ENGINE_IRQ_EXECLIST 1
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229 struct intel_breadcrumbs {
230 spinlock_t irq_lock;
231 struct intel_wait *irq_wait;
232
233 spinlock_t rb_lock;
234 struct rb_root waiters;
235 struct rb_root signals;
236 struct task_struct *signaler;
237 struct drm_i915_gem_request __rcu *first_signal;
238 struct timer_list fake_irq;
239 struct timer_list hangcheck;
240
241 unsigned int hangcheck_interrupts;
242
243 bool irq_armed : 1;
244 bool irq_enabled : 1;
245 I915_SELFTEST_DECLARE(bool mock : 1);
246 } breadcrumbs;
247
248
249
250
251
252
253 struct i915_gem_batch_pool batch_pool;
254
255 struct intel_hw_status_page status_page;
256 struct i915_ctx_workarounds wa_ctx;
257 struct i915_vma *scratch;
258
259 u32 irq_keep_mask;
260 u32 irq_enable_mask;
261 void (*irq_enable)(struct intel_engine_cs *engine);
262 void (*irq_disable)(struct intel_engine_cs *engine);
263
264 int (*init_hw)(struct intel_engine_cs *engine);
265 void (*reset_hw)(struct intel_engine_cs *engine,
266 struct drm_i915_gem_request *req);
267
268 void (*set_default_submission)(struct intel_engine_cs *engine);
269
270 struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
271 struct i915_gem_context *ctx);
272 void (*context_unpin)(struct intel_engine_cs *engine,
273 struct i915_gem_context *ctx);
274 int (*request_alloc)(struct drm_i915_gem_request *req);
275 int (*init_context)(struct drm_i915_gem_request *req);
276
277 int (*emit_flush)(struct drm_i915_gem_request *request,
278 u32 mode);
279#define EMIT_INVALIDATE BIT(0)
280#define EMIT_FLUSH BIT(1)
281#define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH)
282 int (*emit_bb_start)(struct drm_i915_gem_request *req,
283 u64 offset, u32 length,
284 unsigned int dispatch_flags);
285#define I915_DISPATCH_SECURE BIT(0)
286#define I915_DISPATCH_PINNED BIT(1)
287#define I915_DISPATCH_RS BIT(2)
288 void (*emit_breadcrumb)(struct drm_i915_gem_request *req,
289 u32 *cs);
290 int emit_breadcrumb_sz;
291
292
293
294
295
296
297
298 void (*submit_request)(struct drm_i915_gem_request *req);
299
300
301
302
303
304
305
306 void (*schedule)(struct drm_i915_gem_request *request,
307 int priority);
308
309
310
311
312
313
314
315 void (*irq_seqno_barrier)(struct intel_engine_cs *engine);
316 void (*cleanup)(struct intel_engine_cs *engine);
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355 struct {
356 union {
357#define GEN6_SEMAPHORE_LAST VECS_HW
358#define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1)
359#define GEN6_SEMAPHORES_MASK GENMASK(GEN6_SEMAPHORE_LAST, 0)
360 struct {
361
362 u32 wait[GEN6_NUM_SEMAPHORES];
363
364 i915_reg_t signal[GEN6_NUM_SEMAPHORES];
365 } mbox;
366 u64 signal_ggtt[I915_NUM_ENGINES];
367 };
368
369
370 int (*sync_to)(struct drm_i915_gem_request *req,
371 struct drm_i915_gem_request *signal);
372 u32 *(*signal)(struct drm_i915_gem_request *req, u32 *cs);
373 } semaphore;
374
375
376 struct tasklet_struct irq_tasklet;
377 struct i915_priolist default_priolist;
378 bool no_priolist;
379 struct execlist_port {
380 struct drm_i915_gem_request *request_count;
381#define EXECLIST_COUNT_BITS 2
382#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
383#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
384#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
385#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
386#define port_set(p, packed) ((p)->request_count = (packed))
387#define port_isset(p) ((p)->request_count)
388#define port_index(p, e) ((p) - (e)->execlist_port)
389 GEM_DEBUG_DECL(u32 context_id);
390 } execlist_port[2];
391 struct rb_root execlist_queue;
392 struct rb_node *execlist_first;
393 unsigned int fw_domains;
394
395
396
397
398
399
400
401
402
403
404 struct i915_gem_context *last_retired_context;
405
406
407
408
409
410
411
412 struct i915_gem_context *legacy_active_context;
413
414
415 struct atomic_notifier_head context_status_notifier;
416
417 struct intel_engine_hangcheck hangcheck;
418
419 bool needs_cmd_parser;
420
421
422
423
424
425 DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
426
427
428
429
430 const struct drm_i915_reg_table *reg_tables;
431 int reg_table_count;
432
433
434
435
436
437
438
439
440
441
442
443 u32 (*get_cmd_length_mask)(u32 cmd_header);
444};
445
446static inline unsigned int
447intel_engine_flag(const struct intel_engine_cs *engine)
448{
449 return BIT(engine->id);
450}
451
452static inline u32
453intel_read_status_page(struct intel_engine_cs *engine, int reg)
454{
455
456 return READ_ONCE(engine->status_page.page_addr[reg]);
457}
458
459static inline void
460intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
461{
462
463
464
465
466
467 if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
468 mb();
469 clflush(&engine->status_page.page_addr[reg]);
470 engine->status_page.page_addr[reg] = value;
471 clflush(&engine->status_page.page_addr[reg]);
472 mb();
473 } else {
474 WRITE_ONCE(engine->status_page.page_addr[reg], value);
475 }
476}
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494#define I915_GEM_HWS_INDEX 0x30
495#define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
496#define I915_GEM_HWS_SCRATCH_INDEX 0x40
497#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
498
499struct intel_ring *
500intel_engine_create_ring(struct intel_engine_cs *engine, int size);
501int intel_ring_pin(struct intel_ring *ring,
502 struct drm_i915_private *i915,
503 unsigned int offset_bias);
504void intel_ring_reset(struct intel_ring *ring, u32 tail);
505unsigned int intel_ring_update_space(struct intel_ring *ring);
506void intel_ring_unpin(struct intel_ring *ring);
507void intel_ring_free(struct intel_ring *ring);
508
509void intel_engine_stop(struct intel_engine_cs *engine);
510void intel_engine_cleanup(struct intel_engine_cs *engine);
511
512void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
513
514int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
515
516u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
517 unsigned int n);
518
519static inline void
520intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
521{
522
523
524
525
526
527
528
529
530 GEM_BUG_ON((req->ring->vaddr + req->ring->emit) != cs);
531}
532
533static inline u32
534intel_ring_wrap(const struct intel_ring *ring, u32 pos)
535{
536 return pos & (ring->size - 1);
537}
538
539static inline u32
540intel_ring_offset(const struct drm_i915_gem_request *req, void *addr)
541{
542
543 u32 offset = addr - req->ring->vaddr;
544 GEM_BUG_ON(offset > req->ring->size);
545 return intel_ring_wrap(req->ring, offset);
546}
547
548static inline void
549assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
550{
551
552
553
554
555 GEM_BUG_ON(!IS_ALIGNED(tail, 8));
556 GEM_BUG_ON(tail >= ring->size);
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572#define cacheline(a) round_down(a, CACHELINE_BYTES)
573 GEM_BUG_ON(cacheline(tail) == cacheline(ring->head) &&
574 tail < ring->head);
575#undef cacheline
576}
577
578static inline unsigned int
579intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
580{
581
582
583
584
585
586
587 assert_ring_tail_valid(ring, tail);
588 ring->tail = tail;
589 return tail;
590}
591
592void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno);
593
594void intel_engine_setup_common(struct intel_engine_cs *engine);
595int intel_engine_init_common(struct intel_engine_cs *engine);
596int intel_engine_create_scratch(struct intel_engine_cs *engine, int size);
597void intel_engine_cleanup_common(struct intel_engine_cs *engine);
598
599int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
600int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
601int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
602int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
603
604u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
605u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine);
606
607static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
608{
609 return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
610}
611
612static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
613{
614
615
616
617
618
619
620
621 return READ_ONCE(engine->timeline->seqno);
622}
623
624int init_workarounds_ring(struct intel_engine_cs *engine);
625int intel_ring_workarounds_emit(struct drm_i915_gem_request *req);
626
627void intel_engine_get_instdone(struct intel_engine_cs *engine,
628 struct intel_instdone *instdone);
629
630
631
632
633
634
635
636
637#define MIN_SPACE_FOR_ADD_REQUEST 336
638
639static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
640{
641 return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
642}
643
644
645int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
646
647static inline void intel_wait_init(struct intel_wait *wait,
648 struct drm_i915_gem_request *rq)
649{
650 wait->tsk = current;
651 wait->request = rq;
652}
653
654static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno)
655{
656 wait->tsk = current;
657 wait->seqno = seqno;
658}
659
660static inline bool intel_wait_has_seqno(const struct intel_wait *wait)
661{
662 return wait->seqno;
663}
664
665static inline bool
666intel_wait_update_seqno(struct intel_wait *wait, u32 seqno)
667{
668 wait->seqno = seqno;
669 return intel_wait_has_seqno(wait);
670}
671
672static inline bool
673intel_wait_update_request(struct intel_wait *wait,
674 const struct drm_i915_gem_request *rq)
675{
676 return intel_wait_update_seqno(wait, i915_gem_request_global_seqno(rq));
677}
678
679static inline bool
680intel_wait_check_seqno(const struct intel_wait *wait, u32 seqno)
681{
682 return wait->seqno == seqno;
683}
684
685static inline bool
686intel_wait_check_request(const struct intel_wait *wait,
687 const struct drm_i915_gem_request *rq)
688{
689 return intel_wait_check_seqno(wait, i915_gem_request_global_seqno(rq));
690}
691
692static inline bool intel_wait_complete(const struct intel_wait *wait)
693{
694 return RB_EMPTY_NODE(&wait->node);
695}
696
697bool intel_engine_add_wait(struct intel_engine_cs *engine,
698 struct intel_wait *wait);
699void intel_engine_remove_wait(struct intel_engine_cs *engine,
700 struct intel_wait *wait);
701void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
702 bool wakeup);
703void intel_engine_cancel_signaling(struct drm_i915_gem_request *request);
704
705static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
706{
707 return READ_ONCE(engine->breadcrumbs.irq_wait);
708}
709
710unsigned int intel_engine_wakeup(struct intel_engine_cs *engine);
711#define ENGINE_WAKEUP_WAITER BIT(0)
712#define ENGINE_WAKEUP_ASLEEP BIT(1)
713
714void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
715void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
716
717void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
718void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
719bool intel_breadcrumbs_busy(struct intel_engine_cs *engine);
720
721static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
722{
723 memset(batch, 0, 6 * sizeof(u32));
724
725 batch[0] = GFX_OP_PIPE_CONTROL(6);
726 batch[1] = flags;
727 batch[2] = offset;
728
729 return batch + 6;
730}
731
732bool intel_engine_is_idle(struct intel_engine_cs *engine);
733bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
734
735void intel_engines_mark_idle(struct drm_i915_private *i915);
736void intel_engines_reset_default_submission(struct drm_i915_private *i915);
737
738static inline bool
739__intel_engine_can_store_dword(unsigned int gen, unsigned int class)
740{
741 if (gen <= 2)
742 return false;
743
744 if (gen == 6 && class == VIDEO_DECODE_CLASS)
745 return false;
746
747 return true;
748}
749
750#endif
751